From f1af78b0a8fe5d4827f60bc9f06af169a3ed392c Mon Sep 17 00:00:00 2001 From: johnjiang Date: Tue, 20 Nov 2018 14:01:41 +0800 Subject: [PATCH 1/7] Update Launch_F-Stack_on_AWS_EC2_in_one_minute.md Add depends to yum install. --- doc/Launch_F-Stack_on_AWS_EC2_in_one_minute.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/Launch_F-Stack_on_AWS_EC2_in_one_minute.md b/doc/Launch_F-Stack_on_AWS_EC2_in_one_minute.md index cba7d2f6..c6a28aa2 100644 --- a/doc/Launch_F-Stack_on_AWS_EC2_in_one_minute.md +++ b/doc/Launch_F-Stack_on_AWS_EC2_in_one_minute.md @@ -3,7 +3,7 @@ If you have a Redhat7.3 EC2 instanceļ¼Œand then execute the following cmds, you will get the F-Stack server in one minute sudo -i - yum install -y git gcc openssl-devel kernel-devel-$(uname -r) bc numactl-devel + yum install -y git gcc openssl-devel kernel-devel-$(uname -r) bc numactl-devel mkdir make net-tools vim pciutils iproute mkdir /data/f-stack git clone https://github.com/F-Stack/f-stack.git /data/f-stack From afda379ba2a8afa29784907a2b09ed969c1ee95e Mon Sep 17 00:00:00 2001 From: johnjiang Date: Tue, 20 Nov 2018 16:24:16 +0800 Subject: [PATCH 2/7] Update config.ini Modify "idle_sleep" default value to 0. --- config.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.ini b/config.ini index 5b1ee877..0a5a6bea 100644 --- a/config.ini +++ b/config.ini @@ -20,7 +20,7 @@ vlan_strip=1 # sleep when no pkts incomming # unit: microseconds -idle_sleep=100 +idle_sleep=0 # enabled port list # From c1a249e1a63ed2ca8347cbfa7ed9a19f1710a79b Mon Sep 17 00:00:00 2001 From: jfb8856606 Date: Wed, 21 Nov 2018 08:34:11 +0000 Subject: [PATCH 3/7] DPDK: upgrade to 17.11.4 LTS. --- dpdk/MAINTAINERS | 12 +- dpdk/app/proc_info/main.c | 2 +- dpdk/app/test-crypto-perf/cperf_ops.c | 3 + dpdk/app/test-crypto-perf/cperf_test_common.c | 2 +- .../cperf_test_vector_parsing.c | 7 +- dpdk/app/test-crypto-perf/main.c | 14 +- dpdk/app/test-pmd/cmdline.c | 2 +- dpdk/app/test-pmd/cmdline_flow.c | 3 +- dpdk/app/test-pmd/cmdline_tm.c | 16 +- dpdk/app/test-pmd/config.c | 36 +- dpdk/app/test-pmd/parameters.c | 12 +- dpdk/app/test-pmd/testpmd.c | 89 +- dpdk/app/test-pmd/testpmd.h | 2 + dpdk/buildtools/pmdinfogen/Makefile | 2 +- .../config/defconfig_i686-native-linuxapp-icc | 5 - dpdk/doc/api/doxy-html-custom.sh | 0 dpdk/doc/guides/nics/enic.rst | 44 +- dpdk/doc/guides/nics/features/mlx5.ini | 1 + dpdk/doc/guides/nics/mlx5.rst | 6 +- dpdk/doc/guides/nics/nfp.rst | 43 +- dpdk/doc/guides/nics/qede.rst | 13 +- dpdk/doc/guides/rel_notes/release_17_11.rst | 372 +++++ .../guides/testpmd_app_ug/testpmd_funcs.rst | 2 +- dpdk/drivers/bus/dpaa/base/fman/fman.c | 2 + dpdk/drivers/bus/dpaa/base/fman/fman_hw.c | 20 +- dpdk/drivers/bus/dpaa/base/fman/of.c | 5 + dpdk/drivers/bus/dpaa/include/compat.h | 6 + dpdk/drivers/bus/fslmc/fslmc_bus.c | 5 +- dpdk/drivers/bus/pci/linux/pci.c | 16 +- dpdk/drivers/bus/pci/linux/pci_vfio.c | 2 +- dpdk/drivers/bus/pci/pci_common.c | 21 +- dpdk/drivers/bus/pci/rte_bus_pci.h | 3 + dpdk/drivers/bus/vdev/vdev.c | 2 +- .../drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h | 26 +- dpdk/drivers/crypto/qat/qat_crypto.c | 10 +- .../scheduler/rte_cryptodev_scheduler.c | 8 +- .../scheduler/rte_cryptodev_scheduler.h | 2 +- .../crypto/scheduler/scheduler_multicore.c | 53 +- .../scheduler/scheduler_pkt_size_distr.c | 4 +- dpdk/drivers/crypto/scheduler/scheduler_pmd.c | 68 +- .../crypto/scheduler/scheduler_pmd_ops.c | 9 +- .../crypto/scheduler/scheduler_pmd_private.h | 2 +- dpdk/drivers/crypto/zuc/rte_zuc_pmd.c | 108 +- dpdk/drivers/event/dpaa2/dpaa2_eventdev.c | 5 - dpdk/drivers/event/dpaa2/dpaa2_eventdev.h | 1 - dpdk/drivers/event/octeontx/ssovf_evdev.c | 14 +- .../drivers/net/af_packet/rte_eth_af_packet.c | 2 +- dpdk/drivers/net/bnx2x/bnx2x.c | 52 +- dpdk/drivers/net/bnx2x/bnx2x.h | 1 + dpdk/drivers/net/bnx2x/bnx2x_ethdev.c | 214 ++- dpdk/drivers/net/bnx2x/bnx2x_ethdev.h | 3 +- dpdk/drivers/net/bnx2x/elink.c | 344 ++--- dpdk/drivers/net/bnxt/bnxt_ethdev.c | 49 +- dpdk/drivers/net/bnxt/bnxt_filter.c | 14 +- dpdk/drivers/net/bnxt/bnxt_hwrm.c | 65 +- dpdk/drivers/net/bnxt/bnxt_nvm_defs.h | 36 +- dpdk/drivers/net/bnxt/bnxt_rxq.c | 6 +- dpdk/drivers/net/bnxt/bnxt_rxr.c | 6 +- dpdk/drivers/net/bnxt/bnxt_rxr.h | 22 +- dpdk/drivers/net/bnxt/bnxt_stats.c | 3 + dpdk/drivers/net/bnxt/bnxt_txr.c | 51 +- dpdk/drivers/net/bnxt/bnxt_txr.h | 10 + dpdk/drivers/net/bnxt/bnxt_vnic.c | 5 +- dpdk/drivers/net/bnxt/bnxt_vnic.h | 6 +- dpdk/drivers/net/bonding/rte_eth_bond_api.c | 21 +- dpdk/drivers/net/bonding/rte_eth_bond_args.c | 2 +- dpdk/drivers/net/bonding/rte_eth_bond_pmd.c | 42 +- .../net/bonding/rte_eth_bond_private.h | 1 + .../net/bonding/rte_pmd_bond_version.map | 1 + dpdk/drivers/net/cxgbe/base/t4_hw.c | 99 +- dpdk/drivers/net/cxgbe/base/t4fw_interface.h | 8 + dpdk/drivers/net/cxgbe/cxgbe_compat.h | 9 - dpdk/drivers/net/cxgbe/sge.c | 15 +- dpdk/drivers/net/dpaa/dpaa_ethdev.c | 7 +- dpdk/drivers/net/dpaa2/dpaa2_ethdev.c | 11 +- dpdk/drivers/net/dpaa2/mc/dpni.c | 2 +- dpdk/drivers/net/ena/base/ena_plat_dpdk.h | 34 +- dpdk/drivers/net/ena/ena_ethdev.c | 7 +- dpdk/drivers/net/enic/base/vnic_dev.c | 24 +- dpdk/drivers/net/enic/base/vnic_dev.h | 1 + dpdk/drivers/net/enic/enic.h | 1 + dpdk/drivers/net/enic/enic_ethdev.c | 67 + dpdk/drivers/net/enic/enic_main.c | 45 +- dpdk/drivers/net/failsafe/failsafe.c | 4 +- dpdk/drivers/net/failsafe/failsafe_ether.c | 30 + dpdk/drivers/net/failsafe/failsafe_ops.c | 9 +- dpdk/drivers/net/failsafe/failsafe_private.h | 5 + dpdk/drivers/net/i40e/base/i40e_register.h | 24 +- dpdk/drivers/net/i40e/i40e_ethdev.c | 351 +++-- dpdk/drivers/net/i40e/i40e_ethdev.h | 4 +- dpdk/drivers/net/i40e/i40e_flow.c | 2 +- dpdk/drivers/net/i40e/rte_pmd_i40e.c | 7 +- dpdk/drivers/net/ixgbe/ixgbe_ethdev.c | 79 +- dpdk/drivers/net/ixgbe/ixgbe_ethdev.h | 5 + dpdk/drivers/net/ixgbe/ixgbe_fdir.c | 30 +- dpdk/drivers/net/ixgbe/ixgbe_flow.c | 12 +- dpdk/drivers/net/kni/rte_eth_kni.c | 2 +- dpdk/drivers/net/liquidio/lio_ethdev.c | 5 + dpdk/drivers/net/mlx4/mlx4.c | 62 +- dpdk/drivers/net/mlx4/mlx4.h | 4 + dpdk/drivers/net/mlx4/mlx4_ethdev.c | 192 +-- dpdk/drivers/net/mlx4/mlx4_flow.c | 34 +- dpdk/drivers/net/mlx4/mlx4_flow.h | 1 + dpdk/drivers/net/mlx4/mlx4_intr.c | 41 +- dpdk/drivers/net/mlx4/mlx4_rxq.c | 13 + dpdk/drivers/net/mlx4/mlx4_rxtx.c | 3 +- dpdk/drivers/net/mlx4/mlx4_rxtx.h | 2 +- dpdk/drivers/net/mlx5/Makefile | 47 +- dpdk/drivers/net/mlx5/mlx5.c | 539 +++++--- dpdk/drivers/net/mlx5/mlx5.h | 267 ++-- dpdk/drivers/net/mlx5/mlx5_defs.h | 24 +- dpdk/drivers/net/mlx5/mlx5_ethdev.c | 1044 +++++--------- dpdk/drivers/net/mlx5/mlx5_flow.c | 1205 +++++++++-------- dpdk/drivers/net/mlx5/mlx5_mac.c | 49 +- dpdk/drivers/net/mlx5/mlx5_mr.c | 217 +-- dpdk/drivers/net/mlx5/mlx5_rss.c | 178 +-- dpdk/drivers/net/mlx5/mlx5_rxmode.c | 44 +- dpdk/drivers/net/mlx5/mlx5_rxq.c | 712 +++++----- dpdk/drivers/net/mlx5/mlx5_rxtx.c | 127 +- dpdk/drivers/net/mlx5/mlx5_rxtx.h | 181 +-- dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c | 26 +- dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h | 4 +- dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 73 +- dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 10 +- dpdk/drivers/net/mlx5/mlx5_socket.c | 178 ++- dpdk/drivers/net/mlx5/mlx5_stats.c | 248 ++-- dpdk/drivers/net/mlx5/mlx5_trigger.c | 281 ++-- dpdk/drivers/net/mlx5/mlx5_txq.c | 381 +++--- dpdk/drivers/net/mlx5/mlx5_utils.h | 29 +- dpdk/drivers/net/mlx5/mlx5_vlan.c | 116 +- dpdk/drivers/net/mrvl/mrvl_ethdev.c | 12 +- dpdk/drivers/net/nfp/nfp_net.c | 44 +- dpdk/drivers/net/nfp/nfp_nfpu.c | 26 +- dpdk/drivers/net/null/rte_eth_null.c | 2 +- dpdk/drivers/net/octeontx/octeontx_ethdev.c | 12 +- dpdk/drivers/net/pcap/rte_eth_pcap.c | 88 +- dpdk/drivers/net/qede/base/bcm_osal.c | 9 +- dpdk/drivers/net/qede/base/ecore.h | 6 + dpdk/drivers/net/qede/base/ecore_dcbx.c | 5 + dpdk/drivers/net/qede/base/ecore_dcbx_api.h | 1 + dpdk/drivers/net/qede/base/ecore_dev.c | 20 +- dpdk/drivers/net/qede/base/ecore_hsi_common.h | 5 +- dpdk/drivers/net/qede/base/ecore_int.c | 14 +- dpdk/drivers/net/qede/base/ecore_l2.c | 12 +- dpdk/drivers/net/qede/base/ecore_l2_api.h | 2 +- .../drivers/net/qede/base/ecore_sp_commands.c | 29 +- dpdk/drivers/net/qede/base/ecore_sriov.c | 47 +- dpdk/drivers/net/qede/base/ecore_vf.c | 38 +- dpdk/drivers/net/qede/base/ecore_vf.h | 9 + dpdk/drivers/net/qede/base/ecore_vfpf_if.h | 24 +- dpdk/drivers/net/qede/qede_ethdev.c | 538 +++++--- dpdk/drivers/net/qede/qede_ethdev.h | 4 +- dpdk/drivers/net/qede/qede_fdir.c | 7 +- dpdk/drivers/net/qede/qede_main.c | 7 +- dpdk/drivers/net/qede/qede_rxtx.c | 29 +- dpdk/drivers/net/ring/rte_eth_ring.c | 2 +- dpdk/drivers/net/sfc/base/efx_port.c | 2 +- dpdk/drivers/net/sfc/efsys.h | 2 + dpdk/drivers/net/sfc/sfc.c | 2 + dpdk/drivers/net/sfc/sfc_ef10_rx.c | 2 + dpdk/drivers/net/sfc/sfc_ethdev.c | 2 +- dpdk/drivers/net/sfc/sfc_ev.c | 2 +- dpdk/drivers/net/sfc/sfc_flow.c | 9 +- dpdk/drivers/net/sfc/sfc_rx.c | 44 +- dpdk/drivers/net/softnic/rte_eth_softnic.c | 2 +- dpdk/drivers/net/szedata2/rte_eth_szedata2.c | 41 +- dpdk/drivers/net/tap/rte_eth_tap.c | 6 +- dpdk/drivers/net/tap/tap_flow.c | 18 +- dpdk/drivers/net/thunderx/nicvf_ethdev.c | 5 +- dpdk/drivers/net/thunderx/nicvf_rxtx.c | 24 +- dpdk/drivers/net/vhost/rte_eth_vhost.c | 93 +- dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c | 4 + dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c | 4 +- dpdk/examples/exception_path/main.c | 21 +- dpdk/examples/flow_filtering/main.c | 16 + .../ip_pipeline/config/diagram-generator.py | 0 .../config/pipeline-to-core-mapping.py | 0 dpdk/examples/ip_pipeline/config_parse_tm.c | 2 +- dpdk/examples/ipsec-secgw/ipsec-secgw.c | 7 +- dpdk/examples/ipsec-secgw/parser.c | 2 +- dpdk/examples/l2fwd-crypto/main.c | 41 +- dpdk/examples/l3fwd/l3fwd_em.c | 1 - dpdk/examples/l3fwd/l3fwd_lpm.c | 1 - .../performance-thread/common/lthread.c | 3 +- .../performance-thread/common/lthread_api.h | 2 +- .../performance-thread/l3fwd-thread/main.c | 20 +- .../performance-thread/l3fwd-thread/test.sh | 0 .../performance-thread/pthread_shim/main.c | 6 +- .../pthread_shim/pthread_shim.c | 4 +- dpdk/examples/quota_watermark/qw/main.c | 14 +- dpdk/examples/vhost_scsi/scsi.c | 6 +- dpdk/lib/librte_bitratestats/rte_bitrate.c | 6 + dpdk/lib/librte_cryptodev/rte_cryptodev.c | 45 +- .../lib/librte_eal/common/eal_common_memory.c | 43 + .../common/include/arch/ppc_64/rte_atomic.h | 2 +- .../librte_eal/common/include/rte_bitmap.h | 8 +- .../librte_eal/common/include/rte_memory.h | 3 + .../librte_eal/common/include/rte_version.h | 2 +- .../librte_eal/linuxapp/eal/eal_interrupts.c | 2 +- dpdk/lib/librte_eal/linuxapp/eal/eal_memory.c | 63 +- dpdk/lib/librte_eal/linuxapp/eal/eal_thread.c | 4 +- dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c | 10 +- dpdk/lib/librte_eal/linuxapp/kni/compat.h | 5 + .../linuxapp/kni/ethtool/igb/igb_ethtool.c | 7 +- .../linuxapp/kni/ethtool/igb/kcompat.h | 18 +- dpdk/lib/librte_eal/rte_eal_version.map | 1 + dpdk/lib/librte_ether/rte_ethdev.c | 12 + dpdk/lib/librte_ether/rte_ethdev.h | 8 +- dpdk/lib/librte_ether/rte_ethdev_pci.h | 6 +- .../rte_event_eth_rx_adapter.c | 38 +- dpdk/lib/librte_eventdev/rte_event_ring.c | 15 +- dpdk/lib/librte_hash/rte_cuckoo_hash.c | 27 +- dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h | 3 + dpdk/lib/librte_hash/rte_hash.h | 20 +- dpdk/lib/librte_ip_frag/rte_ipv4_reassembly.c | 2 + dpdk/lib/librte_ip_frag/rte_ipv6_reassembly.c | 2 + dpdk/lib/librte_kni/rte_kni.c | 3 + dpdk/lib/librte_mbuf/rte_mbuf.h | 25 +- dpdk/lib/librte_mempool/rte_mempool.c | 15 +- dpdk/lib/librte_metrics/rte_metrics.c | 15 +- dpdk/lib/librte_net/rte_ip.h | 6 + dpdk/lib/librte_pci/rte_pci_version.map | 5 +- dpdk/lib/librte_ring/rte_ring.h | 2 +- dpdk/lib/librte_security/rte_security.c | 3 +- dpdk/lib/librte_vhost/fd_man.c | 32 + dpdk/lib/librte_vhost/fd_man.h | 1 + dpdk/lib/librte_vhost/iotlb.c | 10 +- dpdk/lib/librte_vhost/iotlb.h | 2 +- dpdk/lib/librte_vhost/socket.c | 19 +- dpdk/lib/librte_vhost/vhost.h | 120 +- dpdk/lib/librte_vhost/vhost_user.c | 71 +- dpdk/lib/librte_vhost/virtio_net.c | 41 +- dpdk/mk/rte.sdkconfig.mk | 19 +- dpdk/mk/rte.sdkinstall.mk | 36 +- dpdk/mk/rte.sdkroot.mk | 4 +- dpdk/mk/rte.sdktest.mk | 32 +- dpdk/mk/toolchain/gcc/rte.toolchain-compat.mk | 5 + dpdk/mk/toolchain/gcc/rte.vars.mk | 9 + dpdk/pkg/dpdk.spec | 2 +- dpdk/test/cmdline_test/cmdline_test.py | 0 dpdk/test/test/autotest_runner.py | 135 +- dpdk/test/test/test_cryptodev.c | 4 +- dpdk/test/test/test_distributor_perf.c | 3 +- dpdk/test/test/test_eal_flags.c | 40 +- dpdk/test/test/test_flow_classify.c | 20 +- dpdk/test/test/test_hash_multiwriter.c | 50 +- dpdk/test/test/test_mempool.c | 15 +- dpdk/test/test/test_pmd_ring.c | 2 + dpdk/test/test/test_reorder.c | 47 +- dpdk/test/test/test_table_pipeline.c | 14 +- 250 files changed, 7105 insertions(+), 4721 deletions(-) mode change 100644 => 100755 dpdk/doc/api/doxy-html-custom.sh mode change 100644 => 100755 dpdk/examples/ip_pipeline/config/diagram-generator.py mode change 100644 => 100755 dpdk/examples/ip_pipeline/config/pipeline-to-core-mapping.py mode change 100644 => 100755 dpdk/examples/performance-thread/l3fwd-thread/test.sh mode change 100644 => 100755 dpdk/test/cmdline_test/cmdline_test.py diff --git a/dpdk/MAINTAINERS b/dpdk/MAINTAINERS index f0baeb42..449409a4 100644 --- a/dpdk/MAINTAINERS +++ b/dpdk/MAINTAINERS @@ -155,13 +155,13 @@ F: test/test/test_bitmap.c ARM v7 M: Jan Viktorin -M: Jianbo Liu +M: Gavin Hu F: lib/librte_eal/common/arch/arm/ F: lib/librte_eal/common/include/arch/arm/ ARM v8 M: Jerin Jacob -M: Jianbo Liu +M: Gavin Hu F: lib/librte_eal/common/include/arch/arm/*_64.h F: lib/librte_net/net_crc_neon.h F: lib/librte_acl/acl_run_neon.* @@ -428,14 +428,14 @@ F: drivers/net/fm10k/ F: doc/guides/nics/features/fm10k*.ini Mellanox mlx4 -M: Adrien Mazarguil +M: Matan Azrad +M: Shahaf Shuler F: drivers/net/mlx4/ F: doc/guides/nics/mlx4.rst F: doc/guides/nics/features/mlx4.ini Mellanox mlx5 -M: Adrien Mazarguil -M: Nelio Laranjeiro +M: Shahaf Shuler M: Yongseok Koh F: drivers/net/mlx5/ F: doc/guides/nics/mlx5.rst @@ -446,7 +446,6 @@ M: Jacek Siuda M: Tomasz Duszynski M: Dmitri Epshtein M: Natalie Samsonov -M: Jianbo Liu F: drivers/net/mrvl/ F: doc/guides/nics/mrvl.rst F: doc/guides/nics/features/mrvl.ini @@ -630,7 +629,6 @@ M: Jacek Siuda M: Tomasz Duszynski M: Dmitri Epshtein M: Natalie Samsonov -M: Jianbo Liu F: drivers/crypto/mrvl/ F: doc/guides/cryptodevs/mrvl.rst F: doc/guides/cryptodevs/features/mrvl.ini diff --git a/dpdk/app/proc_info/main.c b/dpdk/app/proc_info/main.c index 64fbbd0f..875d91ea 100644 --- a/dpdk/app/proc_info/main.c +++ b/dpdk/app/proc_info/main.c @@ -188,7 +188,7 @@ proc_info_preparse_args(int argc, char **argv) proc_info_usage(prgname); return -1; } - strncpy(host_id, argv[i+1], sizeof(host_id)); + snprintf(host_id, sizeof(host_id), "%s", argv[i+1]); } } diff --git a/dpdk/app/test-crypto-perf/cperf_ops.c b/dpdk/app/test-crypto-perf/cperf_ops.c index 23d30ca3..0fa1c412 100644 --- a/dpdk/app/test-crypto-perf/cperf_ops.c +++ b/dpdk/app/test-crypto-perf/cperf_ops.c @@ -507,6 +507,7 @@ cperf_create_session(struct rte_mempool *sess_mp, auth_xform.next = NULL; auth_xform.auth.algo = options->auth_algo; auth_xform.auth.op = options->auth_op; + auth_xform.auth.iv.offset = iv_offset; /* auth different than null */ if (options->auth_algo != RTE_CRYPTO_AUTH_NULL) { @@ -561,6 +562,8 @@ cperf_create_session(struct rte_mempool *sess_mp, auth_xform.next = NULL; auth_xform.auth.algo = options->auth_algo; auth_xform.auth.op = options->auth_op; + auth_xform.auth.iv.offset = iv_offset + + cipher_xform.cipher.iv.length; /* auth different than null */ if (options->auth_algo != RTE_CRYPTO_AUTH_NULL) { diff --git a/dpdk/app/test-crypto-perf/cperf_test_common.c b/dpdk/app/test-crypto-perf/cperf_test_common.c index 328744ef..9b08b3f7 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_common.c +++ b/dpdk/app/test-crypto-perf/cperf_test_common.c @@ -119,7 +119,7 @@ mempool_obj_init(struct rte_mempool *mp, op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC; op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; op->sess_type = RTE_CRYPTO_OP_WITH_SESSION; - op->phys_addr = rte_mem_virt2phy(obj); + op->phys_addr = rte_mem_virt2iova(obj); op->mempool = mp; /* Set source buffer */ diff --git a/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c index d4736f9e..56df4e78 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c +++ b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c @@ -534,8 +534,7 @@ parse_file(struct cperf_test_vector *vector, struct cperf_options *opts) if (entry == NULL) return -1; - memset(entry, 0, strlen(line) + 1); - strncpy(entry, line, strlen(line)); + strcpy(entry, line); /* check if entry ends with , or = */ if (entry[strlen(entry) - 1] == ',' @@ -552,8 +551,8 @@ parse_file(struct cperf_test_vector *vector, struct cperf_options *opts) if (entry_extended == NULL) goto err; entry = entry_extended; - - strncat(entry, line, strlen(line)); + /* entry has been allocated accordingly */ + strcpy(&entry[strlen(entry)], line); if (entry[strlen(entry) - 1] != ',') break; diff --git a/dpdk/app/test-crypto-perf/main.c b/dpdk/app/test-crypto-perf/main.c index 29373f5b..13e01218 100644 --- a/dpdk/app/test-crypto-perf/main.c +++ b/dpdk/app/test-crypto-perf/main.c @@ -106,13 +106,19 @@ cperf_initialize_cryptodev(struct cperf_options *opts, uint8_t *enabled_cdevs, nb_lcores = rte_lcore_count() - 1; - if (enabled_cdev_count > nb_lcores) { - printf("Number of capable crypto devices (%d) " - "has to be less or equal to number of slave " - "cores (%d)\n", enabled_cdev_count, nb_lcores); + if (nb_lcores < 1) { + RTE_LOG(ERR, USER1, + "Number of enabled cores need to be higher than 1\n"); return -EINVAL; } + /* + * Use less number of devices, + * if there are more available than cores. + */ + if (enabled_cdev_count > nb_lcores) + enabled_cdev_count = nb_lcores; + /* Create a mempool shared by all the devices */ uint32_t max_sess_size = 0, sess_size; diff --git a/dpdk/app/test-pmd/cmdline.c b/dpdk/app/test-pmd/cmdline.c index b3c3f241..77c11b84 100644 --- a/dpdk/app/test-pmd/cmdline.c +++ b/dpdk/app/test-pmd/cmdline.c @@ -2127,7 +2127,7 @@ cmdline_parse_inst_t cmd_config_rxtx_queue = { .data = NULL, .help_str = "port rxq|txq start|stop", .tokens = { - (void *)&cmd_config_speed_all_port, + (void *)&cmd_config_rxtx_queue_port, (void *)&cmd_config_rxtx_queue_portid, (void *)&cmd_config_rxtx_queue_rxtxq, (void *)&cmd_config_rxtx_queue_qid, diff --git a/dpdk/app/test-pmd/cmdline_flow.c b/dpdk/app/test-pmd/cmdline_flow.c index df16d2ab..35440eaf 100644 --- a/dpdk/app/test-pmd/cmdline_flow.c +++ b/dpdk/app/test-pmd/cmdline_flow.c @@ -2028,7 +2028,7 @@ parse_vc_action_rss_queue(struct context *ctx, const struct token *token, i = ctx->objdata >> 16; if (!strcmp_partial("end", str, len)) { ctx->objdata &= 0xffff; - return len; + goto end; } if (i >= ACTION_RSS_NUM) return -1; @@ -2045,6 +2045,7 @@ parse_vc_action_rss_queue(struct context *ctx, const struct token *token, if (ctx->next_num == RTE_DIM(ctx->next)) return -1; ctx->next[ctx->next_num++] = next; +end: if (!ctx->object) return len; ((struct rte_flow_action_rss *)ctx->object)->num = i; diff --git a/dpdk/app/test-pmd/cmdline_tm.c b/dpdk/app/test-pmd/cmdline_tm.c index 803fae44..d40a427d 100644 --- a/dpdk/app/test-pmd/cmdline_tm.c +++ b/dpdk/app/test-pmd/cmdline_tm.c @@ -1624,10 +1624,12 @@ static void cmd_add_port_tm_nonleaf_node_parsed(void *parsed_result, np.shaper_profile_id = res->shaper_profile_id; np.n_shared_shapers = n_shared_shapers; - if (np.n_shared_shapers) + if (np.n_shared_shapers) { np.shared_shaper_id = &shared_shaper_id[0]; - else - np.shared_shaper_id = NULL; + } else { + free(shared_shaper_id); + shared_shaper_id = NULL; + } np.nonleaf.n_sp_priorities = res->n_sp_priorities; np.stats_mask = res->stats_mask; @@ -1779,10 +1781,12 @@ static void cmd_add_port_tm_leaf_node_parsed(void *parsed_result, np.shaper_profile_id = res->shaper_profile_id; np.n_shared_shapers = n_shared_shapers; - if (np.n_shared_shapers) + if (np.n_shared_shapers) { np.shared_shaper_id = &shared_shaper_id[0]; - else - np.shared_shaper_id = NULL; + } else { + free(shared_shaper_id); + shared_shaper_id = NULL; + } np.leaf.cman = res->cman_mode; np.leaf.wred.wred_profile_id = res->wred_profile_id; diff --git a/dpdk/app/test-pmd/config.c b/dpdk/app/test-pmd/config.c index a0f3c246..61608d18 100644 --- a/dpdk/app/test-pmd/config.c +++ b/dpdk/app/test-pmd/config.c @@ -149,15 +149,11 @@ nic_stats_display(portid_t port_id) struct rte_eth_stats stats; struct rte_port *port = &ports[port_id]; uint8_t i; - portid_t pid; static const char *nic_stats_border = "########################"; if (port_id_is_invalid(port_id, ENABLED_WARN)) { - printf("Valid port range is [0"); - RTE_ETH_FOREACH_DEV(pid) - printf(", %d", pid); - printf("]\n"); + print_valid_ports(); return; } rte_eth_stats_get(port_id, &stats); @@ -231,13 +227,8 @@ nic_stats_display(portid_t port_id) void nic_stats_clear(portid_t port_id) { - portid_t pid; - if (port_id_is_invalid(port_id, ENABLED_WARN)) { - printf("Valid port range is [0"); - RTE_ETH_FOREACH_DEV(pid) - printf(", %d", pid); - printf("]\n"); + print_valid_ports(); return; } rte_eth_stats_reset(port_id); @@ -314,15 +305,11 @@ nic_stats_mapping_display(portid_t port_id) { struct rte_port *port = &ports[port_id]; uint16_t i; - portid_t pid; static const char *nic_stats_mapping_border = "########################"; if (port_id_is_invalid(port_id, ENABLED_WARN)) { - printf("Valid port range is [0"); - RTE_ETH_FOREACH_DEV(pid) - printf(", %d", pid); - printf("]\n"); + print_valid_ports(); return; } @@ -434,14 +421,10 @@ port_infos_display(portid_t port_id) int vlan_offload; struct rte_mempool * mp; static const char *info_border = "*********************"; - portid_t pid; uint16_t mtu; if (port_id_is_invalid(port_id, ENABLED_WARN)) { - printf("Valid port range is [0"); - RTE_ETH_FOREACH_DEV(pid) - printf(", %d", pid); - printf("]\n"); + print_valid_ports(); return; } port = &ports[port_id]; @@ -739,6 +722,17 @@ port_id_is_invalid(portid_t port_id, enum print_warning warning) return 1; } +void print_valid_ports(void) +{ + portid_t pid; + + printf("The valid ports array is ["); + RTE_ETH_FOREACH_DEV(pid) { + printf(" %d", pid); + } + printf(" ]\n"); +} + static int vlan_id_is_invalid(uint16_t vlan_id) { diff --git a/dpdk/app/test-pmd/parameters.c b/dpdk/app/test-pmd/parameters.c index 8fbb5150..5d51808f 100644 --- a/dpdk/app/test-pmd/parameters.c +++ b/dpdk/app/test-pmd/parameters.c @@ -403,7 +403,6 @@ parse_portnuma_config(const char *q_arg) }; unsigned long int_fld[_NUM_FLD]; char *str_fld[_NUM_FLD]; - portid_t pid; /* reset from value set at definition */ while ((p = strchr(p0,'(')) != NULL) { @@ -427,10 +426,7 @@ parse_portnuma_config(const char *q_arg) port_id = (portid_t)int_fld[FLD_PORT]; if (port_id_is_invalid(port_id, ENABLED_WARN) || port_id == (portid_t)RTE_PORT_ALL) { - printf("Valid port range is [0"); - RTE_ETH_FOREACH_DEV(pid) - printf(", %d", pid); - printf("]\n"); + print_valid_ports(); return -1; } socket_id = (uint8_t)int_fld[FLD_SOCKET]; @@ -461,7 +457,6 @@ parse_ringnuma_config(const char *q_arg) }; unsigned long int_fld[_NUM_FLD]; char *str_fld[_NUM_FLD]; - portid_t pid; #define RX_RING_ONLY 0x1 #define TX_RING_ONLY 0x2 #define RXTX_RING 0x3 @@ -488,10 +483,7 @@ parse_ringnuma_config(const char *q_arg) port_id = (portid_t)int_fld[FLD_PORT]; if (port_id_is_invalid(port_id, ENABLED_WARN) || port_id == (portid_t)RTE_PORT_ALL) { - printf("Valid port range is [0"); - RTE_ETH_FOREACH_DEV(pid) - printf(", %d", pid); - printf("]\n"); + print_valid_ports(); return -1; } socket_id = (uint8_t)int_fld[FLD_SOCKET]; diff --git a/dpdk/app/test-pmd/testpmd.c b/dpdk/app/test-pmd/testpmd.c index f66f4c64..32d68717 100644 --- a/dpdk/app/test-pmd/testpmd.c +++ b/dpdk/app/test-pmd/testpmd.c @@ -354,7 +354,7 @@ struct rte_fdir_conf fdir_conf = { .pballoc = RTE_FDIR_PBALLOC_64K, .status = RTE_FDIR_REPORT_STATUS, .mask = { - .vlan_tci_mask = 0x0, + .vlan_tci_mask = 0xFFEF, .ipv4_mask = { .src_ip = 0xFFFFFFFF, .dst_ip = 0xFFFFFFFF, @@ -880,18 +880,23 @@ init_fwd_streams(void) /* init new */ nb_fwd_streams = nb_fwd_streams_new; - fwd_streams = rte_zmalloc("testpmd: fwd_streams", - sizeof(struct fwd_stream *) * nb_fwd_streams, RTE_CACHE_LINE_SIZE); - if (fwd_streams == NULL) - rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_stream *)) " - "failed\n", nb_fwd_streams); + if (nb_fwd_streams) { + fwd_streams = rte_zmalloc("testpmd: fwd_streams", + sizeof(struct fwd_stream *) * nb_fwd_streams, + RTE_CACHE_LINE_SIZE); + if (fwd_streams == NULL) + rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" + " (struct fwd_stream *)) failed\n", + nb_fwd_streams); - for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { - fwd_streams[sm_id] = rte_zmalloc("testpmd: struct fwd_stream", - sizeof(struct fwd_stream), RTE_CACHE_LINE_SIZE); - if (fwd_streams[sm_id] == NULL) - rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_stream)" - " failed\n"); + for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { + fwd_streams[sm_id] = rte_zmalloc("testpmd:" + " struct fwd_stream", sizeof(struct fwd_stream), + RTE_CACHE_LINE_SIZE); + if (fwd_streams[sm_id] == NULL) + rte_exit(EXIT_FAILURE, "rte_zmalloc" + "(struct fwd_stream) failed\n"); + } } return 0; @@ -925,6 +930,9 @@ pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) pktnb_stats[1] = pktnb_stats[0]; burst_stats[0] = nb_burst; pktnb_stats[0] = nb_pkt; + } else if (nb_burst > burst_stats[1]) { + burst_stats[1] = nb_burst; + pktnb_stats[1] = nb_pkt; } } if (total_burst == 0) @@ -1210,6 +1218,31 @@ launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) } } +/* + * Update the forward ports list. + */ +void +update_fwd_ports(portid_t new_pid) +{ + unsigned int i; + unsigned int new_nb_fwd_ports = 0; + int move = 0; + + for (i = 0; i < nb_fwd_ports; ++i) { + if (port_id_is_invalid(fwd_ports_ids[i], DISABLED_WARN)) + move = 1; + else if (move) + fwd_ports_ids[new_nb_fwd_ports++] = fwd_ports_ids[i]; + else + new_nb_fwd_ports++; + } + if (new_pid < RTE_MAX_ETHPORTS) + fwd_ports_ids[new_nb_fwd_ports++] = new_pid; + + nb_fwd_ports = new_nb_fwd_ports; + nb_cfg_ports = new_nb_fwd_ports; +} + /* * Launch packet forwarding configuration. */ @@ -1245,10 +1278,6 @@ start_packet_forwarding(int with_tx_first) return; } - if (init_fwd_streams() < 0) { - printf("Fail from init_fwd_streams()\n"); - return; - } if(dcb_test) { for (i = 0; i < nb_fwd_ports; i++) { @@ -1268,10 +1297,11 @@ start_packet_forwarding(int with_tx_first) } test_done = 0; + fwd_config_setup(); + if(!no_flush_rx) flush_fwd_rx_queues(); - fwd_config_setup(); pkt_fwd_config_display(&cur_fwd_config); rxtx_config_display(); @@ -1876,6 +1906,8 @@ attach_port(char *identifier) ports[pi].port_status = RTE_PORT_STOPPED; + update_fwd_ports(pi); + printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); printf("Done\n"); } @@ -1902,6 +1934,8 @@ detach_port(portid_t port_id) nb_ports = rte_eth_dev_count(); + update_fwd_ports(RTE_MAX_ETHPORTS); + printf("Port '%s' is detached. Now total ports is %d\n", name, nb_ports); printf("Done\n"); @@ -1995,13 +2029,16 @@ check_all_ports_link_status(uint32_t port_mask) static void rmv_event_callback(void *arg) { + int org_no_link_check = no_link_check; struct rte_eth_dev *dev; portid_t port_id = (intptr_t)arg; RTE_ETH_VALID_PORTID_OR_RET(port_id); dev = &rte_eth_devices[port_id]; + no_link_check = 1; stop_port(port_id); + no_link_check = org_no_link_check; close_port(port_id); printf("removing device %s\n", dev->device->name); if (rte_eal_dev_detach(dev->device)) @@ -2246,7 +2283,10 @@ uint8_t port_is_bonding_slave(portid_t slave_pid) struct rte_port *port; port = &ports[slave_pid]; - return port->slave_flag; + if ((rte_eth_devices[slave_pid].data->dev_flags & + RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) + return 1; + return 0; } const uint16_t vlan_tags[] = { @@ -2257,12 +2297,14 @@ const uint16_t vlan_tags[] = { }; static int -get_eth_dcb_conf(struct rte_eth_conf *eth_conf, +get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, enum dcb_mode_enable dcb_mode, enum rte_eth_nb_tcs num_tcs, uint8_t pfc_en) { uint8_t i; + int32_t rc; + struct rte_eth_rss_conf rss_conf; /* * Builds up the correct configuration for dcb+vt based on the vlan tags array @@ -2302,6 +2344,10 @@ get_eth_dcb_conf(struct rte_eth_conf *eth_conf, struct rte_eth_dcb_tx_conf *tx_conf = ð_conf->tx_adv_conf.dcb_tx_conf; + rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); + if (rc != 0) + return rc; + rx_conf->nb_tcs = num_tcs; tx_conf->nb_tcs = num_tcs; @@ -2309,8 +2355,9 @@ get_eth_dcb_conf(struct rte_eth_conf *eth_conf, rx_conf->dcb_tc[i] = i % num_tcs; tx_conf->dcb_tc[i] = i % num_tcs; } + eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; - eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_hf; + eth_conf->rx_adv_conf.rss_conf = rss_conf; eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; } @@ -2341,7 +2388,7 @@ init_port_dcb_config(portid_t pid, dcb_config = 1; /*set configuration of DCB in vt mode and DCB in non-vt mode*/ - retval = get_eth_dcb_conf(&port_conf, dcb_mode, num_tcs, pfc_en); + retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); if (retval < 0) return retval; port_conf.rxmode.hw_vlan_filter = 1; diff --git a/dpdk/app/test-pmd/testpmd.h b/dpdk/app/test-pmd/testpmd.h index 92e16073..b3b26d2c 100644 --- a/dpdk/app/test-pmd/testpmd.h +++ b/dpdk/app/test-pmd/testpmd.h @@ -599,6 +599,7 @@ void fwd_config_setup(void); void set_def_fwd_config(void); void reconfig(portid_t new_port_id, unsigned socket_id); int init_fwd_streams(void); +void update_fwd_ports(portid_t new_pid); void port_mtu_set(portid_t port_id, uint16_t mtu); void port_reg_bit_display(portid_t port_id, uint32_t reg_off, uint8_t bit_pos); @@ -726,6 +727,7 @@ enum print_warning { DISABLED_WARN }; int port_id_is_invalid(portid_t port_id, enum print_warning warning); +void print_valid_ports(void); int new_socket_id(unsigned int socket_id); queueid_t get_allowed_max_nb_rxq(portid_t *pid); diff --git a/dpdk/buildtools/pmdinfogen/Makefile b/dpdk/buildtools/pmdinfogen/Makefile index bf07b6f2..ff7a5fae 100644 --- a/dpdk/buildtools/pmdinfogen/Makefile +++ b/dpdk/buildtools/pmdinfogen/Makefile @@ -41,7 +41,7 @@ HOSTAPP = dpdk-pmdinfogen # SRCS-y += pmdinfogen.c -HOST_CFLAGS += $(WERROR_FLAGS) -g +HOST_CFLAGS += $(HOST_WERROR_FLAGS) -g HOST_CFLAGS += -I$(RTE_OUTPUT)/include include $(RTE_SDK)/mk/rte.hostapp.mk diff --git a/dpdk/config/defconfig_i686-native-linuxapp-icc b/dpdk/config/defconfig_i686-native-linuxapp-icc index 269e88e9..46ffb11e 100644 --- a/dpdk/config/defconfig_i686-native-linuxapp-icc +++ b/dpdk/config/defconfig_i686-native-linuxapp-icc @@ -46,11 +46,6 @@ CONFIG_RTE_TOOLCHAIN_ICC=y # CONFIG_RTE_LIBRTE_KNI=n -# -# Vectorized PMD is not supported on 32-bit -# -CONFIG_RTE_IXGBE_INC_VECTOR=n - # # Solarflare PMD is not supported on 32-bit # diff --git a/dpdk/doc/api/doxy-html-custom.sh b/dpdk/doc/api/doxy-html-custom.sh old mode 100644 new mode 100755 diff --git a/dpdk/doc/guides/nics/enic.rst b/dpdk/doc/guides/nics/enic.rst index cb5ae125..a11627a0 100644 --- a/dpdk/doc/guides/nics/enic.rst +++ b/dpdk/doc/guides/nics/enic.rst @@ -252,6 +252,35 @@ Generic Flow API is supported. The baseline support is: More features may be added in future firmware and new versions of the VIC. Please refer to the release notes. +Ingress VLAN Rewrite +-------------------- + +VIC adapters can tag, untag, or modify the VLAN headers of ingress +packets. The ingress VLAN rewrite mode controls this behavior. By +default, it is set to pass-through, where the NIC does not modify the +VLAN header in any way so that the application can see the original +header. This mode is sufficient for many applications, but may not be +suitable for others. Such applications may change the mode by setting +``devargs`` parameter ``ig-vlan-rewrite`` to one of the following. + +- ``pass``: Pass-through mode. The NIC does not modify the VLAN + header. This is the default mode. + +- ``priority``: Priority-tag default VLAN mode. If the ingress packet + is tagged with the default VLAN, the NIC replaces its VLAN header + with the priority tag (VLAN ID 0). + +- ``trunk``: Default trunk mode. The NIC tags untagged ingress packets + with the default VLAN. Tagged ingress packets are not modified. To + the application, every packet appears as tagged. + +- ``untag``: Untag default VLAN mode. If the ingress packet is tagged + with the default VLAN, the NIC removes or untags its VLAN header so + that the application sees an untagged packet. As a result, the + default VLAN becomes `untagged`. This mode can be useful for + applications such as OVS-DPDK performance benchmarks that utilize + only the default VLAN and want to see only untagged packets. + .. _enic_limitations: Limitations @@ -267,9 +296,10 @@ Limitations In test setups where an Ethernet port of a Cisco adapter in TRUNK mode is connected point-to-point to another adapter port or connected though a router instead of a switch, all ingress packets will be VLAN tagged. Programs such - as l3fwd which do not account for VLAN tags in packets will misbehave. The - solution is to enable VLAN stripping on ingress. The follow code fragment is - example of how to accomplish this: + as l3fwd may not account for VLAN tags in packets and may misbehave. One + solution is to enable VLAN stripping on ingress so the VLAN tag is removed + from the packet and put into the mbuf->vlan_tci field. Here is an example + of how to accomplish this: .. code-block:: console @@ -277,6 +307,14 @@ Limitations vlan_offload |= ETH_VLAN_STRIP_OFFLOAD; rte_eth_dev_set_vlan_offload(port, vlan_offload); +Another alternative is modify the adapter's ingress VLAN rewrite mode so that +packets with the default VLAN tag are stripped by the adapter and presented to +DPDK as untagged packets. In this case mbuf->vlan_tci and the PKT_RX_VLAN and +PKT_RX_VLAN_STRIPPED mbuf flags would not be set. This mode is enabled with the +``devargs`` parameter ``ig-vlan-rewrite=untag``. For example:: + + -w 12:00.0,ig-vlan-rewrite=untag + - Limited flow director support on 1200 series and 1300 series Cisco VIC adapters with old firmware. Please see :ref:`enic-flow-director`. diff --git a/dpdk/doc/guides/nics/features/mlx5.ini b/dpdk/doc/guides/nics/features/mlx5.ini index c3636391..f8ce0877 100644 --- a/dpdk/doc/guides/nics/features/mlx5.ini +++ b/dpdk/doc/guides/nics/features/mlx5.ini @@ -29,6 +29,7 @@ CRC offload = Y VLAN offload = Y L3 checksum offload = Y L4 checksum offload = Y +Timestamp offload = Y Packet type parsing = Y Rx descriptor status = Y Tx descriptor status = Y diff --git a/dpdk/doc/guides/nics/mlx5.rst b/dpdk/doc/guides/nics/mlx5.rst index f9558da8..50fced3f 100644 --- a/dpdk/doc/guides/nics/mlx5.rst +++ b/dpdk/doc/guides/nics/mlx5.rst @@ -108,7 +108,11 @@ Limitations - Port statistics through software counters only. Flow statistics are supported by hardware counters. - Hardware checksum RX offloads for VXLAN inner header are not supported yet. -- Forked secondary process not supported. +- For secondary process: + + - Forked secondary process not supported. + - All mempools must be initialized before rte_eth_dev_start(). + - Flow pattern without any specific vlan will match for vlan packets as well: When VLAN spec is not specified in the pattern, the matching rule will be created with VLAN as a wild card. diff --git a/dpdk/doc/guides/nics/nfp.rst b/dpdk/doc/guides/nics/nfp.rst index 99a3b76e..67e574e0 100644 --- a/dpdk/doc/guides/nics/nfp.rst +++ b/dpdk/doc/guides/nics/nfp.rst @@ -34,14 +34,14 @@ NFP poll mode driver library Netronome's sixth generation of flow processors pack 216 programmable cores and over 100 hardware accelerators that uniquely combine packet, flow, security and content processing in a single device that scales -up to 400 Gbps. +up to 400-Gb/s. This document explains how to use DPDK with the Netronome Poll Mode Driver (PMD) supporting Netronome's Network Flow Processor 6xxx (NFP-6xxx) and Netronome's Flow Processor 4xxx (NFP-4xxx). NFP is a SRIOV capable device and the PMD driver supports the physical -function (PF) and virtual functions (VFs). +function (PF) and the virtual functions (VFs). Dependencies ------------ @@ -49,17 +49,18 @@ Dependencies Before using the Netronome's DPDK PMD some NFP configuration, which is not related to DPDK, is required. The system requires installation of **Netronome's BSP (Board Support Package)** along -with some specific NFP firmware application. Netronome's NSP ABI +with a specific NFP firmware application. Netronome's NSP ABI version should be 0.20 or higher. If you have a NFP device you should already have the code and -documentation for doing all this configuration. Contact +documentation for this configuration. Contact **support@netronome.com** to obtain the latest available firmware. -The NFP Linux netdev kernel driver for VFs is part of vanilla kernel -since kernel version 4.5, and support for the PF since kernel version -4.11. Support for older kernels can be obtained on Github at -**https://github.com/Netronome/nfp-drv-kmods** along with build +The NFP Linux netdev kernel driver for VFs has been a part of the +vanilla kernel since kernel version 4.5, and support for the PF +since kernel version 4.11. Support for older kernels can be obtained +on Github at +**https://github.com/Netronome/nfp-drv-kmods** along with the build instructions. NFP PMD needs to be used along with UIO ``igb_uio`` or VFIO (``vfio-pci``) @@ -70,15 +71,15 @@ Building the software Netronome's PMD code is provided in the **drivers/net/nfp** directory. Although NFP PMD has NetronomeĀ“s BSP dependencies, it is possible to -compile it along with other DPDK PMDs even if no BSP was installed before. +compile it along with other DPDK PMDs even if no BSP was installed previously. Of course, a DPDK app will require such a BSP installed for using the NFP PMD, along with a specific NFP firmware application. -Default PMD configuration is at **common_linuxapp configuration** file: +Default PMD configuration is at the **common_linuxapp configuration** file: - **CONFIG_RTE_LIBRTE_NFP_PMD=y** -Once DPDK is built all the DPDK apps and examples include support for +Once the DPDK is built all the DPDK apps and examples include support for the NFP PMD. @@ -91,18 +92,18 @@ for details. Using the PF ------------ -NFP PMD has support for using the NFP PF as another DPDK port, but it does not +NFP PMD supports using the NFP PF as another DPDK port, but it does not have any functionality for controlling VFs. In fact, it is not possible to use the PMD with the VFs if the PF is being used by DPDK, that is, with the NFP PF -bound to ``igb_uio`` or ``vfio-pci`` kernel drivers. Future DPDK version will +bound to ``igb_uio`` or ``vfio-pci`` kernel drivers. Future DPDK versions will have a PMD able to work with the PF and VFs at the same time and with the PF implementing VF management along with other PF-only functionalities/offloads. -The PMD PF has extra work to do which will delay the DPDK app initialization -like checking if a firmware is already available in the device, uploading the -firmware if necessary, and configure the Link state properly when starting or -stopping a PF port. Note that firmware upload is not always necessary which is -the main delay for NFP PF PMD initialization. +The PMD PF has extra work to do which will delay the DPDK app initialization. +This additional effort could be checking if a firmware is already available in +the device, uploading the firmware if necessary or configuring the Link state +properly when starting or stopping a PF port. Note that firmware upload is not +always necessary which is the main delay for NFP PF PMD initialization. Depending on the Netronome product installed in the system, firmware files should be available under ``/lib/firmware/netronome``. DPDK PMD supporting the @@ -114,14 +115,14 @@ PF multiport support -------------------- Some NFP cards support several physical ports with just one single PCI device. -DPDK core is designed with the 1:1 relationship between PCI devices and DPDK +The DPDK core is designed with a 1:1 relationship between PCI devices and DPDK ports, so NFP PMD PF support requires handling the multiport case specifically. During NFP PF initialization, the PMD will extract the information about the number of PF ports from the firmware and will create as many DPDK ports as needed. Because the unusual relationship between a single PCI device and several DPDK -ports, there are some limitations when using more than one PF DPDK ports: there +ports, there are some limitations when using more than one PF DPDK port: there is no support for RX interrupts and it is not possible either to use those PF ports with the device hotplug functionality. @@ -136,7 +137,7 @@ System configuration get the drivers from the above Github repository and follow the instructions for building and installing it. - Virtual Functions need to be enabled before they can be used with the PMD. + VFs need to be enabled before they can be used with the PMD. Before enabling the VFs it is useful to obtain information about the current NFP PCI device detected by the system: diff --git a/dpdk/doc/guides/nics/qede.rst b/dpdk/doc/guides/nics/qede.rst index 84becc98..19802f43 100644 --- a/dpdk/doc/guides/nics/qede.rst +++ b/dpdk/doc/guides/nics/qede.rst @@ -86,12 +86,13 @@ Prerequisites `QLogic Driver Download Center `_. For downloading firmware file, select adapter category, model and DPDK Poll Mode Driver. -- Requires management firmware (MFW) version **8.30.x.x** or higher to be - flashed on to the adapter. If the required management firmware is not - available then download from - `QLogic Driver Download Center `_. - For downloading firmware upgrade utility, select adapter category, model and Linux distro. - To flash the management firmware refer to the instructions in the QLogic Firmware Upgrade Utility Readme document. +- Requires the NIC be updated minimally with **8.30.x.x** Management firmware(MFW) version supported for that NIC. + It is highly recommended that the NIC be updated with the latest available management firmware version to get latest feature set. + Management Firmware and Firmware Upgrade Utility for Cavium FastLinQ(r) branded adapters can be downloaded from + `Driver Download Center `_. + For downloading Firmware Upgrade Utility, select NIC category, model and Linux distro. + To update the management firmware, refer to the instructions in the Firmware Upgrade Utility Readme document. + For OEM branded adapters please follow the instruction provided by the OEM to update the Management Firmware on the NIC. - SR-IOV requires Linux PF driver version **8.20.x.x** or higher. If the required PF driver is not available then download it from diff --git a/dpdk/doc/guides/rel_notes/release_17_11.rst b/dpdk/doc/guides/rel_notes/release_17_11.rst index fad7a7eb..e17fd2ca 100644 --- a/dpdk/doc/guides/rel_notes/release_17_11.rst +++ b/dpdk/doc/guides/rel_notes/release_17_11.rst @@ -1090,3 +1090,375 @@ Fixes in 17.11 LTS Release * vhost: handle virtually non-contiguous buffers in Rx-mrg (fixes CVE-2018-1059) * vhost: handle virtually non-contiguous buffers in Tx (fixes CVE-2018-1059) * vhost: introduce safe API for GPA translation (fixes CVE-2018-1059) + +17.11.3 +~~~~~~~ + +* app/crypto-perf: check minimum lcore number +* app/crypto-perf: fix excess crypto device error +* app/crypto-perf: fix IOVA translation +* app/crypto-perf: fix parameters copy +* app/crypto-perf: use strcpy for allocated string +* app/procinfo: fix strncpy usage in args parsing +* app/testpmd: fix burst stats reporting +* app/testpmd: fix command token +* app/testpmd: fix empty list of RSS queues for flow +* app/testpmd: fix forward ports Rx flush +* app/testpmd: fix forward ports update +* app/testpmd: fix removed device link status asking +* app/testpmd: fix slave port detection +* app/testpmd: fix synchronic port hotplug +* app/testpmd: fix valid ports prints +* bus/dpaa: fix resource leak +* bus/fslmc: fix find device start condition +* bus/pci: fix find device implementation +* bus/vdev: fix finding device by name +* cryptodev: fix supported size check +* crypto/dpaa2_sec: fix HMAC supported digest sizes +* crypto/scheduler: fix 64-bit mask of workers cores +* crypto/scheduler: fix memory leak +* crypto/scheduler: fix multicore rings re-use +* crypto/scheduler: fix possible duplicated ring names +* crypto/scheduler: set null pointer after freeing +* crypto/zuc: batch ops with same transform +* crypto/zuc: do not set default op status +* doc: add timestamp offload to mlx5 features +* doc: fix NFP NIC guide grammar +* drivers/net: fix link autoneg value for virtual PMDs +* eal/ppc: remove braces in SMP memory barrier macro +* ethdev: fix port accessing after release +* ethdev: fix queue start +* event/dpaa2: remove link from info structure +* examples/exception_path: limit core count to 64 +* examples/l2fwd-crypto: fix the default aead assignments +* examples/performance-thread: fix return type of threads +* examples/quota_watermark: fix return type of threads +* hash: fix missing spinlock unlock in add key +* ip_frag: fix double free of chained mbufs +* kni: fix build on CentOS 7.4 +* kni: fix build on RHEL 7.5 +* mbuf: fix Tx checksum offload API doc +* mbuf: improve tunnel Tx offloads API doc +* mem: do not use physical addresses in IOVA as VA mode +* mempool: fix leak when no objects are populated +* mempool: fix virtual address population +* mk: fix make defconfig on FreeBSD +* net: add IPv6 header fields macros +* net/bnx2x: do not cast function pointers as a policy +* net/bnx2x: fix for PCI FLR after ungraceful exit +* net/bnx2x: fix KR2 device check +* net/bnx2x: fix memzone name overrun +* net/bnxt: avoid invalid vnic id in set L2 Rx mask +* net/bnxt: fix endianness of flag +* net/bnxt: fix license header +* net/bnxt: fix LRO disable +* net/bnxt: fix Rx checksum flags +* net/bnxt: fix Rx checksum flags for tunnel frames +* net/bnxt: fix Rx drop setting +* net/bnxt: fix Rx mbuf and agg ring leak in dev stop +* net/bnxt: fix usage of vnic id +* net/bnxt: free memory allocated for VF filters +* net/bnxt: set padding flags in Rx descriptor +* net/bonding: clear started state if start fails +* net/bonding: export mode 4 slave info routine +* net/bonding: fix primary slave port id storage type +* net/bonding: fix setting VLAN ID on slave ports +* net/bonding: fix slave activation simultaneously +* net/bonding: free mempool used in mode 6 +* net/dpaa2: fix xstats +* net/dpaa: fix oob access +* net/enic: allocate stats DMA buffer upfront during probe +* net/enic: fix crash on MTU update with non-setup queues +* net/failsafe: fix duplicate event registration +* net/failsafe: fix probe cleanup +* net/failsafe: fix removed sub-device cleanup +* net/i40e: fix DDP profile DEL operation +* net/i40e: fix failing to disable FDIR Tx queue +* net/i40e: fix intr callback unregister by adding retry +* net/i40e: fix link status update +* net/i40e: fix link update no wait +* net/i40e: fix shifts of signed values +* net/ixgbe: enable vector PMD for icc 32 bits +* net/ixgbe: fix busy wait during checking link status +* net/ixgbe: fix DCB configuration +* net/ixgbe: fix intr callback unregister by adding retry +* net/ixgbe: fix too many interrupts +* net/liquidio: fix link state fetching during start +* net/mlx4: avoid constant recreations in function +* net/mlx4: fix a typo in header file +* net/mlx4: fix broadcast Rx +* net/mlx4: fix removal detection of stopped port +* net/mlx4: fix RSS resource leak in case of error +* net/mlx4: fix Rx resource leak in case of error +* net/mlx4: fix single port configuration +* net/mlx4: fix UDP flow rule limitation enforcement +* net/mlx4: store RSS hash result in mbufs +* net/mlx5: add data-plane debug message macro +* net/mlx5: add missing function documentation +* net/mlx5: add packet type index for TCP ack +* net/mlx5: change device reference for secondary process +* net/mlx5: change non failing function return values +* net/mlx5: change pkt burst select function prototype +* net/mlx5: change tunnel flow priority +* net/mlx5: enforce RSS key length limitation +* net/mlx5: fix allocation when no memory on device NUMA node +* net/mlx5: fix build with clang on ARM +* net/mlx5: fix calculation of Tx TSO inline room size +* net/mlx5: fix close after start failure +* net/mlx5: fix count in xstats +* net/mlx5: fix CRC strip capability query +* net/mlx5: fix disabling Tx packet inlining +* net/mlx5: fix double free on error handling +* net/mlx5: fix ethtool link setting call order +* net/mlx5: fix existing file removal +* net/mlx5: fix flow creation with a single target queue +* net/mlx5: fix flow director conversion +* net/mlx5: fix flow director drop rule deletion crash +* net/mlx5: fix flow director mask +* net/mlx5: fix flow director rule deletion crash +* net/mlx5: fix flow validation +* net/mlx5: fix icc build +* net/mlx5: fix invalid flow item check +* net/mlx5: fix IPv6 header fields +* net/mlx5: fix link status behavior +* net/mlx5: fix link status initialization +* net/mlx5: fix link status to use wait to complete +* net/mlx5: fix probe return value polarity +* net/mlx5: fix reception of multiple MAC addresses +* net/mlx5: fix resource leak in case of error +* net/mlx5: fix RSS flow action bounds check +* net/mlx5: fix RSS key length query +* net/mlx5: fix secondary process mempool registration +* net/mlx5: fix socket connection return value +* net/mlx5: fix sriov flag +* net/mlx5: fix synchronization on polling Rx completions +* net/mlx5: improve flow error explanation +* net/mlx5: map UAR address around huge pages +* net/mlx5: mark parameters with unused attribute +* net/mlx5: name parameters in function prototypes +* net/mlx5: normalize function prototypes +* net/mlx5: prefix all functions with mlx5 +* net/mlx5: refuse empty VLAN flow specification +* net/mlx5: remove 32-bit support +* net/mlx5: remove assert un-accessible from secondary process +* net/mlx5: remove control path locks +* net/mlx5: remove excessive data prefetch +* net/mlx5: remove get priv internal function +* net/mlx5: remove kernel version check +* net/mlx5: remove useless empty lines +* net/mlx5: setup RSS regardless of queue count +* net/mlx5: split L3/L4 in flow director +* net/mlx5: standardize on negative errno values +* net/mlx5: use dynamic logging +* net/mlx5: use port id in PMD log +* net/mlx5: warn for unsuccessful memory registration +* net/mlx: control netdevices through ioctl only +* net/mrvl: fix crash when port is closed without starting +* net/mrvl: fix Rx descriptors number +* net/nfp: fix assigning port id in mbuf +* net/nfp: fix barrier location +* net/nfp: fix link speed capabilities +* net/nfp: fix mbufs releasing when stop or close +* net/octeontx: fix null pointer dereference +* net/octeontx: fix uninitialized speed variable +* net/octeontx: fix uninitialized variable in port open +* net/qede/base: fix to support OVLAN mode +* net/qede: fix alloc from socket 0 +* net/qede: fix device stop to remove primary MAC +* net/qede: fix L2-handles used for RSS hash update +* net/qede: fix memory alloc for multiple port reconfig +* net/qede: fix missing loop index in Tx SG mode +* net/qede: fix multicast filtering +* net/qede: fix to prevent overwriting packet type +* net/qede: fix unicast filter routine return code +* net/qede: fix VF port creation sequence +* net/sfc: add missing defines for SAL annotation +* net/sfc: add missing Rx fini on RSS setup fail path +* net/sfc/base: fix comparison always true warning +* net/sfc: fix mbuf data alignment calculation +* net/sfc: fix type of opaque pointer in perf profile handler +* net/sfc: ignore spec bits not covered by mask +* net/sfc: process RSS settings on Rx configure step +* net/szedata2: fix format string for PCI address +* net/szedata2: fix total stats +* net/tap: fix icc build +* net/vhost: fix crash when creating vdev dynamically +* net/vhost: fix invalid state +* net/vhost: initialise device as inactive +* net/vmxnet3: set the queue shared buffer at start +* nfp: allow for non-root user +* nfp: restore the unlink operation +* nfp: unlink the appropriate lock file +* pci: remove duplicated symbol from map file +* test/distributor: fix return type of thread function +* test: fix memory flags test for low NUMA nodes number +* test/mempool: fix autotest retry +* test/pipeline: fix return type of stub miss +* test/pipeline: fix type of table entry parameter +* test/reorder: fix freeing mbuf twice +* vfio: do not needlessly check for IOVA mode +* vhost: check cmsg not null +* vhost: fix compilation issue when vhost debug enabled +* vhost: fix dead lock on closing in server mode +* vhost: fix device cleanup at stop +* vhost: fix message payload union in setting ring address +* vhost: fix offset while mmaping log base address +* vhost: fix realloc failure +* vhost: fix ring index returned to master on stop + +17.11.4 +~~~~~~~ + +* app/crypto-perf: fix auth IV offset +* app/testpmd: fix buffer leak in TM command +* app/testpmd: fix DCB config +* app/testpmd: fix VLAN TCI mask set error for FDIR +* bitrate: add sanity check on parameters +* bus/dpaa: fix buffer offset setting in FMAN +* bus/dpaa: fix build +* bus/dpaa: fix phandle support for Linux 4.16 +* bus/pci: use IOVAs check when setting IOVA mode +* crypto/qat: fix checks for 3GPP algo bit params +* doc: fix bonding command in testpmd +* doc: update qede management firmware guide +* eal: fix bitmap documentation +* eal: fix return codes on thread naming failure +* eal/linux: fix invalid syntax in interrupts +* eal/linux: fix uninitialized value +* ethdev: fix a doxygen comment for port allocation +* ethdev: fix queue statistics mapping documentation +* eventdev: add event buffer flush in Rx adapter +* eventdev: fix internal port logic in Rx adapter +* eventdev: fix missing update to Rx adaper WRR position +* eventdev: fix port in Rx adapter internal function +* eventdev: fix Rx SW adapter stop +* event: fix ring init failure handling +* event/octeontx: remove unnecessary port start and stop +* examples/exception_path: fix out-of-bounds read +* examples: fix strncpy error for GCC8 +* examples/flow_filtering: add flow director config for i40e +* examples/ipsec-secgw: fix bypass rule processing +* examples/ipsec-secgw: fix IPv4 checksum at Tx +* examples/l2fwd-crypto: check return value on IV size check +* examples/l2fwd-crypto: fix digest with AEAD algo +* examples/l2fwd-crypto: skip device not supporting operation +* examples/l3fwd: remove useless include +* hash: fix a multi-writer race condition +* hash: fix doxygen of return values +* hash: fix key slot size accuracy +* hash: fix multiwriter lock memory allocation +* kni: fix build on RHEL 7.5 +* kni: fix build with gcc 8.1 +* kni: fix crash with null name +* maintainers: claim maintainership for ARM v7 and v8 +* maintainers: update for Mellanox PMDs +* mem: add function for checking memsegs IOVAs addresses +* mem: fix max DMA maskbit size +* mem: use address hint for mapping hugepages +* metrics: add check for invalid key +* metrics: disallow null as metric name +* metrics: do not fail silently when uninitialised +* mk: fix cross build +* mk: fix permissions when using make install +* mk: remove unnecessary test rules +* mk: update targets for classified tests +* net/bnx2x: fix FW command timeout during stop +* net/bnx2x: fix poll link status +* net/bnx2x: fix to set device link status +* net/bnxt: add missing ids in xstats +* net/bnxt: check access denied for HWRM commands +* net/bnxt: check for invalid vNIC id +* net/bnxt: fix filter freeing +* net/bnxt: fix HW Tx checksum offload check +* net/bnxt: fix lock release on NVM write failure +* net/bnxt: fix memory leaks in NVM commands +* net/bnxt: fix RETA size +* net/bnxt: fix Rx ring count limitation +* net/bnxt: fix set MTU +* net/bnxt: fix to move a flow to a different queue +* net/bnxt: use correct flags during VLAN configuration +* net/bonding: always update bonding link status +* net/bonding: do not clear active slave count +* net/bonding: fix MAC address reset +* net/bonding: fix race condition +* net/cxgbe: fix init failure due to new flash parts +* net/cxgbe: fix Rx channel map and queue type +* net/dpaa2: remove loop for unused pool entries +* net/ena: change memory type +* net/ena: check pointer before memset +* net/ena: fix GENMASK_ULL macro +* net/ena: fix SIGFPE with 0 Rx queue +* net/ena: set link speed as none +* net/enic: add devarg to specify ingress VLAN rewrite mode +* net/enic: do not overwrite admin Tx queue limit +* net/i40e: fix check of flow director programming status +* net/i40e: fix link speed +* net/i40e: fix packet type parsing with DDP +* net/i40e: fix setting TPID with AQ command +* net/i40e: fix shifts of 32-bit value +* net/i40e: revert fix of flow director check +* net/i40e: workaround performance degradation +* net/ixgbe: add support for VLAN in IP mode FDIR +* net/ixgbe: fix mask bits register set error for FDIR +* net/ixgbe: fix tunnel id format error for FDIR +* net/ixgbe: fix tunnel type set error for FDIR +* net/mlx4: check RSS queues number limitation +* net/mlx4: fix minor resource leak during init +* net/mlx5: add missing sanity checks for Tx completion queue +* net/mlx5: fix assert for Tx completion queue count +* net/mlx5: fix build with old kernels +* net/mlx5: fix compilation for rdma-core v19 +* net/mlx5: fix crash in device probe +* net/mlx5: fix error number handling +* net/mlx5: fix flow search on FDIR deletion +* net/mlx5: fix queue rollback when starting device +* net/mlx5: fix return value when deleting fdir filter +* net/mlx5: fix Rx buffer replenishment threshold +* net/mlx5: fix secondary process resource leakage +* net/mlx5: fix TCI mask filter +* net/mlx5: preserve allmulticast flag for flow isolation mode +* net/mlx5: preserve promiscuous flag for flow isolation mode +* net/mvpp2: check pointer before using it +* net/nfp: check hugepages IOVAs based on DMA mask +* net/nfp: fix field initialization in Tx descriptor +* net/nfp: support IOVA VA mode +* net/octeontx: fix stop clearing Rx/Tx functions +* net/pcap: fix multiple queues +* net/qede/base: fix GRC attention callback +* net/qede/base: fix to clear HW indication +* net/qede: fix default extended VLAN offload config +* net/qede: fix for devargs +* net/qede: fix incorrect link status update +* net/qede: fix interrupt handler unregister +* net/qede: fix legacy interrupt mode +* net/qede: fix link change event notification +* net/qede: fix MAC address removal failure message +* net/qede: fix ntuple filter configuration +* net/qede: fix unicast MAC address handling in VF +* net/qede: fix VF MTU update +* net/qede: remove primary MAC removal +* net/sfc: cut non VLAN ID bits from TCI +* net/sfc: fix assert in set multicast address list +* net/sfc: handle unknown L3 packet class in EF10 event parser +* net/tap: fix zeroed flow mask configurations +* net/thunderx: avoid sq door bell write on zero packet +* net/thunderx: fix build with gcc optimization on +* ring: fix sign conversion warning +* security: fix crash on destroy null session +* test/crypto: fix device id when stopping port +* test: fix code on report +* test: fix EAL flags autotest on FreeBSD +* test: fix result printing +* test: fix uninitialized port configuration +* test/flow_classify: fix return types +* test/hash: fix multiwriter with non consecutive cores +* test/hash: fix potential memory leak +* test: improve filtering +* test: make autotest runner python 2/3 compliant +* test: print autotest categories +* vfio: fix PCI address comparison +* vhost: fix missing increment of log cache count +* vhost: flush IOTLB cache on new mem table handling +* vhost: improve dirty pages logging performance +* vhost: release locks on RARP packet failure +* vhost: retranslate vring addr when memory table changes diff --git a/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst b/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst index 9789139a..c9ce85c9 100644 --- a/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst +++ b/dpdk/doc/guides/testpmd_app_ug/testpmd_funcs.rst @@ -1865,7 +1865,7 @@ Create a new bonding device:: For example, to create a bonded device in mode 1 on socket 0:: - testpmd> create bonded 1 0 + testpmd> create bonded device 1 0 created new bonded device (port X) add bonding slave diff --git a/dpdk/drivers/bus/dpaa/base/fman/fman.c b/dpdk/drivers/bus/dpaa/base/fman/fman.c index 3816dba5..a9c88ddc 100644 --- a/dpdk/drivers/bus/dpaa/base/fman/fman.c +++ b/dpdk/drivers/bus/dpaa/base/fman/fman.c @@ -475,6 +475,7 @@ fman_if_init(const struct device_node *dpa_node) if (!pool_node) { FMAN_ERR(-ENXIO, "%s: bad fsl,bman-buffer-pools\n", dname); + free(bpool); goto err; } pname = pool_node->full_name; @@ -482,6 +483,7 @@ fman_if_init(const struct device_node *dpa_node) prop = of_get_property(pool_node, "fsl,bpid", &proplen); if (!prop) { FMAN_ERR(-EINVAL, "%s: no fsl,bpid\n", pname); + free(bpool); goto err; } assert(proplen == sizeof(*prop)); diff --git a/dpdk/drivers/bus/dpaa/base/fman/fman_hw.c b/dpdk/drivers/bus/dpaa/base/fman/fman_hw.c index 077c17c0..d66efa12 100644 --- a/dpdk/drivers/bus/dpaa/base/fman/fman_hw.c +++ b/dpdk/drivers/bus/dpaa/base/fman/fman_hw.c @@ -39,6 +39,8 @@ #include #include +#define FMAN_SP_EXT_BUF_MARG_START_SHIFT 16 + /* Instantiate the global variable that the inline CRC64 implementation (in * ) depends on. */ @@ -445,20 +447,16 @@ fman_if_set_fc_quanta(struct fman_if *fm_if, u16 pause_quanta) int fman_if_get_fdoff(struct fman_if *fm_if) { - u32 fmbm_ricp; + u32 fmbm_rebm; int fdoff; - int iceof_mask = 0x001f0000; - int icsz_mask = 0x0000001f; struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); assert(fman_ccsr_map_fd != -1); - fmbm_ricp = - in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_ricp); - /*iceof + icsz*/ - fdoff = ((fmbm_ricp & iceof_mask) >> 16) * 16 + - (fmbm_ricp & icsz_mask) * 16; + fmbm_rebm = in_be32(&((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm); + + fdoff = (fmbm_rebm >> FMAN_SP_EXT_BUF_MARG_START_SHIFT) & 0x1ff; return fdoff; } @@ -525,12 +523,16 @@ fman_if_set_fdoff(struct fman_if *fm_if, uint32_t fd_offset) { struct __fman_if *__if = container_of(fm_if, struct __fman_if, __if); unsigned int *fmbm_rebm; + int val = 0; + int fmbm_mask = 0x01ff0000; + + val = fd_offset << FMAN_SP_EXT_BUF_MARG_START_SHIFT; assert(fman_ccsr_map_fd != -1); fmbm_rebm = &((struct rx_bmi_regs *)__if->bmi_map)->fmbm_rebm; - out_be32(fmbm_rebm, in_be32(fmbm_rebm) | (fd_offset << 16)); + out_be32(fmbm_rebm, (in_be32(fmbm_rebm) & ~fmbm_mask) | val); } void diff --git a/dpdk/drivers/bus/dpaa/base/fman/of.c b/dpdk/drivers/bus/dpaa/base/fman/of.c index b2d7c020..cada4124 100644 --- a/dpdk/drivers/bus/dpaa/base/fman/of.c +++ b/dpdk/drivers/bus/dpaa/base/fman/of.c @@ -215,6 +215,11 @@ linear_dir(struct dt_dir *d) DPAA_BUS_LOG(DEBUG, "Duplicate lphandle in %s", d->node.node.full_name); d->lphandle = f; + } else if (!strcmp(f->node.node.name, "phandle")) { + if (d->lphandle) + DPAA_BUS_LOG(DEBUG, "Duplicate lphandle in %s", + d->node.node.full_name); + d->lphandle = f; } else if (!strcmp(f->node.node.name, "#address-cells")) { if (d->a_cells) DPAA_BUS_LOG(DEBUG, "Duplicate a_cells in %s", diff --git a/dpdk/drivers/bus/dpaa/include/compat.h b/dpdk/drivers/bus/dpaa/include/compat.h index 42733aeb..1f56a4b5 100644 --- a/dpdk/drivers/bus/dpaa/include/compat.h +++ b/dpdk/drivers/bus/dpaa/include/compat.h @@ -80,9 +80,15 @@ */ /* Required compiler attributes */ +#ifndef __maybe_unused #define __maybe_unused __rte_unused +#endif +#ifndef __always_unused #define __always_unused __rte_unused +#endif +#ifndef __packed #define __packed __rte_packed +#endif #define noinline __attribute__((noinline)) #define L1_CACHE_BYTES 64 diff --git a/dpdk/drivers/bus/fslmc/fslmc_bus.c b/dpdk/drivers/bus/fslmc/fslmc_bus.c index 480857e5..001e56ca 100644 --- a/dpdk/drivers/bus/fslmc/fslmc_bus.c +++ b/dpdk/drivers/bus/fslmc/fslmc_bus.c @@ -310,8 +310,9 @@ rte_fslmc_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, struct rte_dpaa2_device *dev; TAILQ_FOREACH(dev, &rte_fslmc_bus.device_list, next) { - if (start && &dev->device == start) { - start = NULL; /* starting point found */ + if (start != NULL) { + if (&dev->device == start) + start = NULL; /* starting point found */ continue; } diff --git a/dpdk/drivers/bus/pci/linux/pci.c b/dpdk/drivers/bus/pci/linux/pci.c index 74deef3a..44440f22 100644 --- a/dpdk/drivers/bus/pci/linux/pci.c +++ b/dpdk/drivers/bus/pci/linux/pci.c @@ -43,6 +43,7 @@ #include #include #include +#include #include "eal_private.h" #include "eal_filesystem.h" @@ -582,7 +583,6 @@ pci_one_device_iommu_support_va(struct rte_pci_device *dev) { #define VTD_CAP_MGAW_SHIFT 16 #define VTD_CAP_MGAW_MASK (0x3fULL << VTD_CAP_MGAW_SHIFT) -#define X86_VA_WIDTH 47 /* From Documentation/x86/x86_64/mm.txt */ struct rte_pci_addr *addr = &dev->addr; char filename[PATH_MAX]; FILE *fp; @@ -613,10 +613,12 @@ pci_one_device_iommu_support_va(struct rte_pci_device *dev) fclose(fp); mgaw = ((vtd_cap_reg & VTD_CAP_MGAW_MASK) >> VTD_CAP_MGAW_SHIFT) + 1; - if (mgaw < X86_VA_WIDTH) + + if (!rte_eal_check_dma_mask(mgaw)) + return true; + else return false; - return true; } #elif defined(RTE_ARCH_PPC_64) static bool @@ -640,13 +642,17 @@ pci_devices_iommu_support_va(void) { struct rte_pci_device *dev = NULL; struct rte_pci_driver *drv = NULL; + int iommu_dma_mask_check_done = 0; FOREACH_DRIVER_ON_PCIBUS(drv) { FOREACH_DEVICE_ON_PCIBUS(dev) { if (!rte_pci_match(drv, dev)) continue; - if (!pci_one_device_iommu_support_va(dev)) - return false; + if (!iommu_dma_mask_check_done) { + if (!pci_one_device_iommu_support_va(dev)) + return false; + iommu_dma_mask_check_done = 1; + } } } return true; diff --git a/dpdk/drivers/bus/pci/linux/pci_vfio.c b/dpdk/drivers/bus/pci/linux/pci_vfio.c index e7d7f5dc..745db260 100644 --- a/dpdk/drivers/bus/pci/linux/pci_vfio.c +++ b/dpdk/drivers/bus/pci/linux/pci_vfio.c @@ -671,7 +671,7 @@ pci_vfio_unmap_resource(struct rte_pci_device *dev) vfio_res_list = RTE_TAILQ_CAST(rte_vfio_tailq.head, mapped_pci_res_list); /* Get vfio_res */ TAILQ_FOREACH(vfio_res, vfio_res_list, next) { - if (memcmp(&vfio_res->pci_addr, &dev->addr, sizeof(dev->addr))) + if (rte_pci_addr_cmp(&vfio_res->pci_addr, &dev->addr)) continue; break; } diff --git a/dpdk/drivers/bus/pci/pci_common.c b/dpdk/drivers/bus/pci/pci_common.c index 104fdf90..6789748e 100644 --- a/dpdk/drivers/bus/pci/pci_common.c +++ b/dpdk/drivers/bus/pci/pci_common.c @@ -488,17 +488,20 @@ static struct rte_device * pci_find_device(const struct rte_device *start, rte_dev_cmp_t cmp, const void *data) { - struct rte_pci_device *dev; + const struct rte_pci_device *pstart; + struct rte_pci_device *pdev; - FOREACH_DEVICE_ON_PCIBUS(dev) { - if (start && &dev->device == start) { - start = NULL; /* starting point found */ - continue; - } - if (cmp(&dev->device, data) == 0) - return &dev->device; + if (start != NULL) { + pstart = RTE_DEV_TO_PCI_CONST(start); + pdev = TAILQ_NEXT(pstart, next); + } else { + pdev = TAILQ_FIRST(&rte_pci_bus.device_list); + } + while (pdev != NULL) { + if (cmp(&pdev->device, data) == 0) + return &pdev->device; + pdev = TAILQ_NEXT(pdev, next); } - return NULL; } diff --git a/dpdk/drivers/bus/pci/rte_bus_pci.h b/dpdk/drivers/bus/pci/rte_bus_pci.h index d4a29964..c6af61d7 100644 --- a/dpdk/drivers/bus/pci/rte_bus_pci.h +++ b/dpdk/drivers/bus/pci/rte_bus_pci.h @@ -103,6 +103,9 @@ struct rte_pci_device { */ #define RTE_DEV_TO_PCI(ptr) container_of(ptr, struct rte_pci_device, device) +#define RTE_DEV_TO_PCI_CONST(ptr) \ + container_of(ptr, const struct rte_pci_device, device) + #define RTE_ETH_DEV_TO_PCI(eth_dev) RTE_DEV_TO_PCI((eth_dev)->device) /** Any PCI device identifier (vendor, device, ...) */ diff --git a/dpdk/drivers/bus/vdev/vdev.c b/dpdk/drivers/bus/vdev/vdev.c index ba0ed7a2..a0ffb534 100644 --- a/dpdk/drivers/bus/vdev/vdev.c +++ b/dpdk/drivers/bus/vdev/vdev.c @@ -129,7 +129,7 @@ find_vdev(const char *name) TAILQ_FOREACH(dev, &vdev_device_list, next) { const char *devname = rte_vdev_device_name(dev); - if (!strncmp(devname, name, strlen(name))) + if (!strcmp(devname, name)) return dev; } diff --git a/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h index 8e583803..ae8c0c30 100644 --- a/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h +++ b/dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h @@ -211,9 +211,9 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .increment = 1 }, .digest_size = { - .min = 16, + .min = 1, .max = 16, - .increment = 0 + .increment = 1 }, .iv_size = { 0 } }, } @@ -232,9 +232,9 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .increment = 1 }, .digest_size = { - .min = 20, + .min = 1, .max = 20, - .increment = 0 + .increment = 1 }, .iv_size = { 0 } }, } @@ -253,9 +253,9 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .increment = 1 }, .digest_size = { - .min = 28, + .min = 1, .max = 28, - .increment = 0 + .increment = 1 }, .iv_size = { 0 } }, } @@ -274,9 +274,9 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .increment = 1 }, .digest_size = { - .min = 32, - .max = 32, - .increment = 0 + .min = 1, + .max = 32, + .increment = 1 }, .iv_size = { 0 } }, } @@ -295,9 +295,9 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .increment = 1 }, .digest_size = { - .min = 48, + .min = 1, .max = 48, - .increment = 0 + .increment = 1 }, .iv_size = { 0 } }, } @@ -316,9 +316,9 @@ static const struct rte_cryptodev_capabilities dpaa2_sec_capabilities[] = { .increment = 1 }, .digest_size = { - .min = 64, + .min = 1, .max = 64, - .increment = 0 + .increment = 1 }, .iv_size = { 0 } }, } diff --git a/dpdk/drivers/crypto/qat/qat_crypto.c b/dpdk/drivers/crypto/qat/qat_crypto.c index acd979d2..85a9ba0d 100644 --- a/dpdk/drivers/crypto/qat/qat_crypto.c +++ b/dpdk/drivers/crypto/qat/qat_crypto.c @@ -1292,9 +1292,8 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) { if (unlikely( - (cipher_param->cipher_length % BYTE_LENGTH != 0) - || (cipher_param->cipher_offset - % BYTE_LENGTH != 0))) { + (op->sym->cipher.data.length % BYTE_LENGTH != 0) || + (op->sym->cipher.data.offset % BYTE_LENGTH != 0))) { PMD_DRV_LOG(ERR, "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values"); op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; @@ -1327,8 +1326,9 @@ qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg, ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 || ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) { - if (unlikely((auth_param->auth_off % BYTE_LENGTH != 0) - || (auth_param->auth_len % BYTE_LENGTH != 0))) { + if (unlikely( + (op->sym->auth.data.offset % BYTE_LENGTH != 0) || + (op->sym->auth.data.length % BYTE_LENGTH != 0))) { PMD_DRV_LOG(ERR, "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values"); op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; diff --git a/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.c b/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.c index 822ce27c..59ece957 100644 --- a/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.c +++ b/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.c @@ -119,8 +119,10 @@ update_scheduler_capability(struct scheduler_ctx *sched_ctx) struct rte_cryptodev_capabilities tmp_caps[256] = { {0} }; uint32_t nb_caps = 0, i; - if (sched_ctx->capabilities) + if (sched_ctx->capabilities) { rte_free(sched_ctx->capabilities); + sched_ctx->capabilities = NULL; + } for (i = 0; i < sched_ctx->nb_slaves; i++) { struct rte_cryptodev_info dev_info; @@ -490,8 +492,10 @@ rte_cryptodev_scheduler_load_user_scheduler(uint8_t scheduler_id, sched_ctx->ops.option_set = scheduler->ops->option_set; sched_ctx->ops.option_get = scheduler->ops->option_get; - if (sched_ctx->private_ctx) + if (sched_ctx->private_ctx) { rte_free(sched_ctx->private_ctx); + sched_ctx->private_ctx = NULL; + } if (sched_ctx->ops.create_private_ctx) { int ret = (*sched_ctx->ops.create_private_ctx)(dev); diff --git a/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.h b/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.h index df22f2a9..84917d1d 100644 --- a/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.h +++ b/dpdk/drivers/crypto/scheduler/rte_cryptodev_scheduler.h @@ -59,7 +59,7 @@ extern "C" { #endif /** Maximum number of multi-core worker cores */ -#define RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES (64) +#define RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES (RTE_MAX_LCORE - 1) /** Round-robin scheduling mode string */ #define SCHEDULER_MODE_NAME_ROUND_ROBIN round-robin diff --git a/dpdk/drivers/crypto/scheduler/scheduler_multicore.c b/dpdk/drivers/crypto/scheduler/scheduler_multicore.c index 0cd5bce5..14c33b98 100644 --- a/dpdk/drivers/crypto/scheduler/scheduler_multicore.c +++ b/dpdk/drivers/crypto/scheduler/scheduler_multicore.c @@ -49,8 +49,8 @@ struct mc_scheduler_ctx { uint32_t num_workers; /**< Number of workers polling */ uint32_t stop_signal; - struct rte_ring *sched_enq_ring[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; - struct rte_ring *sched_deq_ring[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; + struct rte_ring *sched_enq_ring[RTE_MAX_LCORE]; + struct rte_ring *sched_deq_ring[RTE_MAX_LCORE]; }; struct mc_scheduler_qp_ctx { @@ -356,11 +356,13 @@ static int scheduler_create_private_ctx(struct rte_cryptodev *dev) { struct scheduler_ctx *sched_ctx = dev->data->dev_private; - struct mc_scheduler_ctx *mc_ctx; + struct mc_scheduler_ctx *mc_ctx = NULL; uint16_t i; - if (sched_ctx->private_ctx) + if (sched_ctx->private_ctx) { rte_free(sched_ctx->private_ctx); + sched_ctx->private_ctx = NULL; + } mc_ctx = rte_zmalloc_socket(NULL, sizeof(struct mc_scheduler_ctx), 0, rte_socket_id()); @@ -373,25 +375,48 @@ scheduler_create_private_ctx(struct rte_cryptodev *dev) for (i = 0; i < sched_ctx->nb_wc; i++) { char r_name[16]; - snprintf(r_name, sizeof(r_name), MC_SCHED_ENQ_RING_NAME_PREFIX "%u", i); - mc_ctx->sched_enq_ring[i] = rte_ring_create(r_name, PER_SLAVE_BUFF_SIZE, - rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); + snprintf(r_name, sizeof(r_name), MC_SCHED_ENQ_RING_NAME_PREFIX + "%u_%u", dev->data->dev_id, i); + mc_ctx->sched_enq_ring[i] = rte_ring_lookup(r_name); if (!mc_ctx->sched_enq_ring[i]) { - CS_LOG_ERR("Cannot create ring for worker %u", i); - return -1; + mc_ctx->sched_enq_ring[i] = rte_ring_create(r_name, + PER_SLAVE_BUFF_SIZE, + rte_socket_id(), + RING_F_SC_DEQ | RING_F_SP_ENQ); + if (!mc_ctx->sched_enq_ring[i]) { + CS_LOG_ERR("Cannot create ring for worker %u", + i); + goto exit; + } } - snprintf(r_name, sizeof(r_name), MC_SCHED_DEQ_RING_NAME_PREFIX "%u", i); - mc_ctx->sched_deq_ring[i] = rte_ring_create(r_name, PER_SLAVE_BUFF_SIZE, - rte_socket_id(), RING_F_SC_DEQ | RING_F_SP_ENQ); + snprintf(r_name, sizeof(r_name), MC_SCHED_DEQ_RING_NAME_PREFIX + "%u_%u", dev->data->dev_id, i); + mc_ctx->sched_deq_ring[i] = rte_ring_lookup(r_name); if (!mc_ctx->sched_deq_ring[i]) { - CS_LOG_ERR("Cannot create ring for worker %u", i); - return -1; + mc_ctx->sched_deq_ring[i] = rte_ring_create(r_name, + PER_SLAVE_BUFF_SIZE, + rte_socket_id(), + RING_F_SC_DEQ | RING_F_SP_ENQ); + if (!mc_ctx->sched_deq_ring[i]) { + CS_LOG_ERR("Cannot create ring for worker %u", + i); + goto exit; + } } } sched_ctx->private_ctx = (void *)mc_ctx; return 0; + +exit: + for (i = 0; i < sched_ctx->nb_wc; i++) { + rte_ring_free(mc_ctx->sched_enq_ring[i]); + rte_ring_free(mc_ctx->sched_deq_ring[i]); + } + rte_free(mc_ctx); + + return -1; } struct rte_cryptodev_scheduler_ops scheduler_mc_ops = { diff --git a/dpdk/drivers/crypto/scheduler/scheduler_pkt_size_distr.c b/dpdk/drivers/crypto/scheduler/scheduler_pkt_size_distr.c index 1dd1bc32..4874191b 100644 --- a/dpdk/drivers/crypto/scheduler/scheduler_pkt_size_distr.c +++ b/dpdk/drivers/crypto/scheduler/scheduler_pkt_size_distr.c @@ -362,8 +362,10 @@ scheduler_create_private_ctx(struct rte_cryptodev *dev) struct scheduler_ctx *sched_ctx = dev->data->dev_private; struct psd_scheduler_ctx *psd_ctx; - if (sched_ctx->private_ctx) + if (sched_ctx->private_ctx) { rte_free(sched_ctx->private_ctx); + sched_ctx->private_ctx = NULL; + } psd_ctx = rte_zmalloc_socket(NULL, sizeof(struct psd_scheduler_ctx), 0, rte_socket_id()); diff --git a/dpdk/drivers/crypto/scheduler/scheduler_pmd.c b/dpdk/drivers/crypto/scheduler/scheduler_pmd.c index acdf6361..fcba1197 100644 --- a/dpdk/drivers/crypto/scheduler/scheduler_pmd.c +++ b/dpdk/drivers/crypto/scheduler/scheduler_pmd.c @@ -48,7 +48,8 @@ struct scheduler_init_params { uint32_t nb_slaves; enum rte_cryptodev_scheduler_mode mode; uint32_t enable_ordering; - uint64_t wcmask; + uint16_t wc_pool[RTE_MAX_LCORE]; + uint16_t nb_wc; char slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES] [RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; }; @@ -114,10 +115,6 @@ cryptodev_scheduler_create(const char *name, return -EFAULT; } - if (init_params->wcmask != 0) - RTE_LOG(INFO, PMD, " workers core mask = %"PRIx64"\n", - init_params->wcmask); - dev->driver_id = cryptodev_driver_id; dev->dev_ops = rte_crypto_scheduler_pmd_ops; @@ -128,15 +125,12 @@ cryptodev_scheduler_create(const char *name, if (init_params->mode == CDEV_SCHED_MODE_MULTICORE) { uint16_t i; - sched_ctx->nb_wc = 0; + sched_ctx->nb_wc = init_params->nb_wc; - for (i = 0; i < RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES; i++) { - if (init_params->wcmask & (1ULL << i)) { - sched_ctx->wc_pool[sched_ctx->nb_wc++] = i; - RTE_LOG(INFO, PMD, - " Worker core[%u]=%u added\n", - sched_ctx->nb_wc-1, i); - } + for (i = 0; i < sched_ctx->nb_wc; i++) { + sched_ctx->wc_pool[i] = init_params->wc_pool[i]; + RTE_LOG(INFO, PMD, " Worker core[%u]=%u added\n", + i, sched_ctx->wc_pool[i]); } } @@ -260,9 +254,47 @@ static int parse_coremask_arg(const char *key __rte_unused, const char *value, void *extra_args) { + int i, j, val; + uint16_t idx = 0; + char c; struct scheduler_init_params *params = extra_args; - params->wcmask = strtoull(value, NULL, 16); + params->nb_wc = 0; + + if (value == NULL) + return -1; + /* Remove all blank characters ahead and after . + * Remove 0x/0X if exists. + */ + while (isblank(*value)) + value++; + if (value[0] == '0' && ((value[1] == 'x') || (value[1] == 'X'))) + value += 2; + i = strlen(value); + while ((i > 0) && isblank(value[i - 1])) + i--; + + if (i == 0) + return -1; + + for (i = i - 1; i >= 0 && idx < RTE_MAX_LCORE; i--) { + c = value[i]; + if (isxdigit(c) == 0) { + /* invalid characters */ + return -1; + } + if (isdigit(c)) + val = c - '0'; + else if (isupper(c)) + val = c - 'A' + 10; + else + val = c - 'a' + 10; + + for (j = 0; j < 4 && idx < RTE_MAX_LCORE; j++, idx++) { + if ((1 << j) & val) + params->wc_pool[params->nb_wc++] = idx; + } + } return 0; } @@ -274,7 +306,7 @@ parse_corelist_arg(const char *key __rte_unused, { struct scheduler_init_params *params = extra_args; - params->wcmask = 0ULL; + params->nb_wc = 0; const char *token = value; @@ -282,7 +314,11 @@ parse_corelist_arg(const char *key __rte_unused, char *rval; unsigned int core = strtoul(token, &rval, 10); - params->wcmask |= 1ULL << core; + if (core >= RTE_MAX_LCORE) { + CS_LOG_ERR("Invalid worker core %u, should be smaller " + "than %u.\n", core, RTE_MAX_LCORE); + } + params->wc_pool[params->nb_wc++] = (uint16_t)core; token = (const char *)rval; if (token[0] == '\0') break; diff --git a/dpdk/drivers/crypto/scheduler/scheduler_pmd_ops.c b/dpdk/drivers/crypto/scheduler/scheduler_pmd_ops.c index d9b52352..75433db2 100644 --- a/dpdk/drivers/crypto/scheduler/scheduler_pmd_ops.c +++ b/dpdk/drivers/crypto/scheduler/scheduler_pmd_ops.c @@ -74,6 +74,7 @@ scheduler_attach_init_slave(struct rte_cryptodev *dev) sched_ctx->init_slave_names[i]); rte_free(sched_ctx->init_slave_names[i]); + sched_ctx->init_slave_names[i] = NULL; sched_ctx->nb_init_slaves -= 1; } @@ -289,11 +290,15 @@ scheduler_pmd_close(struct rte_cryptodev *dev) } } - if (sched_ctx->private_ctx) + if (sched_ctx->private_ctx) { rte_free(sched_ctx->private_ctx); + sched_ctx->private_ctx = NULL; + } - if (sched_ctx->capabilities) + if (sched_ctx->capabilities) { rte_free(sched_ctx->capabilities); + sched_ctx->capabilities = NULL; + } return 0; } diff --git a/dpdk/drivers/crypto/scheduler/scheduler_pmd_private.h b/dpdk/drivers/crypto/scheduler/scheduler_pmd_private.h index e606716a..bab4334d 100644 --- a/dpdk/drivers/crypto/scheduler/scheduler_pmd_private.h +++ b/dpdk/drivers/crypto/scheduler/scheduler_pmd_private.h @@ -89,7 +89,7 @@ struct scheduler_ctx { char name[RTE_CRYPTODEV_SCHEDULER_NAME_MAX_LEN]; char description[RTE_CRYPTODEV_SCHEDULER_DESC_MAX_LEN]; - uint16_t wc_pool[RTE_CRYPTODEV_SCHEDULER_MAX_NB_WORKER_CORES]; + uint16_t wc_pool[RTE_MAX_LCORE]; uint16_t nb_wc; char *init_slave_names[RTE_CRYPTODEV_SCHEDULER_MAX_NB_SLAVES]; diff --git a/dpdk/drivers/crypto/zuc/rte_zuc_pmd.c b/dpdk/drivers/crypto/zuc/rte_zuc_pmd.c index 590224bf..8b13be9e 100644 --- a/dpdk/drivers/crypto/zuc/rte_zuc_pmd.c +++ b/dpdk/drivers/crypto/zuc/rte_zuc_pmd.c @@ -40,7 +40,7 @@ #include "rte_zuc_pmd_private.h" -#define ZUC_MAX_BURST 8 +#define ZUC_MAX_BURST 4 #define BYTE_LEN 8 static uint8_t cryptodev_driver_id; @@ -196,10 +196,10 @@ zuc_get_session(struct zuc_qp *qp, struct rte_crypto_op *op) return sess; } -/** Encrypt/decrypt mbufs with same cipher key. */ +/** Encrypt/decrypt mbufs. */ static uint8_t process_zuc_cipher_op(struct rte_crypto_op **ops, - struct zuc_session *session, + struct zuc_session **sessions, uint8_t num_ops) { unsigned i; @@ -208,6 +208,7 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, uint8_t *iv[ZUC_MAX_BURST]; uint32_t num_bytes[ZUC_MAX_BURST]; uint8_t *cipher_keys[ZUC_MAX_BURST]; + struct zuc_session *sess; for (i = 0; i < num_ops; i++) { if (((ops[i]->sym->cipher.data.length % BYTE_LEN) != 0) @@ -218,6 +219,8 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, break; } + sess = sessions[i]; + #ifdef RTE_LIBRTE_PMD_ZUC_DEBUG if (!rte_pktmbuf_is_contiguous(ops[i]->sym->m_src) || (ops[i]->sym->m_dst != NULL && @@ -239,10 +242,10 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->cipher.data.offset >> 3); iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *, - session->cipher_iv_offset); + sess->cipher_iv_offset); num_bytes[i] = ops[i]->sym->cipher.data.length >> 3; - cipher_keys[i] = session->pKey_cipher; + cipher_keys[i] = sess->pKey_cipher; processed_ops++; } @@ -253,10 +256,10 @@ process_zuc_cipher_op(struct rte_crypto_op **ops, return processed_ops; } -/** Generate/verify hash from mbufs with same hash key. */ +/** Generate/verify hash from mbufs. */ static int process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops, - struct zuc_session *session, + struct zuc_session **sessions, uint8_t num_ops) { unsigned i; @@ -265,6 +268,7 @@ process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops, uint32_t *dst; uint32_t length_in_bits; uint8_t *iv; + struct zuc_session *sess; for (i = 0; i < num_ops; i++) { /* Data must be byte aligned */ @@ -274,17 +278,19 @@ process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops, break; } + sess = sessions[i]; + length_in_bits = ops[i]->sym->auth.data.length; src = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) + (ops[i]->sym->auth.data.offset >> 3); iv = rte_crypto_op_ctod_offset(ops[i], uint8_t *, - session->auth_iv_offset); + sess->auth_iv_offset); - if (session->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { + if (sess->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY) { dst = (uint32_t *)qp->temp_digest; - sso_zuc_eia3_1_buffer(session->pKey_hash, + sso_zuc_eia3_1_buffer(sess->pKey_hash, iv, src, length_in_bits, dst); /* Verify digest. */ @@ -294,7 +300,7 @@ process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops, } else { dst = (uint32_t *)ops[i]->sym->auth.digest.data; - sso_zuc_eia3_1_buffer(session->pKey_hash, + sso_zuc_eia3_1_buffer(sess->pKey_hash, iv, src, length_in_bits, dst); } @@ -304,33 +310,34 @@ process_zuc_hash_op(struct zuc_qp *qp, struct rte_crypto_op **ops, return processed_ops; } -/** Process a batch of crypto ops which shares the same session. */ +/** Process a batch of crypto ops which shares the same operation type. */ static int -process_ops(struct rte_crypto_op **ops, struct zuc_session *session, +process_ops(struct rte_crypto_op **ops, enum zuc_operation op_type, + struct zuc_session **sessions, struct zuc_qp *qp, uint8_t num_ops, uint16_t *accumulated_enqueued_ops) { unsigned i; unsigned enqueued_ops, processed_ops; - switch (session->op) { + switch (op_type) { case ZUC_OP_ONLY_CIPHER: processed_ops = process_zuc_cipher_op(ops, - session, num_ops); + sessions, num_ops); break; case ZUC_OP_ONLY_AUTH: - processed_ops = process_zuc_hash_op(qp, ops, session, + processed_ops = process_zuc_hash_op(qp, ops, sessions, num_ops); break; case ZUC_OP_CIPHER_AUTH: - processed_ops = process_zuc_cipher_op(ops, session, + processed_ops = process_zuc_cipher_op(ops, sessions, num_ops); - process_zuc_hash_op(qp, ops, session, processed_ops); + process_zuc_hash_op(qp, ops, sessions, processed_ops); break; case ZUC_OP_AUTH_CIPHER: - processed_ops = process_zuc_hash_op(qp, ops, session, + processed_ops = process_zuc_hash_op(qp, ops, sessions, num_ops); - process_zuc_cipher_op(ops, session, processed_ops); + process_zuc_cipher_op(ops, sessions, processed_ops); break; default: /* Operation not supported. */ @@ -346,10 +353,10 @@ process_ops(struct rte_crypto_op **ops, struct zuc_session *session, ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; /* Free session if a session-less crypto op. */ if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { - memset(session, 0, sizeof(struct zuc_session)); + memset(sessions[i], 0, sizeof(struct zuc_session)); memset(ops[i]->sym->session, 0, rte_cryptodev_get_header_session_size()); - rte_mempool_put(qp->sess_mp, session); + rte_mempool_put(qp->sess_mp, sessions[i]); rte_mempool_put(qp->sess_mp, ops[i]->sym->session); ops[i]->sym->session = NULL; } @@ -370,7 +377,10 @@ zuc_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, struct rte_crypto_op *c_ops[ZUC_MAX_BURST]; struct rte_crypto_op *curr_c_op; - struct zuc_session *prev_sess = NULL, *curr_sess = NULL; + struct zuc_session *curr_sess; + struct zuc_session *sessions[ZUC_MAX_BURST]; + enum zuc_operation prev_zuc_op = ZUC_OP_NOT_SUPPORTED; + enum zuc_operation curr_zuc_op; struct zuc_qp *qp = queue_pair; unsigned i; uint8_t burst_size = 0; @@ -380,9 +390,6 @@ zuc_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, for (i = 0; i < nb_ops; i++) { curr_c_op = ops[i]; - /* Set status as enqueued (not processed yet) by default. */ - curr_c_op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; - curr_sess = zuc_get_session(qp, curr_c_op); if (unlikely(curr_sess == NULL || curr_sess->op == ZUC_OP_NOT_SUPPORTED)) { @@ -391,50 +398,63 @@ zuc_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, break; } - /* Batch ops that share the same session. */ - if (prev_sess == NULL) { - prev_sess = curr_sess; - c_ops[burst_size++] = curr_c_op; - } else if (curr_sess == prev_sess) { - c_ops[burst_size++] = curr_c_op; + curr_zuc_op = curr_sess->op; + + /* + * Batch ops that share the same operation type + * (cipher only, auth only...). + */ + if (burst_size == 0) { + prev_zuc_op = curr_zuc_op; + c_ops[0] = curr_c_op; + sessions[0] = curr_sess; + burst_size++; + } else if (curr_zuc_op == prev_zuc_op) { + c_ops[burst_size] = curr_c_op; + sessions[burst_size] = curr_sess; + burst_size++; /* * When there are enough ops to process in a batch, * process them, and start a new batch. */ if (burst_size == ZUC_MAX_BURST) { - processed_ops = process_ops(c_ops, prev_sess, - qp, burst_size, &enqueued_ops); + processed_ops = process_ops(c_ops, curr_zuc_op, + sessions, qp, burst_size, + &enqueued_ops); if (processed_ops < burst_size) { burst_size = 0; break; } burst_size = 0; - prev_sess = NULL; } } else { /* - * Different session, process the ops - * of the previous session. + * Different operation type, process the ops + * of the previous type. */ - processed_ops = process_ops(c_ops, prev_sess, - qp, burst_size, &enqueued_ops); + processed_ops = process_ops(c_ops, prev_zuc_op, + sessions, qp, burst_size, + &enqueued_ops); if (processed_ops < burst_size) { burst_size = 0; break; } burst_size = 0; - prev_sess = curr_sess; + prev_zuc_op = curr_zuc_op; - c_ops[burst_size++] = curr_c_op; + c_ops[0] = curr_c_op; + sessions[0] = curr_sess; + burst_size++; } } if (burst_size != 0) { - /* Process the crypto ops of the last session. */ - processed_ops = process_ops(c_ops, prev_sess, - qp, burst_size, &enqueued_ops); + /* Process the crypto ops of the last operation type. */ + processed_ops = process_ops(c_ops, prev_zuc_op, + sessions, qp, burst_size, + &enqueued_ops); } qp->qp_stats.enqueue_err_count += nb_ops - enqueued_ops; diff --git a/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c b/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c index eeeb2312..56ea124c 100644 --- a/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c +++ b/dpdk/drivers/event/dpaa2/dpaa2_eventdev.c @@ -489,7 +489,6 @@ dpaa2_eventdev_port_unlink(struct rte_eventdev *dev, void *port, dpio_remove_static_dequeue_channel(dpaa2_portal->dpio_dev->dpio, 0, dpaa2_portal->dpio_dev->token, evq_info->dpcon->dpcon_id); - evq_info->link = 0; } return (int)nb_unlinks; @@ -510,8 +509,6 @@ dpaa2_eventdev_port_link(struct rte_eventdev *dev, void *port, for (i = 0; i < nb_links; i++) { evq_info = &priv->evq_info[queues[i]]; - if (evq_info->link) - continue; ret = dpio_add_static_dequeue_channel( dpaa2_portal->dpio_dev->dpio, @@ -526,7 +523,6 @@ dpaa2_eventdev_port_link(struct rte_eventdev *dev, void *port, qbman_swp_push_set(dpaa2_portal->dpio_dev->sw_portal, channel_index, 1); evq_info->dpcon->channel_index = channel_index; - evq_info->link = 1; } RTE_SET_USED(priorities); @@ -540,7 +536,6 @@ err: dpio_remove_static_dequeue_channel(dpaa2_portal->dpio_dev->dpio, 0, dpaa2_portal->dpio_dev->token, evq_info->dpcon->dpcon_id); - evq_info->link = 0; } return ret; } diff --git a/dpdk/drivers/event/dpaa2/dpaa2_eventdev.h b/dpdk/drivers/event/dpaa2/dpaa2_eventdev.h index ae8e07e9..5b9c80e5 100644 --- a/dpdk/drivers/event/dpaa2/dpaa2_eventdev.h +++ b/dpdk/drivers/event/dpaa2/dpaa2_eventdev.h @@ -100,7 +100,6 @@ struct evq_info_t { struct dpaa2_dpci_dev *dpci; /* Configuration provided by the user */ uint32_t event_queue_cfg; - uint8_t link; }; struct dpaa2_eventdev { diff --git a/dpdk/drivers/event/octeontx/ssovf_evdev.c b/dpdk/drivers/event/octeontx/ssovf_evdev.c index 117b1453..9a468948 100644 --- a/dpdk/drivers/event/octeontx/ssovf_evdev.c +++ b/dpdk/drivers/event/octeontx/ssovf_evdev.c @@ -485,14 +485,9 @@ static int ssovf_eth_rx_adapter_start(const struct rte_eventdev *dev, const struct rte_eth_dev *eth_dev) { - int ret; - const struct octeontx_nic *nic = eth_dev->data->dev_private; RTE_SET_USED(dev); + RTE_SET_USED(eth_dev); - ret = strncmp(eth_dev->data->name, "eth_octeontx", 12); - if (ret) - return 0; - octeontx_pki_port_start(nic->port_id); return 0; } @@ -501,14 +496,9 @@ static int ssovf_eth_rx_adapter_stop(const struct rte_eventdev *dev, const struct rte_eth_dev *eth_dev) { - int ret; - const struct octeontx_nic *nic = eth_dev->data->dev_private; RTE_SET_USED(dev); + RTE_SET_USED(eth_dev); - ret = strncmp(eth_dev->data->name, "eth_octeontx", 12); - if (ret) - return 0; - octeontx_pki_port_stop(nic->port_id); return 0; } diff --git a/dpdk/drivers/net/af_packet/rte_eth_af_packet.c b/dpdk/drivers/net/af_packet/rte_eth_af_packet.c index d5154089..5a101cef 100644 --- a/dpdk/drivers/net/af_packet/rte_eth_af_packet.c +++ b/dpdk/drivers/net/af_packet/rte_eth_af_packet.c @@ -124,7 +124,7 @@ static struct rte_eth_link pmd_link = { .link_speed = ETH_SPEED_NUM_10G, .link_duplex = ETH_LINK_FULL_DUPLEX, .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_AUTONEG + .link_autoneg = ETH_LINK_FIXED, }; static uint16_t diff --git a/dpdk/drivers/net/bnx2x/bnx2x.c b/dpdk/drivers/net/bnx2x/bnx2x.c index 9394f6c5..e58684d6 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x.c +++ b/dpdk/drivers/net/bnx2x/bnx2x.c @@ -125,7 +125,6 @@ int bnx2x_nic_load(struct bnx2x_softc *sc); static int bnx2x_handle_sp_tq(struct bnx2x_softc *sc); static void bnx2x_handle_fp_tq(struct bnx2x_fastpath *fp, int scan_fp); -static void bnx2x_periodic_stop(struct bnx2x_softc *sc); static void bnx2x_ack_sb(struct bnx2x_softc *sc, uint8_t igu_sb_id, uint8_t storm, uint16_t index, uint8_t op, uint8_t update); @@ -170,10 +169,10 @@ bnx2x_dma_alloc(struct bnx2x_softc *sc, size_t size, struct bnx2x_dma *dma, dma->sc = sc; if (IS_PF(sc)) - sprintf(mz_name, "bnx2x%d_%s_%" PRIx64, SC_ABS_FUNC(sc), msg, + snprintf(mz_name, sizeof(mz_name), "bnx2x%d_%s_%" PRIx64, SC_ABS_FUNC(sc), msg, rte_get_timer_cycles()); else - sprintf(mz_name, "bnx2x%d_%s_%" PRIx64, sc->pcie_device, msg, + snprintf(mz_name, sizeof(mz_name), "bnx2x%d_%s_%" PRIx64, sc->pcie_device, msg, rte_get_timer_cycles()); /* Caller must take care that strlen(mz_name) < RTE_MEMZONE_NAMESIZE */ @@ -1971,9 +1970,6 @@ bnx2x_nic_unload(struct bnx2x_softc *sc, uint32_t unload_mode, uint8_t keep_link PMD_DRV_LOG(DEBUG, "Starting NIC unload..."); - /* stop the periodic callout */ - bnx2x_periodic_stop(sc); - /* mark driver as unloaded in shmem2 */ if (IS_PF(sc) && SHMEM2_HAS(sc, drv_capabilities_flag)) { val = SHMEM2_RD(sc, drv_capabilities_flag[SC_FW_MB_IDX(sc)]); @@ -4492,6 +4488,8 @@ static void bnx2x_handle_fp_tq(struct bnx2x_fastpath *fp, int scan_fp) struct bnx2x_softc *sc = fp->sc; uint8_t more_rx = FALSE; + PMD_DRV_LOG(DEBUG, "---> FP TASK QUEUE (%d) <--", fp->index); + /* update the fastpath index */ bnx2x_update_fp_sb_idx(fp); @@ -4508,7 +4506,7 @@ static void bnx2x_handle_fp_tq(struct bnx2x_fastpath *fp, int scan_fp) } bnx2x_ack_sb(sc, fp->igu_sb_id, USTORM_ID, - le16toh(fp->fp_hc_idx), IGU_INT_DISABLE, 1); + le16toh(fp->fp_hc_idx), IGU_INT_ENABLE, 1); } /* @@ -6999,16 +6997,6 @@ void bnx2x_link_status_update(struct bnx2x_softc *sc) } } -static void bnx2x_periodic_start(struct bnx2x_softc *sc) -{ - atomic_store_rel_long(&sc->periodic_flags, PERIODIC_GO); -} - -static void bnx2x_periodic_stop(struct bnx2x_softc *sc) -{ - atomic_store_rel_long(&sc->periodic_flags, PERIODIC_STOP); -} - static int bnx2x_initial_phy_init(struct bnx2x_softc *sc, int load_mode) { int rc, cfg_idx = bnx2x_get_link_cfg_idx(sc); @@ -7043,10 +7031,6 @@ static int bnx2x_initial_phy_init(struct bnx2x_softc *sc, int load_mode) bnx2x_link_report(sc); } - if (!CHIP_REV_IS_SLOW(sc)) { - bnx2x_periodic_start(sc); - } - sc->link_params.req_line_speed[cfg_idx] = req_line_speed; return rc; } @@ -7078,7 +7062,7 @@ void bnx2x_periodic_callout(struct bnx2x_softc *sc) { if ((sc->state != BNX2X_STATE_OPEN) || (atomic_load_acq_long(&sc->periodic_flags) == PERIODIC_STOP)) { - PMD_DRV_LOG(WARNING, "periodic callout exit (state=0x%x)", + PMD_DRV_LOG(INFO, "periodic callout exit (state=0x%x)", sc->state); return; } @@ -8289,16 +8273,6 @@ static int bnx2x_get_device_info(struct bnx2x_softc *sc) REG_WR(sc, PXP2_REG_PGL_ADDR_90_F1, 0); REG_WR(sc, PXP2_REG_PGL_ADDR_94_F1, 0); } - -/* - * Enable internal target-read (in case we are probed after PF - * FLR). Must be done prior to any BAR read access. Only for - * 57712 and up - */ - if (!CHIP_IS_E1x(sc)) { - REG_WR(sc, PGLUE_B_REG_INTERNAL_PFID_ENABLE_TARGET_READ, - 1); - } } /* get the nvram size */ @@ -9675,7 +9649,17 @@ int bnx2x_attach(struct bnx2x_softc *sc) bnx2x_init_rte(sc); if (IS_PF(sc)) { -/* get device info and set params */ + /* Enable internal target-read (in case we are probed after PF + * FLR). Must be done prior to any BAR read access. Only for + * 57712 and up + */ + if (!CHIP_IS_E1x(sc)) { + REG_WR(sc, PGLUE_B_REG_INTERNAL_PFID_ENABLE_TARGET_READ, + 1); + DELAY(200000); + } + + /* get device info and set params */ if (bnx2x_get_device_info(sc) != 0) { PMD_DRV_LOG(NOTICE, "getting device info"); return -ENXIO; @@ -9684,7 +9668,7 @@ int bnx2x_attach(struct bnx2x_softc *sc) /* get phy settings from shmem and 'and' against admin settings */ bnx2x_get_phy_info(sc); } else { -/* Left mac of VF unfilled, PF should set it for VF */ + /* Left mac of VF unfilled, PF should set it for VF */ memset(sc->link_params.mac_addr, 0, ETHER_ADDR_LEN); } diff --git a/dpdk/drivers/net/bnx2x/bnx2x.h b/dpdk/drivers/net/bnx2x/bnx2x.h index 17075d38..c93f148b 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x.h +++ b/dpdk/drivers/net/bnx2x/bnx2x.h @@ -1930,6 +1930,7 @@ void bnx2x_link_status_update(struct bnx2x_softc *sc); int bnx2x_complete_sp(struct bnx2x_softc *sc); int bnx2x_set_storm_rx_mode(struct bnx2x_softc *sc); void bnx2x_periodic_callout(struct bnx2x_softc *sc); +void bnx2x_periodic_stop(void *param); int bnx2x_vf_get_resources(struct bnx2x_softc *sc, uint8_t tx_count, uint8_t rx_count); void bnx2x_vf_close(struct bnx2x_softc *sc); diff --git a/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c b/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c index 95861a06..650d6ce6 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c +++ b/dpdk/drivers/net/bnx2x/bnx2x_ethdev.c @@ -13,6 +13,8 @@ #include #include +#include +#include /* * The set of PCI devices this driver supports @@ -78,26 +80,87 @@ static const struct rte_bnx2x_xstats_name_off bnx2x_xstats_strings[] = { offsetof(struct bnx2x_eth_stats, pfc_frames_received_lo)} }; -static void +/** + * Atomically reads the link status information from global + * structure rte_eth_dev. + * + * @param dev + * - Pointer to the structure rte_eth_dev to read from. + * - Pointer to the buffer to be saved with the link status. + * + * @return + * - On success, zero. + * - On failure, negative value. + */ +static inline int +bnx2x_dev_atomic_read_link_status(struct rte_eth_dev *dev, + struct rte_eth_link *link) +{ + struct rte_eth_link *dst = link; + struct rte_eth_link *src = &dev->data->dev_link; + + if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, + *(uint64_t *)src) == 0) + return -1; + + return 0; +} + +/** + * Atomically writes the link status information into global + * structure rte_eth_dev. + * + * @param dev + * - Pointer to the structure rte_eth_dev to read from. + * - Pointer to the buffer to be saved with the link status. + * + * @return + * - On success, zero. + * - On failure, negative value. + */ +static inline int +bnx2x_dev_atomic_write_link_status(struct rte_eth_dev *dev, + struct rte_eth_link *link) +{ + struct rte_eth_link *dst = &dev->data->dev_link; + struct rte_eth_link *src = link; + + if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst, + *(uint64_t *)src) == 0) + return -1; + + return 0; +} + +static int bnx2x_link_update(struct rte_eth_dev *dev) { struct bnx2x_softc *sc = dev->data->dev_private; + struct rte_eth_link orig; + struct rte_eth_link link; PMD_INIT_FUNC_TRACE(); + bnx2x_link_status_update(sc); + memset(&orig, 0, sizeof(orig)); + memset(&link, 0, sizeof(link)); + bnx2x_dev_atomic_read_link_status(dev, &orig); mb(); - dev->data->dev_link.link_speed = sc->link_vars.line_speed; + link.link_speed = sc->link_vars.line_speed; switch (sc->link_vars.duplex) { case DUPLEX_FULL: - dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; + link.link_duplex = ETH_LINK_FULL_DUPLEX; break; case DUPLEX_HALF: - dev->data->dev_link.link_duplex = ETH_LINK_HALF_DUPLEX; + link.link_duplex = ETH_LINK_HALF_DUPLEX; break; } - dev->data->dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & + link.link_autoneg = !(dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED); - dev->data->dev_link.link_status = sc->link_vars.link_up; + link.link_status = sc->link_vars.link_up; + bnx2x_dev_atomic_write_link_status(dev, &link); + + return (link.link_status == orig.link_status) ? -1 : 0; } static void @@ -106,8 +169,6 @@ bnx2x_interrupt_action(struct rte_eth_dev *dev) struct bnx2x_softc *sc = dev->data->dev_private; uint32_t link_status; - PMD_DEBUG_PERIODIC_LOG(INFO, "Interrupt handled"); - bnx2x_intr_legacy(sc, 0); if (sc->periodic_flags & PERIODIC_GO) @@ -125,14 +186,73 @@ bnx2x_interrupt_handler(void *param) struct rte_eth_dev *dev = (struct rte_eth_dev *)param; struct bnx2x_softc *sc = dev->data->dev_private; + PMD_DEBUG_PERIODIC_LOG(INFO, "Interrupt handled"); + bnx2x_interrupt_action(dev); rte_intr_enable(&sc->pci_dev->intr_handle); } +static void bnx2x_periodic_start(void *param) +{ + struct rte_eth_dev *dev = (struct rte_eth_dev *)param; + struct bnx2x_softc *sc = dev->data->dev_private; + int ret = 0; + + atomic_store_rel_long(&sc->periodic_flags, PERIODIC_GO); + bnx2x_interrupt_action(dev); + if (IS_PF(sc)) { + ret = rte_eal_alarm_set(BNX2X_SP_TIMER_PERIOD, + bnx2x_periodic_start, (void *)dev); + if (ret) { + PMD_DRV_LOG(ERR, "Unable to start periodic" + " timer rc %d", ret); + assert(false && "Unable to start periodic timer"); + } + } +} + +void bnx2x_periodic_stop(void *param) +{ + struct rte_eth_dev *dev = (struct rte_eth_dev *)param; + struct bnx2x_softc *sc = dev->data->dev_private; + + atomic_store_rel_long(&sc->periodic_flags, PERIODIC_STOP); + + rte_eal_alarm_cancel(bnx2x_periodic_start, (void *)dev); +} + /* * Devops - helper functions can be called from user application */ +static int +bnx2x_dev_link_update(struct rte_eth_dev *dev, + __rte_unused int wait_to_complete) +{ + PMD_INIT_FUNC_TRACE(); + + return bnx2x_link_update(dev); +} + +static int +bnx2xvf_dev_link_update(struct rte_eth_dev *dev, + __rte_unused int wait_to_complete) +{ + struct bnx2x_softc *sc = dev->data->dev_private; + int ret = 0; + + ret = bnx2x_link_update(dev); + + bnx2x_check_bull(sc); + if (sc->old_bulletin.valid_bitmap & (1 << CHANNEL_DOWN)) { + PMD_DRV_LOG(ERR, "PF indicated channel is down." + "VF device is no longer operational"); + dev->data->dev_link.link_status = ETH_LINK_DOWN; + } + + return ret; +} + static int bnx2x_dev_configure(struct rte_eth_dev *dev) { @@ -182,6 +302,10 @@ bnx2x_dev_start(struct rte_eth_dev *dev) PMD_INIT_FUNC_TRACE(); + /* start the periodic callout */ + if (sc->periodic_flags & PERIODIC_STOP) + bnx2x_periodic_start(dev); + ret = bnx2x_init(sc); if (ret) { PMD_DRV_LOG(DEBUG, "bnx2x_init failed (%d)", ret); @@ -222,12 +346,21 @@ bnx2x_dev_stop(struct rte_eth_dev *dev) bnx2x_interrupt_handler, (void *)dev); } + /* stop the periodic callout */ + bnx2x_periodic_stop(dev); + ret = bnx2x_nic_unload(sc, UNLOAD_NORMAL, FALSE); if (ret) { PMD_DRV_LOG(DEBUG, "bnx2x_nic_unload failed (%d)", ret); return; } + /* Update device link status */ + if (IS_PF(sc)) + bnx2x_dev_link_update(dev, 0); + else + bnx2xvf_dev_link_update(dev, 0); + return; } @@ -299,36 +432,6 @@ bnx2x_dev_allmulticast_disable(struct rte_eth_dev *dev) bnx2x_set_rx_mode(sc); } -static int -bnx2x_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) -{ - PMD_INIT_FUNC_TRACE(); - - int old_link_status = dev->data->dev_link.link_status; - - bnx2x_link_update(dev); - - return old_link_status == dev->data->dev_link.link_status ? -1 : 0; -} - -static int -bnx2xvf_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete) -{ - int old_link_status = dev->data->dev_link.link_status; - struct bnx2x_softc *sc = dev->data->dev_private; - - bnx2x_link_update(dev); - - bnx2x_check_bull(sc); - if (sc->old_bulletin.valid_bitmap & (1 << CHANNEL_DOWN)) { - PMD_DRV_LOG(ERR, "PF indicated channel is down." - "VF device is no longer operational"); - dev->data->dev_link.link_status = ETH_LINK_DOWN; - } - - return old_link_status == dev->data->dev_link.link_status ? -1 : 0; -} - static int bnx2x_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { @@ -580,6 +683,17 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf) return ret; } + /* schedule periodic poll for slowpath link events */ + if (IS_PF(sc)) { + ret = rte_eal_alarm_set(BNX2X_SP_TIMER_PERIOD, + bnx2x_periodic_start, (void *)eth_dev); + if (ret) { + PMD_DRV_LOG(ERR, "Unable to start periodic" + " timer rc %d", ret); + return -EINVAL; + } + } + eth_dev->data->mac_addrs = (struct ether_addr *)sc->link_params.mac_addr; PMD_DRV_LOG(INFO, "pcie_bus=%d, pcie_device=%d", @@ -594,18 +708,20 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf) if (IS_VF(sc)) { rte_spinlock_init(&sc->vf2pf_lock); - if (bnx2x_dma_alloc(sc, sizeof(struct bnx2x_vf_mbx_msg), - &sc->vf2pf_mbox_mapping, "vf2pf_mbox", - RTE_CACHE_LINE_SIZE) != 0) - return -ENOMEM; + ret = bnx2x_dma_alloc(sc, sizeof(struct bnx2x_vf_mbx_msg), + &sc->vf2pf_mbox_mapping, "vf2pf_mbox", + RTE_CACHE_LINE_SIZE); + if (ret) + goto out; sc->vf2pf_mbox = (struct bnx2x_vf_mbx_msg *) sc->vf2pf_mbox_mapping.vaddr; - if (bnx2x_dma_alloc(sc, sizeof(struct bnx2x_vf_bulletin), - &sc->pf2vf_bulletin_mapping, "vf2pf_bull", - RTE_CACHE_LINE_SIZE) != 0) - return -ENOMEM; + ret = bnx2x_dma_alloc(sc, sizeof(struct bnx2x_vf_bulletin), + &sc->pf2vf_bulletin_mapping, "vf2pf_bull", + RTE_CACHE_LINE_SIZE); + if (ret) + goto out; sc->pf2vf_bulletin = (struct bnx2x_vf_bulletin *) sc->pf2vf_bulletin_mapping.vaddr; @@ -613,10 +729,14 @@ bnx2x_common_dev_init(struct rte_eth_dev *eth_dev, int is_vf) ret = bnx2x_vf_get_resources(sc, sc->max_tx_queues, sc->max_rx_queues); if (ret) - return ret; + goto out; } return 0; + +out: + bnx2x_periodic_stop(eth_dev); + return ret; } static int diff --git a/dpdk/drivers/net/bnx2x/bnx2x_ethdev.h b/dpdk/drivers/net/bnx2x/bnx2x_ethdev.h index 967d6dc5..a53d437d 100644 --- a/dpdk/drivers/net/bnx2x/bnx2x_ethdev.h +++ b/dpdk/drivers/net/bnx2x/bnx2x_ethdev.h @@ -58,7 +58,6 @@ #define wmb() rte_wmb() #define rmb() rte_rmb() - #define MAX_QUEUES sysconf(_SC_NPROCESSORS_CONF) #define BNX2X_MIN_RX_BUF_SIZE 1024 @@ -72,6 +71,8 @@ /* Maximum number of Rx packets to process at a time */ #define BNX2X_RX_BUDGET 0xffffffff +#define BNX2X_SP_TIMER_PERIOD US_PER_S /* 1 second */ + #endif /* MAC address operations */ diff --git a/dpdk/drivers/net/bnx2x/elink.c b/dpdk/drivers/net/bnx2x/elink.c index 9d0f3136..74e1bead 100644 --- a/dpdk/drivers/net/bnx2x/elink.c +++ b/dpdk/drivers/net/bnx2x/elink.c @@ -4143,9 +4143,9 @@ static void elink_sfp_e3_set_transmitter(struct elink_params *params, elink_set_cfg_pin(sc, cfg_pin + 3, tx_en ^ 1); } -static void elink_warpcore_config_init(struct elink_phy *phy, - struct elink_params *params, - struct elink_vars *vars) +static uint8_t elink_warpcore_config_init(struct elink_phy *phy, + struct elink_params *params, + struct elink_vars *vars) { struct bnx2x_softc *sc = params->sc; uint32_t serdes_net_if; @@ -4222,7 +4222,7 @@ static void elink_warpcore_config_init(struct elink_phy *phy, case PORT_HW_CFG_NET_SERDES_IF_DXGXS: if (vars->line_speed != ELINK_SPEED_20000) { PMD_DRV_LOG(DEBUG, "Speed not supported yet"); - return; + return 0; } PMD_DRV_LOG(DEBUG, "Setting 20G DXGXS"); elink_warpcore_set_20G_DXGXS(sc, phy, lane); @@ -4242,13 +4242,15 @@ static void elink_warpcore_config_init(struct elink_phy *phy, PMD_DRV_LOG(DEBUG, "Unsupported Serdes Net Interface 0x%x", serdes_net_if); - return; + return 0; } } /* Take lane out of reset after configuration is finished */ elink_warpcore_reset_lane(sc, phy, 0); PMD_DRV_LOG(DEBUG, "Exit config init"); + + return 0; } static void elink_warpcore_link_reset(struct elink_phy *phy, @@ -5226,9 +5228,9 @@ static elink_status_t elink_get_link_speed_duplex(struct elink_phy *phy, return ELINK_STATUS_OK; } -static elink_status_t elink_link_settings_status(struct elink_phy *phy, - struct elink_params *params, - struct elink_vars *vars) +static uint8_t elink_link_settings_status(struct elink_phy *phy, + struct elink_params *params, + struct elink_vars *vars) { struct bnx2x_softc *sc = params->sc; @@ -5299,9 +5301,9 @@ static elink_status_t elink_link_settings_status(struct elink_phy *phy, return rc; } -static elink_status_t elink_warpcore_read_status(struct elink_phy *phy, - struct elink_params *params, - struct elink_vars *vars) +static uint8_t elink_warpcore_read_status(struct elink_phy *phy, + struct elink_params *params, + struct elink_vars *vars) { struct bnx2x_softc *sc = params->sc; uint8_t lane; @@ -5520,9 +5522,9 @@ static void elink_set_preemphasis(struct elink_phy *phy, } } -static void elink_xgxs_config_init(struct elink_phy *phy, - struct elink_params *params, - struct elink_vars *vars) +static uint8_t elink_xgxs_config_init(struct elink_phy *phy, + struct elink_params *params, + struct elink_vars *vars) { uint8_t enable_cl73 = (ELINK_SINGLE_MEDIA_DIRECT(params) || (params->loopback_mode == ELINK_LOOPBACK_XGXS)); @@ -5567,6 +5569,8 @@ static void elink_xgxs_config_init(struct elink_phy *phy, elink_initialize_sgmii_process(phy, params, vars); } + + return 0; } static elink_status_t elink_prepare_xgxs(struct elink_phy *phy, @@ -5751,8 +5755,8 @@ static void elink_link_int_ack(struct elink_params *params, } } -static elink_status_t elink_format_ver(uint32_t num, uint8_t * str, - uint16_t * len) +static uint8_t elink_format_ver(uint32_t num, uint8_t * str, + uint16_t * len) { uint8_t *str_ptr = str; uint32_t mask = 0xf0000000; @@ -5790,8 +5794,8 @@ static elink_status_t elink_format_ver(uint32_t num, uint8_t * str, return ELINK_STATUS_OK; } -static elink_status_t elink_null_format_ver(__rte_unused uint32_t spirom_ver, - uint8_t * str, uint16_t * len) +static uint8_t elink_null_format_ver(__rte_unused uint32_t spirom_ver, + uint8_t * str, uint16_t * len) { str[0] = '\0'; (*len)--; @@ -6802,9 +6806,9 @@ static void elink_8073_specific_func(struct elink_phy *phy, } } -static elink_status_t elink_8073_config_init(struct elink_phy *phy, - struct elink_params *params, - struct elink_vars *vars) +static uint8_t elink_8073_config_init(struct elink_phy *phy, + struct elink_params *params, + struct elink_vars *vars) { struct bnx2x_softc *sc = params->sc; uint16_t val = 0, tmp1; @@ -7097,9 +7101,9 @@ static void elink_8073_link_reset(__rte_unused struct elink_phy *phy, /******************************************************************/ /* BNX2X8705 PHY SECTION */ /******************************************************************/ -static elink_status_t elink_8705_config_init(struct elink_phy *phy, - struct elink_params *params, - __rte_unused struct elink_vars +static uint8_t elink_8705_config_init(struct elink_phy *phy, + struct elink_params *params, + __rte_unused struct elink_vars *vars) { struct bnx2x_softc *sc = params->sc; @@ -8403,9 +8407,9 @@ static uint8_t elink_8706_config_init(struct elink_phy *phy, return ELINK_STATUS_OK; } -static elink_status_t elink_8706_read_status(struct elink_phy *phy, - struct elink_params *params, - struct elink_vars *vars) +static uint8_t elink_8706_read_status(struct elink_phy *phy, + struct elink_params *params, + struct elink_vars *vars) { return elink_8706_8726_read_status(phy, params, vars); } @@ -8477,9 +8481,9 @@ static uint8_t elink_8726_read_status(struct elink_phy *phy, return link_up; } -static elink_status_t elink_8726_config_init(struct elink_phy *phy, - struct elink_params *params, - struct elink_vars *vars) +static uint8_t elink_8726_config_init(struct elink_phy *phy, + struct elink_params *params, + struct elink_vars *vars) { struct bnx2x_softc *sc = params->sc; PMD_DRV_LOG(DEBUG, "Initializing BNX2X8726"); @@ -8684,9 +8688,9 @@ static void elink_8727_config_speed(struct elink_phy *phy, } } -static elink_status_t elink_8727_config_init(struct elink_phy *phy, - struct elink_params *params, - __rte_unused struct elink_vars +static uint8_t elink_8727_config_init(struct elink_phy *phy, + struct elink_params *params, + __rte_unused struct elink_vars *vars) { uint32_t tx_en_mode; @@ -9291,7 +9295,7 @@ static elink_status_t elink_848xx_cmn_config_init(struct elink_phy *phy, return ELINK_STATUS_OK; } -static elink_status_t elink_8481_config_init(struct elink_phy *phy, +static uint8_t elink_8481_config_init(struct elink_phy *phy, struct elink_params *params, struct elink_vars *vars) { @@ -9442,8 +9446,8 @@ static uint8_t elink_84833_get_reset_gpios(struct bnx2x_softc *sc, return reset_gpios; } -static elink_status_t elink_84833_hw_reset_phy(struct elink_phy *phy, - struct elink_params *params) +static void elink_84833_hw_reset_phy(struct elink_phy *phy, + struct elink_params *params) { struct bnx2x_softc *sc = params->sc; uint8_t reset_gpios; @@ -9471,8 +9475,6 @@ static elink_status_t elink_84833_hw_reset_phy(struct elink_phy *phy, MISC_REGISTERS_GPIO_OUTPUT_LOW); DELAY(10); PMD_DRV_LOG(DEBUG, "84833 hw reset on pin values 0x%x", reset_gpios); - - return ELINK_STATUS_OK; } static elink_status_t elink_8483x_disable_eee(struct elink_phy *phy, @@ -9513,9 +9515,9 @@ static elink_status_t elink_8483x_enable_eee(struct elink_phy *phy, } #define PHY84833_CONSTANT_LATENCY 1193 -static elink_status_t elink_848x3_config_init(struct elink_phy *phy, - struct elink_params *params, - struct elink_vars *vars) +static uint8_t elink_848x3_config_init(struct elink_phy *phy, + struct elink_params *params, + struct elink_vars *vars) { struct bnx2x_softc *sc = params->sc; uint8_t port, initialize = 1; @@ -9819,7 +9821,7 @@ static uint8_t elink_848xx_read_status(struct elink_phy *phy, return link_up; } -static elink_status_t elink_848xx_format_ver(uint32_t raw_ver, uint8_t * str, +static uint8_t elink_848xx_format_ver(uint32_t raw_ver, uint8_t * str, uint16_t * len) { elink_status_t status = ELINK_STATUS_OK; @@ -10146,9 +10148,9 @@ static void elink_54618se_specific_func(struct elink_phy *phy, } } -static elink_status_t elink_54618se_config_init(struct elink_phy *phy, - struct elink_params *params, - struct elink_vars *vars) +static uint8_t elink_54618se_config_init(struct elink_phy *phy, + struct elink_params *params, + struct elink_vars *vars) { struct bnx2x_softc *sc = params->sc; uint8_t port; @@ -10542,9 +10544,9 @@ static void elink_7101_config_loopback(struct elink_phy *phy, MDIO_XS_DEVAD, MDIO_XS_SFX7101_XGXS_TEST1, 0x100); } -static elink_status_t elink_7101_config_init(struct elink_phy *phy, - struct elink_params *params, - struct elink_vars *vars) +static uint8_t elink_7101_config_init(struct elink_phy *phy, + struct elink_params *params, + struct elink_vars *vars) { uint16_t fw_ver1, fw_ver2, val; struct bnx2x_softc *sc = params->sc; @@ -10614,8 +10616,8 @@ static uint8_t elink_7101_read_status(struct elink_phy *phy, return link_up; } -static elink_status_t elink_7101_format_ver(uint32_t spirom_ver, uint8_t * str, - uint16_t * len) +static uint8_t elink_7101_format_ver(uint32_t spirom_ver, uint8_t * str, + uint16_t * len) { if (*len < 5) return ELINK_STATUS_ERROR; @@ -10680,14 +10682,14 @@ static const struct elink_phy phy_null = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) NULL, - .read_status = (read_status_t) NULL, - .link_reset = (link_reset_t) NULL, - .config_loopback = (config_loopback_t) NULL, - .format_fw_ver = (format_fw_ver_t) NULL, - .hw_reset = (hw_reset_t) NULL, - .set_link_led = (set_link_led_t) NULL, - .phy_specific_func = (phy_specific_func_t) NULL + .config_init = NULL, + .read_status = NULL, + .link_reset = NULL, + .config_loopback = NULL, + .format_fw_ver = NULL, + .hw_reset = NULL, + .set_link_led = NULL, + .phy_specific_func = NULL }; static const struct elink_phy phy_serdes = { @@ -10714,14 +10716,14 @@ static const struct elink_phy phy_serdes = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_xgxs_config_init, - .read_status = (read_status_t) elink_link_settings_status, - .link_reset = (link_reset_t) elink_int_link_reset, - .config_loopback = (config_loopback_t) NULL, - .format_fw_ver = (format_fw_ver_t) NULL, - .hw_reset = (hw_reset_t) NULL, - .set_link_led = (set_link_led_t) NULL, - .phy_specific_func = (phy_specific_func_t) NULL + .config_init = elink_xgxs_config_init, + .read_status = elink_link_settings_status, + .link_reset = elink_int_link_reset, + .config_loopback = NULL, + .format_fw_ver = NULL, + .hw_reset = NULL, + .set_link_led = NULL, + .phy_specific_func = NULL }; static const struct elink_phy phy_xgxs = { @@ -10749,14 +10751,14 @@ static const struct elink_phy phy_xgxs = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_xgxs_config_init, - .read_status = (read_status_t) elink_link_settings_status, - .link_reset = (link_reset_t) elink_int_link_reset, - .config_loopback = (config_loopback_t) elink_set_xgxs_loopback, - .format_fw_ver = (format_fw_ver_t) NULL, - .hw_reset = (hw_reset_t) NULL, - .set_link_led = (set_link_led_t) NULL, - .phy_specific_func = (phy_specific_func_t) elink_xgxs_specific_func + .config_init = elink_xgxs_config_init, + .read_status = elink_link_settings_status, + .link_reset = elink_int_link_reset, + .config_loopback = elink_set_xgxs_loopback, + .format_fw_ver = NULL, + .hw_reset = NULL, + .set_link_led = NULL, + .phy_specific_func = elink_xgxs_specific_func }; static const struct elink_phy phy_warpcore = { @@ -10785,14 +10787,14 @@ static const struct elink_phy phy_warpcore = { .speed_cap_mask = 0, /* req_duplex = */ 0, /* rsrv = */ 0, - .config_init = (config_init_t) elink_warpcore_config_init, - .read_status = (read_status_t) elink_warpcore_read_status, - .link_reset = (link_reset_t) elink_warpcore_link_reset, - .config_loopback = (config_loopback_t) elink_set_warpcore_loopback, - .format_fw_ver = (format_fw_ver_t) NULL, - .hw_reset = (hw_reset_t) elink_warpcore_hw_reset, - .set_link_led = (set_link_led_t) NULL, - .phy_specific_func = (phy_specific_func_t) NULL + .config_init = elink_warpcore_config_init, + .read_status = elink_warpcore_read_status, + .link_reset = elink_warpcore_link_reset, + .config_loopback = elink_set_warpcore_loopback, + .format_fw_ver = NULL, + .hw_reset = elink_warpcore_hw_reset, + .set_link_led = NULL, + .phy_specific_func = NULL }; static const struct elink_phy phy_7101 = { @@ -10814,14 +10816,14 @@ static const struct elink_phy phy_7101 = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_7101_config_init, - .read_status = (read_status_t) elink_7101_read_status, - .link_reset = (link_reset_t) elink_common_ext_link_reset, - .config_loopback = (config_loopback_t) elink_7101_config_loopback, - .format_fw_ver = (format_fw_ver_t) elink_7101_format_ver, - .hw_reset = (hw_reset_t) elink_7101_hw_reset, - .set_link_led = (set_link_led_t) elink_7101_set_link_led, - .phy_specific_func = (phy_specific_func_t) NULL + .config_init = elink_7101_config_init, + .read_status = elink_7101_read_status, + .link_reset = elink_common_ext_link_reset, + .config_loopback = elink_7101_config_loopback, + .format_fw_ver = elink_7101_format_ver, + .hw_reset = elink_7101_hw_reset, + .set_link_led = elink_7101_set_link_led, + .phy_specific_func = NULL }; static const struct elink_phy phy_8073 = { @@ -10845,14 +10847,14 @@ static const struct elink_phy phy_8073 = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_8073_config_init, - .read_status = (read_status_t) elink_8073_read_status, - .link_reset = (link_reset_t) elink_8073_link_reset, - .config_loopback = (config_loopback_t) NULL, - .format_fw_ver = (format_fw_ver_t) elink_format_ver, - .hw_reset = (hw_reset_t) NULL, - .set_link_led = (set_link_led_t) NULL, - .phy_specific_func = (phy_specific_func_t) elink_8073_specific_func + .config_init = elink_8073_config_init, + .read_status = elink_8073_read_status, + .link_reset = elink_8073_link_reset, + .config_loopback = NULL, + .format_fw_ver = elink_format_ver, + .hw_reset = NULL, + .set_link_led = NULL, + .phy_specific_func = elink_8073_specific_func }; static const struct elink_phy phy_8705 = { @@ -10873,14 +10875,14 @@ static const struct elink_phy phy_8705 = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_8705_config_init, - .read_status = (read_status_t) elink_8705_read_status, - .link_reset = (link_reset_t) elink_common_ext_link_reset, - .config_loopback = (config_loopback_t) NULL, - .format_fw_ver = (format_fw_ver_t) elink_null_format_ver, - .hw_reset = (hw_reset_t) NULL, - .set_link_led = (set_link_led_t) NULL, - .phy_specific_func = (phy_specific_func_t) NULL + .config_init = elink_8705_config_init, + .read_status = elink_8705_read_status, + .link_reset = elink_common_ext_link_reset, + .config_loopback = NULL, + .format_fw_ver = elink_null_format_ver, + .hw_reset = NULL, + .set_link_led = NULL, + .phy_specific_func = NULL }; static const struct elink_phy phy_8706 = { @@ -10902,14 +10904,14 @@ static const struct elink_phy phy_8706 = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_8706_config_init, - .read_status = (read_status_t) elink_8706_read_status, - .link_reset = (link_reset_t) elink_common_ext_link_reset, - .config_loopback = (config_loopback_t) NULL, - .format_fw_ver = (format_fw_ver_t) elink_format_ver, - .hw_reset = (hw_reset_t) NULL, - .set_link_led = (set_link_led_t) NULL, - .phy_specific_func = (phy_specific_func_t) NULL + .config_init = elink_8706_config_init, + .read_status = elink_8706_read_status, + .link_reset = elink_common_ext_link_reset, + .config_loopback = NULL, + .format_fw_ver = elink_format_ver, + .hw_reset = NULL, + .set_link_led = NULL, + .phy_specific_func = NULL }; static const struct elink_phy phy_8726 = { @@ -10932,14 +10934,14 @@ static const struct elink_phy phy_8726 = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_8726_config_init, - .read_status = (read_status_t) elink_8726_read_status, - .link_reset = (link_reset_t) elink_8726_link_reset, - .config_loopback = (config_loopback_t) elink_8726_config_loopback, - .format_fw_ver = (format_fw_ver_t) elink_format_ver, - .hw_reset = (hw_reset_t) NULL, - .set_link_led = (set_link_led_t) NULL, - .phy_specific_func = (phy_specific_func_t) NULL + .config_init = elink_8726_config_init, + .read_status = elink_8726_read_status, + .link_reset = elink_8726_link_reset, + .config_loopback = elink_8726_config_loopback, + .format_fw_ver = elink_format_ver, + .hw_reset = NULL, + .set_link_led = NULL, + .phy_specific_func = NULL }; static const struct elink_phy phy_8727 = { @@ -10961,14 +10963,14 @@ static const struct elink_phy phy_8727 = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_8727_config_init, - .read_status = (read_status_t) elink_8727_read_status, - .link_reset = (link_reset_t) elink_8727_link_reset, - .config_loopback = (config_loopback_t) NULL, - .format_fw_ver = (format_fw_ver_t) elink_format_ver, - .hw_reset = (hw_reset_t) elink_8727_hw_reset, - .set_link_led = (set_link_led_t) elink_8727_set_link_led, - .phy_specific_func = (phy_specific_func_t) elink_8727_specific_func + .config_init = elink_8727_config_init, + .read_status = elink_8727_read_status, + .link_reset = elink_8727_link_reset, + .config_loopback = NULL, + .format_fw_ver = elink_format_ver, + .hw_reset = elink_8727_hw_reset, + .set_link_led = elink_8727_set_link_led, + .phy_specific_func = elink_8727_specific_func }; static const struct elink_phy phy_8481 = { @@ -10996,14 +10998,14 @@ static const struct elink_phy phy_8481 = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_8481_config_init, - .read_status = (read_status_t) elink_848xx_read_status, - .link_reset = (link_reset_t) elink_8481_link_reset, - .config_loopback = (config_loopback_t) NULL, - .format_fw_ver = (format_fw_ver_t) elink_848xx_format_ver, - .hw_reset = (hw_reset_t) elink_8481_hw_reset, - .set_link_led = (set_link_led_t) elink_848xx_set_link_led, - .phy_specific_func = (phy_specific_func_t) NULL + .config_init = elink_8481_config_init, + .read_status = elink_848xx_read_status, + .link_reset = elink_8481_link_reset, + .config_loopback = NULL, + .format_fw_ver = elink_848xx_format_ver, + .hw_reset = elink_8481_hw_reset, + .set_link_led = elink_848xx_set_link_led, + .phy_specific_func = NULL }; static const struct elink_phy phy_84823 = { @@ -11031,14 +11033,14 @@ static const struct elink_phy phy_84823 = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_848x3_config_init, - .read_status = (read_status_t) elink_848xx_read_status, - .link_reset = (link_reset_t) elink_848x3_link_reset, - .config_loopback = (config_loopback_t) NULL, - .format_fw_ver = (format_fw_ver_t) elink_848xx_format_ver, - .hw_reset = (hw_reset_t) NULL, - .set_link_led = (set_link_led_t) elink_848xx_set_link_led, - .phy_specific_func = (phy_specific_func_t) elink_848xx_specific_func + .config_init = elink_848x3_config_init, + .read_status = elink_848xx_read_status, + .link_reset = elink_848x3_link_reset, + .config_loopback = NULL, + .format_fw_ver = elink_848xx_format_ver, + .hw_reset = NULL, + .set_link_led = elink_848xx_set_link_led, + .phy_specific_func = elink_848xx_specific_func }; static const struct elink_phy phy_84833 = { @@ -11065,14 +11067,14 @@ static const struct elink_phy phy_84833 = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_848x3_config_init, - .read_status = (read_status_t) elink_848xx_read_status, - .link_reset = (link_reset_t) elink_848x3_link_reset, - .config_loopback = (config_loopback_t) NULL, - .format_fw_ver = (format_fw_ver_t) elink_848xx_format_ver, - .hw_reset = (hw_reset_t) elink_84833_hw_reset_phy, - .set_link_led = (set_link_led_t) elink_848xx_set_link_led, - .phy_specific_func = (phy_specific_func_t) elink_848xx_specific_func + .config_init = elink_848x3_config_init, + .read_status = elink_848xx_read_status, + .link_reset = elink_848x3_link_reset, + .config_loopback = NULL, + .format_fw_ver = elink_848xx_format_ver, + .hw_reset = elink_84833_hw_reset_phy, + .set_link_led = elink_848xx_set_link_led, + .phy_specific_func = elink_848xx_specific_func }; static const struct elink_phy phy_84834 = { @@ -11098,14 +11100,14 @@ static const struct elink_phy phy_84834 = { .speed_cap_mask = 0, .req_duplex = 0, .rsrv = 0, - .config_init = (config_init_t) elink_848x3_config_init, - .read_status = (read_status_t) elink_848xx_read_status, - .link_reset = (link_reset_t) elink_848x3_link_reset, - .config_loopback = (config_loopback_t) NULL, - .format_fw_ver = (format_fw_ver_t) elink_848xx_format_ver, - .hw_reset = (hw_reset_t) elink_84833_hw_reset_phy, - .set_link_led = (set_link_led_t) elink_848xx_set_link_led, - .phy_specific_func = (phy_specific_func_t) elink_848xx_specific_func + .config_init = elink_848x3_config_init, + .read_status = elink_848xx_read_status, + .link_reset = elink_848x3_link_reset, + .config_loopback = NULL, + .format_fw_ver = elink_848xx_format_ver, + .hw_reset = elink_84833_hw_reset_phy, + .set_link_led = elink_848xx_set_link_led, + .phy_specific_func = elink_848xx_specific_func }; static const struct elink_phy phy_54618se = { @@ -11131,14 +11133,14 @@ static const struct elink_phy phy_54618se = { .speed_cap_mask = 0, /* req_duplex = */ 0, /* rsrv = */ 0, - .config_init = (config_init_t) elink_54618se_config_init, - .read_status = (read_status_t) elink_54618se_read_status, - .link_reset = (link_reset_t) elink_54618se_link_reset, - .config_loopback = (config_loopback_t) elink_54618se_config_loopback, - .format_fw_ver = (format_fw_ver_t) NULL, - .hw_reset = (hw_reset_t) NULL, - .set_link_led = (set_link_led_t) elink_5461x_set_link_led, - .phy_specific_func = (phy_specific_func_t) elink_54618se_specific_func + .config_init = elink_54618se_config_init, + .read_status = elink_54618se_read_status, + .link_reset = elink_54618se_link_reset, + .config_loopback = elink_54618se_config_loopback, + .format_fw_ver = NULL, + .hw_reset = NULL, + .set_link_led = elink_5461x_set_link_led, + .phy_specific_func = elink_54618se_specific_func }; /*****************************************************************/ @@ -12919,7 +12921,7 @@ static void elink_check_kr2_wa(struct elink_params *params, */ not_kr2_device = (((base_page & 0x8000) == 0) || (((base_page & 0x8000) && - ((next_page & 0xe0) == 0x2)))); + ((next_page & 0xe0) == 0x20)))); /* In case KR2 is already disabled, check if we need to re-enable it */ if (!(vars->link_attr_sync & LINK_ATTR_SYNC_KR2_ENABLE)) { diff --git a/dpdk/drivers/net/bnxt/bnxt_ethdev.c b/dpdk/drivers/net/bnxt/bnxt_ethdev.c index 3eeca6f4..7466a642 100644 --- a/dpdk/drivers/net/bnxt/bnxt_ethdev.c +++ b/dpdk/drivers/net/bnxt/bnxt_ethdev.c @@ -248,6 +248,17 @@ static int bnxt_init_chip(struct bnxt *bp) /* VNIC configuration */ for (i = 0; i < bp->nr_vnics; i++) { struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; + uint32_t size = sizeof(*vnic->fw_grp_ids) * bp->max_ring_grps; + + vnic->fw_grp_ids = rte_zmalloc("vnic_fw_grp_ids", size, 0); + if (!vnic->fw_grp_ids) { + RTE_LOG(ERR, PMD, + "Failed to alloc %d bytes for group ids\n", + size); + rc = -ENOMEM; + goto err_out; + } + memset(vnic->fw_grp_ids, -1, size); rc = bnxt_hwrm_vnic_alloc(bp, vnic); if (rc) { @@ -400,10 +411,6 @@ static int bnxt_init_nic(struct bnxt *bp) bnxt_init_vnics(bp); bnxt_init_filters(bp); - rc = bnxt_init_chip(bp); - if (rc) - return rc; - return 0; } @@ -433,7 +440,7 @@ static void bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev, /* For the sake of symmetry, max_rx_queues = max_tx_queues */ dev_info->max_rx_queues = max_rx_rings; dev_info->max_tx_queues = max_rx_rings; - dev_info->reta_size = bp->max_rsscos_ctx; + dev_info->reta_size = HW_HASH_INDEX_SIZE; dev_info->hash_key_size = 40; max_vnics = bp->max_vnics; @@ -465,7 +472,8 @@ static void bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev, .wthresh = 0, }, .rx_free_thresh = 32, - .rx_drop_en = 0, + /* If no descriptors available, pkts are dropped by default */ + .rx_drop_en = 1, }; dev_info->default_txconf = (struct rte_eth_txconf) { @@ -572,7 +580,7 @@ static int bnxt_dev_start_op(struct rte_eth_dev *eth_dev) } bp->dev_stopped = 0; - rc = bnxt_init_nic(bp); + rc = bnxt_init_chip(bp); if (rc) goto error; @@ -631,6 +639,8 @@ static void bnxt_dev_stop_op(struct rte_eth_dev *eth_dev) } bnxt_set_hwrm_link_config(bp, false); bnxt_hwrm_port_clr_stats(bp); + bnxt_free_tx_mbufs(bp); + bnxt_free_rx_mbufs(bp); bnxt_shutdown_nic(bp); bp->dev_stopped = 1; } @@ -642,8 +652,6 @@ static void bnxt_dev_close_op(struct rte_eth_dev *eth_dev) if (bp->dev_stopped == 0) bnxt_dev_stop_op(eth_dev); - bnxt_free_tx_mbufs(bp); - bnxt_free_rx_mbufs(bp); bnxt_free_mem(bp); if (eth_dev->data->mac_addrs != NULL) { rte_free(eth_dev->data->mac_addrs); @@ -1271,9 +1279,9 @@ static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id) struct bnxt_vnic_info *vnic; unsigned int i; int rc = 0; - uint32_t en = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN | - HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK; - uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN; + uint32_t en = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN | + HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK; + uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN; /* Cycle through all VNICs */ for (i = 0; i < bp->nr_vnics; i++) { @@ -1320,8 +1328,8 @@ static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id) memcpy(new_filter->l2_addr, filter->l2_addr, ETHER_ADDR_LEN); /* MAC + VLAN ID filter */ - new_filter->l2_ovlan = vlan_id; - new_filter->l2_ovlan_mask = 0xF000; + new_filter->l2_ivlan = vlan_id; + new_filter->l2_ivlan_mask = 0xF000; new_filter->enables |= en; rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, @@ -1544,6 +1552,7 @@ static int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu) for (i = 0; i < bp->nr_vnics; i++) { struct bnxt_vnic_info *vnic = &bp->vnic_info[i]; + uint16_t size = 0; vnic->mru = bp->eth_dev->data->mtu + ETHER_HDR_LEN + ETHER_CRC_LEN + VLAN_TAG_SIZE * 2; @@ -1551,9 +1560,14 @@ static int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu) if (rc) break; - rc = bnxt_hwrm_vnic_plcmode_cfg(bp, vnic); - if (rc) - return rc; + size = rte_pktmbuf_data_room_size(bp->rx_queues[0]->mb_pool); + size -= RTE_PKTMBUF_HEADROOM; + + if (size < new_mtu) { + rc = bnxt_hwrm_vnic_plcmode_cfg(bp, vnic); + if (rc) + return rc; + } } return rc; @@ -3057,6 +3071,7 @@ skip_init: goto error_free_int; bnxt_enable_int(bp); + bnxt_init_nic(bp); return 0; diff --git a/dpdk/drivers/net/bnxt/bnxt_filter.c b/dpdk/drivers/net/bnxt/bnxt_filter.c index 32af6061..67daec41 100644 --- a/dpdk/drivers/net/bnxt/bnxt_filter.c +++ b/dpdk/drivers/net/bnxt/bnxt_filter.c @@ -159,6 +159,14 @@ void bnxt_free_filter_mem(struct bnxt *bp) rte_free(bp->filter_info); bp->filter_info = NULL; + + for (i = 0; i < bp->pf.max_vfs; i++) { + STAILQ_FOREACH(filter, &bp->pf.vf_info[i].filter, next) { + rte_free(filter); + STAILQ_REMOVE(&bp->pf.vf_info[i].filter, filter, + bnxt_filter_info, next); + } + } } int bnxt_alloc_filter_mem(struct bnxt *bp) @@ -1045,9 +1053,13 @@ bnxt_match_filter(struct bnxt *bp, struct bnxt_filter_info *nf) sizeof(nf->dst_ipaddr_mask))) { if (mf->dst_id == nf->dst_id) return -EEXIST; - /* Same Flow, Different queue + /* + * Same Flow, Different queue * Clear the old ntuple filter + * Reuse the matching L2 filter + * ID for the new filter */ + nf->fw_l2_filter_id = mf->fw_l2_filter_id; if (nf->filter_type == HWRM_CFA_EM_FILTER) bnxt_hwrm_clear_em_filter(bp, mf); if (nf->filter_type == HWRM_CFA_NTUPLE_FILTER) diff --git a/dpdk/drivers/net/bnxt/bnxt_hwrm.c b/dpdk/drivers/net/bnxt/bnxt_hwrm.c index ce214d7c..db3222f4 100644 --- a/dpdk/drivers/net/bnxt/bnxt_hwrm.c +++ b/dpdk/drivers/net/bnxt/bnxt_hwrm.c @@ -197,6 +197,10 @@ err_ret: RTE_LOG(ERR, PMD, "%s failed rc:%d\n", \ __func__, rc); \ rte_spinlock_unlock(&bp->hwrm_lock); \ + if (rc == HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED) \ + rc = -EACCES; \ + else if (rc > 0) \ + rc = -EINVAL; \ return rc; \ } \ if (resp->error_code) { \ @@ -218,6 +222,10 @@ err_ret: "%s error %d\n", __func__, rc); \ } \ rte_spinlock_unlock(&bp->hwrm_lock); \ + if (rc == HWRM_ERR_CODE_RESOURCE_ACCESS_DENIED) \ + rc = -EACCES; \ + else if (rc > 0) \ + rc = -EINVAL; \ return rc; \ } \ } while (0) @@ -252,6 +260,9 @@ int bnxt_hwrm_cfa_l2_set_rx_mask(struct bnxt *bp, struct hwrm_cfa_l2_set_rx_mask_output *resp = bp->hwrm_cmd_resp_addr; uint32_t mask = 0; + if (vnic->fw_vnic_id == INVALID_HW_RING_ID) + return rc; + HWRM_PREP(req, CFA_L2_SET_RX_MASK); req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); @@ -403,13 +414,13 @@ int bnxt_hwrm_set_l2_filter(struct bnxt *bp, req.l2_ovlan = filter->l2_ovlan; if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN) - req.l2_ovlan = filter->l2_ivlan; + req.l2_ivlan = filter->l2_ivlan; if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_OVLAN_MASK) req.l2_ovlan_mask = filter->l2_ovlan_mask; if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK) - req.l2_ovlan_mask = filter->l2_ivlan_mask; + req.l2_ivlan_mask = filter->l2_ivlan_mask; if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_ID) req.src_id = rte_cpu_to_le_32(filter->src_id); if (enables & HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_SRC_TYPE) @@ -1089,8 +1100,9 @@ int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic) /* map ring groups to this vnic */ RTE_LOG(DEBUG, PMD, "Alloc VNIC. Start %x, End %x\n", vnic->start_grp_id, vnic->end_grp_id); - for (i = vnic->start_grp_id, j = 0; i <= vnic->end_grp_id; i++, j++) + for (i = vnic->start_grp_id, j = 0; i < vnic->end_grp_id; i++, j++) vnic->fw_grp_ids[j] = bp->grp_info[i].fw_grp_id; + vnic->dflt_ring_grp = bp->grp_info[vnic->start_grp_id].fw_grp_id; vnic->rss_rule = (uint16_t)HWRM_NA_SIGNATURE; vnic->cos_rule = (uint16_t)HWRM_NA_SIGNATURE; @@ -1100,7 +1112,8 @@ int bnxt_hwrm_vnic_alloc(struct bnxt *bp, struct bnxt_vnic_info *vnic) HWRM_PREP(req, VNIC_ALLOC); if (vnic->func_default) - req.flags = HWRM_VNIC_ALLOC_INPUT_FLAGS_DEFAULT; + req.flags = + rte_cpu_to_le_32(HWRM_VNIC_ALLOC_INPUT_FLAGS_DEFAULT); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); HWRM_CHECK_RESULT(); @@ -1121,7 +1134,7 @@ static int bnxt_hwrm_vnic_plcmodes_qcfg(struct bnxt *bp, HWRM_PREP(req, VNIC_PLCMODES_QCFG); - req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); + req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); @@ -1149,7 +1162,7 @@ static int bnxt_hwrm_vnic_plcmodes_cfg(struct bnxt *bp, HWRM_PREP(req, VNIC_PLCMODES_CFG); - req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); + req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); req.flags = rte_cpu_to_le_32(pmode->flags); req.jumbo_thresh = rte_cpu_to_le_16(pmode->jumbo_thresh); req.hds_offset = rte_cpu_to_le_16(pmode->hds_offset); @@ -1381,6 +1394,11 @@ int bnxt_hwrm_vnic_plcmode_cfg(struct bnxt *bp, struct hwrm_vnic_plcmodes_cfg_output *resp = bp->hwrm_cmd_resp_addr; uint16_t size; + if (vnic->fw_vnic_id == INVALID_HW_RING_ID) { + RTE_LOG(DEBUG, PMD, "VNIC ID %x\n", vnic->fw_vnic_id); + return rc; + } + HWRM_PREP(req, VNIC_PLCMODES_CFG); req.flags = rte_cpu_to_le_32( @@ -1393,7 +1411,7 @@ int bnxt_hwrm_vnic_plcmode_cfg(struct bnxt *bp, size -= RTE_PKTMBUF_HEADROOM; req.jumbo_thresh = rte_cpu_to_le_16(size); - req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); + req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); @@ -1424,12 +1442,12 @@ int bnxt_hwrm_vnic_tpa_cfg(struct bnxt *bp, HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO | HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_ECN | HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_SAME_GRE_SEQ); - req.vnic_id = rte_cpu_to_le_32(vnic->fw_vnic_id); req.max_agg_segs = rte_cpu_to_le_16(5); req.max_aggs = rte_cpu_to_le_16(HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX); req.min_agg_len = rte_cpu_to_le_32(512); } + req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); @@ -1794,6 +1812,7 @@ int bnxt_clear_hwrm_vnic_filters(struct bnxt *bp, struct bnxt_vnic_info *vnic) rc = bnxt_hwrm_clear_ntuple_filter(bp, filter); else rc = bnxt_hwrm_clear_l2_filter(bp, filter); + STAILQ_REMOVE(&vnic->filter, filter, bnxt_filter_info, next); //if (rc) //break; } @@ -1881,6 +1900,8 @@ void bnxt_free_all_hwrm_resources(struct bnxt *bp) bnxt_hwrm_vnic_tpa_cfg(bp, vnic, false); bnxt_hwrm_vnic_free(bp, vnic); + + rte_free(vnic->fw_grp_ids); } /* Ring resources */ bnxt_free_all_hwrm_rings(bp); @@ -3093,13 +3114,12 @@ int bnxt_get_nvram_directory(struct bnxt *bp, uint32_t len, uint8_t *data) req.host_dest_addr = rte_cpu_to_le_64(dma_handle); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT(); - HWRM_UNLOCK(); - if (rc == 0) memcpy(data, buf, len > buflen ? buflen : len); rte_free(buf); + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); return rc; } @@ -3131,12 +3151,13 @@ int bnxt_hwrm_get_nvram_item(struct bnxt *bp, uint32_t index, req.offset = rte_cpu_to_le_32(offset); req.len = rte_cpu_to_le_32(length); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); - HWRM_CHECK_RESULT(); - HWRM_UNLOCK(); if (rc == 0) memcpy(data, buf, length); rte_free(buf); + HWRM_CHECK_RESULT(); + HWRM_UNLOCK(); + return rc; } @@ -3167,14 +3188,6 @@ int bnxt_hwrm_flash_nvram(struct bnxt *bp, uint16_t dir_type, rte_iova_t dma_handle; uint8_t *buf; - HWRM_PREP(req, NVM_WRITE); - - req.dir_type = rte_cpu_to_le_16(dir_type); - req.dir_ordinal = rte_cpu_to_le_16(dir_ordinal); - req.dir_ext = rte_cpu_to_le_16(dir_ext); - req.dir_attr = rte_cpu_to_le_16(dir_attr); - req.dir_data_length = rte_cpu_to_le_32(data_len); - buf = rte_malloc("nvm_write", data_len, 0); rte_mem_lock_page(buf); if (!buf) @@ -3187,14 +3200,22 @@ int bnxt_hwrm_flash_nvram(struct bnxt *bp, uint16_t dir_type, return -ENOMEM; } memcpy(buf, data, data_len); + + HWRM_PREP(req, NVM_WRITE); + + req.dir_type = rte_cpu_to_le_16(dir_type); + req.dir_ordinal = rte_cpu_to_le_16(dir_ordinal); + req.dir_ext = rte_cpu_to_le_16(dir_ext); + req.dir_attr = rte_cpu_to_le_16(dir_attr); + req.dir_data_length = rte_cpu_to_le_32(data_len); req.host_src_addr = rte_cpu_to_le_64(dma_handle); rc = bnxt_hwrm_send_message(bp, &req, sizeof(req)); + rte_free(buf); HWRM_CHECK_RESULT(); HWRM_UNLOCK(); - rte_free(buf); return rc; } diff --git a/dpdk/drivers/net/bnxt/bnxt_nvm_defs.h b/dpdk/drivers/net/bnxt/bnxt_nvm_defs.h index c5ccc9bc..6ce94bf2 100644 --- a/dpdk/drivers/net/bnxt/bnxt_nvm_defs.h +++ b/dpdk/drivers/net/bnxt/bnxt_nvm_defs.h @@ -1,13 +1,37 @@ -/* Broadcom NetXtreme-C/E network driver. +/*- + * BSD LICENSE * - * Copyright (c) 2014-2016 Broadcom Corporation - * Copyright (c) 2016-2017 Broadcom Limited + * Copyright(c) Broadcom Limited. + * All rights reserved. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Broadcom Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef _BNXT_NVM_DEFS_H_ #define _BNXT_NVM_DEFS_H_ diff --git a/dpdk/drivers/net/bnxt/bnxt_rxq.c b/dpdk/drivers/net/bnxt/bnxt_rxq.c index b4e9f38d..5088e9df 100644 --- a/dpdk/drivers/net/bnxt/bnxt_rxq.c +++ b/dpdk/drivers/net/bnxt/bnxt_rxq.c @@ -237,7 +237,8 @@ static void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq) if (rxq) { sw_ring = rxq->rx_ring->rx_buf_ring; if (sw_ring) { - for (i = 0; i < rxq->nb_rx_desc; i++) { + for (i = 0; + i < rxq->rx_ring->rx_ring_struct->ring_size; i++) { if (sw_ring[i].mbuf) { rte_pktmbuf_free_seg(sw_ring[i].mbuf); sw_ring[i].mbuf = NULL; @@ -247,7 +248,8 @@ static void bnxt_rx_queue_release_mbufs(struct bnxt_rx_queue *rxq) /* Free up mbufs in Agg ring */ sw_ring = rxq->rx_ring->ag_buf_ring; if (sw_ring) { - for (i = 0; i < rxq->nb_rx_desc; i++) { + for (i = 0; + i < rxq->rx_ring->ag_ring_struct->ring_size; i++) { if (sw_ring[i].mbuf) { rte_pktmbuf_free_seg(sw_ring[i].mbuf); sw_ring[i].mbuf = NULL; diff --git a/dpdk/drivers/net/bnxt/bnxt_rxr.c b/dpdk/drivers/net/bnxt/bnxt_rxr.c index 5128335f..c5c54847 100644 --- a/dpdk/drivers/net/bnxt/bnxt_rxr.c +++ b/dpdk/drivers/net/bnxt/bnxt_rxr.c @@ -469,11 +469,15 @@ static int bnxt_rx_pkt(struct rte_mbuf **rx_pkt, if (likely(RX_CMP_IP_CS_OK(rxcmp1))) mbuf->ol_flags |= PKT_RX_IP_CKSUM_GOOD; + else if (likely(RX_CMP_IP_CS_UNKNOWN(rxcmp1))) + mbuf->ol_flags |= PKT_RX_IP_CKSUM_UNKNOWN; else mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD; if (likely(RX_CMP_L4_CS_OK(rxcmp1))) mbuf->ol_flags |= PKT_RX_L4_CKSUM_GOOD; + else if (likely(RX_CMP_L4_CS_UNKNOWN(rxcmp1))) + mbuf->ol_flags |= PKT_RX_L4_CKSUM_UNKNOWN; else mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD; @@ -730,7 +734,7 @@ int bnxt_init_one_rx_ring(struct bnxt_rx_queue *rxq) if (rxq->rx_buf_use_size <= size) size = rxq->rx_buf_use_size; - type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT; + type = RX_PROD_PKT_BD_TYPE_RX_PROD_PKT | RX_PROD_PKT_BD_FLAGS_EOP_PAD; rxr = rxq->rx_ring; ring = rxr->rx_ring_struct; diff --git a/dpdk/drivers/net/bnxt/bnxt_rxr.h b/dpdk/drivers/net/bnxt/bnxt_rxr.h index a94373d1..4daa7e24 100644 --- a/dpdk/drivers/net/bnxt/bnxt_rxr.h +++ b/dpdk/drivers/net/bnxt/bnxt_rxr.h @@ -52,22 +52,36 @@ #define BNXT_TPA_OUTER_L3_OFF(hdr_info) \ ((hdr_info) & 0x1ff) -#define RX_CMP_L4_CS_BITS rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_L4_CS_CALC) +#define RX_CMP_L4_CS_BITS \ + rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_L4_CS_CALC | \ + RX_PKT_CMPL_FLAGS2_T_L4_CS_CALC) -#define RX_CMP_L4_CS_ERR_BITS rte_cpu_to_le_32(RX_PKT_CMPL_ERRORS_L4_CS_ERROR) +#define RX_CMP_L4_CS_ERR_BITS \ + rte_cpu_to_le_32(RX_PKT_CMPL_ERRORS_L4_CS_ERROR | \ + RX_PKT_CMPL_ERRORS_T_L4_CS_ERROR) #define RX_CMP_L4_CS_OK(rxcmp1) \ (((rxcmp1)->flags2 & RX_CMP_L4_CS_BITS) && \ !((rxcmp1)->errors_v2 & RX_CMP_L4_CS_ERR_BITS)) -#define RX_CMP_IP_CS_ERR_BITS rte_cpu_to_le_32(RX_PKT_CMPL_ERRORS_IP_CS_ERROR) +#define RX_CMP_L4_CS_UNKNOWN(rxcmp1) \ + !((rxcmp1)->flags2 & RX_CMP_L4_CS_BITS) -#define RX_CMP_IP_CS_BITS rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_IP_CS_CALC) +#define RX_CMP_IP_CS_ERR_BITS \ + rte_cpu_to_le_32(RX_PKT_CMPL_ERRORS_IP_CS_ERROR | \ + RX_PKT_CMPL_ERRORS_T_IP_CS_ERROR) + +#define RX_CMP_IP_CS_BITS \ + rte_cpu_to_le_32(RX_PKT_CMPL_FLAGS2_IP_CS_CALC | \ + RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC) #define RX_CMP_IP_CS_OK(rxcmp1) \ (((rxcmp1)->flags2 & RX_CMP_IP_CS_BITS) && \ !((rxcmp1)->errors_v2 & RX_CMP_IP_CS_ERR_BITS)) +#define RX_CMP_IP_CS_UNKNOWN(rxcmp1) \ + !((rxcmp1)->flags2 & RX_CMP_IP_CS_BITS) + enum pkt_hash_types { PKT_HASH_TYPE_NONE, /* Undefined type */ PKT_HASH_TYPE_L2, /* Input: src_MAC, dest_MAC */ diff --git a/dpdk/drivers/net/bnxt/bnxt_stats.c b/dpdk/drivers/net/bnxt/bnxt_stats.c index fe83d370..f8bb4ed9 100644 --- a/dpdk/drivers/net/bnxt/bnxt_stats.c +++ b/dpdk/drivers/net/bnxt/bnxt_stats.c @@ -296,6 +296,7 @@ int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev, count = 0; for (i = 0; i < RTE_DIM(bnxt_rx_stats_strings); i++) { uint64_t *rx_stats = (uint64_t *)bp->hw_rx_port_stats; + xstats[count].id = count; xstats[count].value = rte_le_to_cpu_64( *(uint64_t *)((char *)rx_stats + bnxt_rx_stats_strings[i].offset)); @@ -304,6 +305,7 @@ int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev, for (i = 0; i < RTE_DIM(bnxt_tx_stats_strings); i++) { uint64_t *tx_stats = (uint64_t *)bp->hw_tx_port_stats; + xstats[count].id = count; xstats[count].value = rte_le_to_cpu_64( *(uint64_t *)((char *)tx_stats + bnxt_tx_stats_strings[i].offset)); @@ -311,6 +313,7 @@ int bnxt_dev_xstats_get_op(struct rte_eth_dev *eth_dev, } /* The Tx drop pkts aka the Anti spoof coounter */ + xstats[count].id = count; xstats[count].value = rte_le_to_cpu_64(tx_drop_pkts); count++; diff --git a/dpdk/drivers/net/bnxt/bnxt_txr.c b/dpdk/drivers/net/bnxt/bnxt_txr.c index 03d2652c..f5ed03f1 100644 --- a/dpdk/drivers/net/bnxt/bnxt_txr.c +++ b/dpdk/drivers/net/bnxt/bnxt_txr.c @@ -161,7 +161,9 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt, if (tx_pkt->ol_flags & (PKT_TX_TCP_SEG | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | PKT_TX_IP_CKSUM | - PKT_TX_VLAN_PKT | PKT_TX_OUTER_IP_CKSUM)) + PKT_TX_VLAN_PKT | PKT_TX_OUTER_IP_CKSUM | + PKT_TX_TUNNEL_GRE | PKT_TX_TUNNEL_VXLAN | + PKT_TX_TUNNEL_GENEVE)) long_bd = true; tx_buf = &txr->tx_buf_ring[txr->tx_prod]; @@ -222,16 +224,46 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt, /* Outer IP, Inner IP, Inner TCP/UDP CSO */ txbd1->lflags |= TX_BD_FLG_TIP_IP_TCP_UDP_CHKSUM; txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_TCP_CKSUM) == + PKT_TX_OIP_IIP_TCP_CKSUM) { + /* Outer IP, Inner IP, Inner TCP/UDP CSO */ + txbd1->lflags |= TX_BD_FLG_TIP_IP_TCP_UDP_CHKSUM; + txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_UDP_CKSUM) == + PKT_TX_OIP_IIP_UDP_CKSUM) { + /* Outer IP, Inner IP, Inner TCP/UDP CSO */ + txbd1->lflags |= TX_BD_FLG_TIP_IP_TCP_UDP_CHKSUM; + txbd1->mss = 0; } else if ((tx_pkt->ol_flags & PKT_TX_IIP_TCP_UDP_CKSUM) == PKT_TX_IIP_TCP_UDP_CKSUM) { /* (Inner) IP, (Inner) TCP/UDP CSO */ txbd1->lflags |= TX_BD_FLG_IP_TCP_UDP_CHKSUM; txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_IIP_UDP_CKSUM) == + PKT_TX_IIP_UDP_CKSUM) { + /* (Inner) IP, (Inner) TCP/UDP CSO */ + txbd1->lflags |= TX_BD_FLG_IP_TCP_UDP_CHKSUM; + txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_IIP_TCP_CKSUM) == + PKT_TX_IIP_TCP_CKSUM) { + /* (Inner) IP, (Inner) TCP/UDP CSO */ + txbd1->lflags |= TX_BD_FLG_IP_TCP_UDP_CHKSUM; + txbd1->mss = 0; } else if ((tx_pkt->ol_flags & PKT_TX_OIP_TCP_UDP_CKSUM) == PKT_TX_OIP_TCP_UDP_CKSUM) { /* Outer IP, (Inner) TCP/UDP CSO */ txbd1->lflags |= TX_BD_FLG_TIP_TCP_UDP_CHKSUM; txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_OIP_UDP_CKSUM) == + PKT_TX_OIP_UDP_CKSUM) { + /* Outer IP, (Inner) TCP/UDP CSO */ + txbd1->lflags |= TX_BD_FLG_TIP_TCP_UDP_CHKSUM; + txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_OIP_TCP_CKSUM) == + PKT_TX_OIP_TCP_CKSUM) { + /* Outer IP, (Inner) TCP/UDP CSO */ + txbd1->lflags |= TX_BD_FLG_TIP_TCP_UDP_CHKSUM; + txbd1->mss = 0; } else if ((tx_pkt->ol_flags & PKT_TX_OIP_IIP_CKSUM) == PKT_TX_OIP_IIP_CKSUM) { /* Outer IP, Inner IP CSO */ @@ -242,11 +274,23 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt, /* TCP/UDP CSO */ txbd1->lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM; txbd1->mss = 0; - } else if (tx_pkt->ol_flags & PKT_TX_IP_CKSUM) { + } else if ((tx_pkt->ol_flags & PKT_TX_TCP_CKSUM) == + PKT_TX_TCP_CKSUM) { + /* TCP/UDP CSO */ + txbd1->lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM; + txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_UDP_CKSUM) == + PKT_TX_UDP_CKSUM) { + /* TCP/UDP CSO */ + txbd1->lflags |= TX_BD_LONG_LFLAGS_TCP_UDP_CHKSUM; + txbd1->mss = 0; + } else if ((tx_pkt->ol_flags & PKT_TX_IP_CKSUM) == + PKT_TX_IP_CKSUM) { /* IP CSO */ txbd1->lflags |= TX_BD_LONG_LFLAGS_IP_CHKSUM; txbd1->mss = 0; - } else if (tx_pkt->ol_flags & PKT_TX_OUTER_IP_CKSUM) { + } else if ((tx_pkt->ol_flags & PKT_TX_OUTER_IP_CKSUM) == + PKT_TX_OUTER_IP_CKSUM) { /* IP CSO */ txbd1->lflags |= TX_BD_LONG_LFLAGS_T_IP_CHKSUM; txbd1->mss = 0; @@ -270,6 +314,7 @@ static uint16_t bnxt_start_xmit(struct rte_mbuf *tx_pkt, } txbd->flags_type |= TX_BD_LONG_FLAGS_PACKET_END; + txbd1->lflags = rte_cpu_to_le_32(txbd1->lflags); txr->tx_prod = RING_NEXT(txr->tx_ring_struct, txr->tx_prod); diff --git a/dpdk/drivers/net/bnxt/bnxt_txr.h b/dpdk/drivers/net/bnxt/bnxt_txr.h index 2feac51d..0bc217f0 100644 --- a/dpdk/drivers/net/bnxt/bnxt_txr.h +++ b/dpdk/drivers/net/bnxt/bnxt_txr.h @@ -71,10 +71,20 @@ uint16_t bnxt_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, #define PKT_TX_OIP_IIP_TCP_UDP_CKSUM (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | \ PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM) +#define PKT_TX_OIP_IIP_UDP_CKSUM (PKT_TX_UDP_CKSUM | \ + PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM) +#define PKT_TX_OIP_IIP_TCP_CKSUM (PKT_TX_TCP_CKSUM | \ + PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM) #define PKT_TX_IIP_TCP_UDP_CKSUM (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | \ PKT_TX_IP_CKSUM) +#define PKT_TX_IIP_TCP_CKSUM (PKT_TX_TCP_CKSUM | PKT_TX_IP_CKSUM) +#define PKT_TX_IIP_UDP_CKSUM (PKT_TX_UDP_CKSUM | PKT_TX_IP_CKSUM) #define PKT_TX_OIP_TCP_UDP_CKSUM (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM | \ PKT_TX_OUTER_IP_CKSUM) +#define PKT_TX_OIP_UDP_CKSUM (PKT_TX_UDP_CKSUM | \ + PKT_TX_OUTER_IP_CKSUM) +#define PKT_TX_OIP_TCP_CKSUM (PKT_TX_TCP_CKSUM | \ + PKT_TX_OUTER_IP_CKSUM) #define PKT_TX_OIP_IIP_CKSUM (PKT_TX_IP_CKSUM | \ PKT_TX_OUTER_IP_CKSUM) #define PKT_TX_TCP_UDP_CKSUM (PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM) diff --git a/dpdk/drivers/net/bnxt/bnxt_vnic.c b/dpdk/drivers/net/bnxt/bnxt_vnic.c index 5bac2605..293f9604 100644 --- a/dpdk/drivers/net/bnxt/bnxt_vnic.c +++ b/dpdk/drivers/net/bnxt/bnxt_vnic.c @@ -67,7 +67,7 @@ void bnxt_init_vnics(struct bnxt *bp) { struct bnxt_vnic_info *vnic; uint16_t max_vnics; - int i, j; + int i; max_vnics = bp->max_vnics; STAILQ_INIT(&bp->free_vnic_list); @@ -78,9 +78,6 @@ void bnxt_init_vnics(struct bnxt *bp) vnic->cos_rule = (uint16_t)HWRM_NA_SIGNATURE; vnic->lb_rule = (uint16_t)HWRM_NA_SIGNATURE; - for (j = 0; j < MAX_QUEUES_PER_VNIC; j++) - vnic->fw_grp_ids[j] = (uint16_t)HWRM_NA_SIGNATURE; - prandom_bytes(vnic->rss_hash_key, HW_HASH_KEY_SIZE); STAILQ_INIT(&vnic->filter); STAILQ_INIT(&vnic->flow_list); diff --git a/dpdk/drivers/net/bnxt/bnxt_vnic.h b/dpdk/drivers/net/bnxt/bnxt_vnic.h index 875dc3c1..f18fb0a4 100644 --- a/dpdk/drivers/net/bnxt/bnxt_vnic.h +++ b/dpdk/drivers/net/bnxt/bnxt_vnic.h @@ -43,13 +43,9 @@ struct bnxt_vnic_info { uint16_t fw_vnic_id; /* returned by Chimp during alloc */ uint16_t rss_rule; -#define MAX_NUM_TRAFFIC_CLASSES 8 -#define MAX_NUM_RSS_QUEUES_PER_VNIC 16 -#define MAX_QUEUES_PER_VNIC (MAX_NUM_RSS_QUEUES_PER_VNIC + \ - MAX_NUM_TRAFFIC_CLASSES) uint16_t start_grp_id; uint16_t end_grp_id; - uint16_t fw_grp_ids[MAX_QUEUES_PER_VNIC]; + uint16_t *fw_grp_ids; uint16_t dflt_ring_grp; uint16_t mru; uint16_t hash_type; diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_api.c b/dpdk/drivers/net/bonding/rte_eth_bond_api.c index b8340356..e80338aa 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_api.c +++ b/dpdk/drivers/net/bonding/rte_eth_bond_api.c @@ -240,9 +240,12 @@ slave_vlan_filter_set(uint16_t bonded_port_id, uint16_t slave_port_id) for (i = 0, mask = 1; i < RTE_BITMAP_SLAB_BIT_SIZE; i ++, mask <<= 1) { - if (unlikely(slab & mask)) + if (unlikely(slab & mask)) { + uint16_t vlan_id = pos + i; + res = rte_eth_dev_vlan_filter(slave_port_id, - (uint16_t)pos, 1); + vlan_id, 1); + } } found = rte_bitmap_scan(internals->vlan_filter_bmp, &pos, &slab); @@ -672,9 +675,21 @@ rte_eth_bond_mac_address_reset(uint16_t bonded_port_id) internals->user_defined_mac = 0; if (internals->slave_count > 0) { + int slave_port; + /* Get the primary slave location based on the primary port + * number as, while slave_add(), we will keep the primary + * slave based on slave_count,but not based on the primary port. + */ + for (slave_port = 0; slave_port < internals->slave_count; + slave_port++) { + if (internals->slaves[slave_port].port_id == + internals->primary_port) + break; + } + /* Set MAC Address of Bonded Device */ if (mac_address_set(bonded_eth_dev, - &internals->slaves[internals->primary_port].persisted_mac_addr) + &internals->slaves[slave_port].persisted_mac_addr) != 0) { RTE_BOND_LOG(ERR, "Failed to set MAC address on bonded device"); return -1; diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_args.c b/dpdk/drivers/net/bonding/rte_eth_bond_args.c index e816da31..8c94cc61 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_args.c +++ b/dpdk/drivers/net/bonding/rte_eth_bond_args.c @@ -273,7 +273,7 @@ bond_ethdev_parse_primary_slave_port_id_kvarg(const char *key __rte_unused, if (primary_slave_port_id < 0) return -1; - *(uint8_t *)extra_args = (uint8_t)primary_slave_port_id; + *(uint16_t *)extra_args = (uint16_t)primary_slave_port_id; return 0; } diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c b/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c index 1d3fbeba..8880231e 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c +++ b/dpdk/drivers/net/bonding/rte_eth_bond_pmd.c @@ -1912,7 +1912,7 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) if (internals->slave_count == 0) { RTE_BOND_LOG(ERR, "Cannot start port since there are no slave devices"); - return -1; + goto out_err; } if (internals->user_defined_mac == 0) { @@ -1923,19 +1923,15 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) new_mac_addr = &internals->slaves[i].persisted_mac_addr; if (new_mac_addr == NULL) - return -1; + goto out_err; if (mac_address_set(eth_dev, new_mac_addr) != 0) { RTE_BOND_LOG(ERR, "bonded port (%d) failed to update MAC address", eth_dev->data->port_id); - return -1; + goto out_err; } } - /* Update all slave devices MACs*/ - if (mac_address_slaves_update(eth_dev) != 0) - return -1; - /* If bonded device is configure in promiscuous mode then re-apply config */ if (internals->promiscuous_en) bond_ethdev_promiscuous_enable(eth_dev); @@ -1959,7 +1955,7 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) "bonded port (%d) failed to reconfigure slave device (%d)", eth_dev->data->port_id, internals->slaves[i].port_id); - return -1; + goto out_err; } /* We will need to poll for link status if any slave doesn't * support interrupts @@ -1967,6 +1963,7 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) if (internals->slaves[i].link_status_poll_enabled) internals->link_status_polling_enabled = 1; } + /* start polling if needed */ if (internals->link_status_polling_enabled) { rte_eal_alarm_set( @@ -1975,6 +1972,10 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) (void *)&rte_eth_devices[internals->port_id]); } + /* Update all slave devices MACs*/ + if (mac_address_slaves_update(eth_dev) != 0) + goto out_err; + if (internals->user_defined_primary_port) bond_ethdev_primary_set(internals, internals->primary_port); @@ -1986,6 +1987,10 @@ bond_ethdev_start(struct rte_eth_dev *eth_dev) bond_tlb_enable(internals); return 0; + +out_err: + eth_dev->data->dev_started = 0; + return -1; } static void @@ -2043,7 +2048,6 @@ bond_ethdev_stop(struct rte_eth_dev *eth_dev) tlb_last_obytets[internals->active_slaves[i]] = 0; } - internals->active_slave_count = 0; internals->link_status_polling_enabled = 0; for (i = 0; i < internals->slave_count; i++) internals->slaves[i].last_link_status = 0; @@ -2519,6 +2523,11 @@ bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type, if (!valid_slave) return rc; + /* Synchronize lsc callback parallel calls either by real link event + * from the slaves PMDs or by the bonding PMD itself. + */ + rte_spinlock_lock(&internals->lsc_lock); + /* Search for port in active port list */ active_pos = find_slave_by_id(internals->active_slaves, internals->active_slave_count, port_id); @@ -2526,7 +2535,7 @@ bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type, rte_eth_link_get_nowait(port_id, &link); if (link.link_status) { if (active_pos < internals->active_slave_count) - return rc; + goto link_update; /* if no active slave ports then set this port to be primary port */ if (internals->active_slave_count < 1) { @@ -2546,7 +2555,7 @@ bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type, bond_ethdev_primary_set(internals, port_id); } else { if (active_pos == internals->active_slave_count) - return rc; + goto link_update; /* Remove from active slave list */ deactivate_slave(bonded_eth_dev, port_id); @@ -2565,6 +2574,7 @@ bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type, } } +link_update: /** * Update bonded device link properties after any change to active * slaves @@ -2599,7 +2609,10 @@ bond_ethdev_lsc_event_callback(uint16_t port_id, enum rte_eth_event_type type, NULL, NULL); } } - return 0; + + rte_spinlock_unlock(&internals->lsc_lock); + + return rc; } static int @@ -2766,6 +2779,7 @@ bond_alloc(struct rte_vdev_device *dev, uint8_t mode) eth_dev->data->dev_flags = RTE_ETH_DEV_INTR_LSC; rte_spinlock_init(&internals->lock); + rte_spinlock_init(&internals->lsc_lock); internals->port_id = eth_dev->data->port_id; internals->mode = BONDING_MODE_INVALID; @@ -2967,6 +2981,10 @@ bond_remove(struct rte_vdev_device *dev) eth_dev->tx_pkt_burst = NULL; internals = eth_dev->data->dev_private; + /* Try to release mempool used in mode6. If the bond + * device is not mode6, free the NULL is not problem. + */ + rte_mempool_free(internals->mode6.mempool); rte_bitmap_free(internals->vlan_filter_bmp); rte_free(internals->vlan_filter_bmpmem); rte_free(eth_dev->data->dev_private); diff --git a/dpdk/drivers/net/bonding/rte_eth_bond_private.h b/dpdk/drivers/net/bonding/rte_eth_bond_private.h index a5cfa6ac..dae8aab0 100644 --- a/dpdk/drivers/net/bonding/rte_eth_bond_private.h +++ b/dpdk/drivers/net/bonding/rte_eth_bond_private.h @@ -118,6 +118,7 @@ struct bond_dev_private { uint8_t mode; /**< Link Bonding Mode */ rte_spinlock_t lock; + rte_spinlock_t lsc_lock; uint16_t primary_port; /**< Primary Slave Port */ uint16_t current_primary_port; /**< Primary Slave Port */ diff --git a/dpdk/drivers/net/bonding/rte_pmd_bond_version.map b/dpdk/drivers/net/bonding/rte_pmd_bond_version.map index ec3374b0..03ddb44e 100644 --- a/dpdk/drivers/net/bonding/rte_pmd_bond_version.map +++ b/dpdk/drivers/net/bonding/rte_pmd_bond_version.map @@ -1,6 +1,7 @@ DPDK_2.0 { global: + rte_eth_bond_8023ad_slave_info; rte_eth_bond_active_slaves_get; rte_eth_bond_create; rte_eth_bond_link_monitoring_set; diff --git a/dpdk/drivers/net/cxgbe/base/t4_hw.c b/dpdk/drivers/net/cxgbe/base/t4_hw.c index 282e2e62..c3d46e02 100644 --- a/dpdk/drivers/net/cxgbe/base/t4_hw.c +++ b/dpdk/drivers/net/cxgbe/base/t4_hw.c @@ -4242,9 +4242,8 @@ struct flash_desc { int t4_get_flash_params(struct adapter *adapter) { /* - * Table for non-Numonix supported flash parts. Numonix parts are left - * to the preexisting well-tested code. All flash parts have 64KB - * sectors. + * Table for non-standard supported Flash parts. Note, all Flash + * parts must have 64KB sectors. */ static struct flash_desc supported_flash[] = { { 0x00150201, 4 << 20 }, /* Spansion 4MB S25FL032P */ @@ -4253,7 +4252,7 @@ int t4_get_flash_params(struct adapter *adapter) int ret; u32 flashid = 0; unsigned int part, manufacturer; - unsigned int density, size; + unsigned int density, size = 0; /** * Issue a Read ID Command to the Flash part. We decode supported @@ -4268,6 +4267,9 @@ int t4_get_flash_params(struct adapter *adapter) if (ret < 0) return ret; + /** + * Check to see if it's one of our non-standard supported Flash parts. + */ for (part = 0; part < ARRAY_SIZE(supported_flash); part++) { if (supported_flash[part].vendor_and_model_id == flashid) { adapter->params.sf_size = @@ -4278,6 +4280,15 @@ int t4_get_flash_params(struct adapter *adapter) } } + /** + * Decode Flash part size. The code below looks repetative with + * common encodings, but that's not guaranteed in the JEDEC + * specification for the Read JADEC ID command. The only thing that + * we're guaranteed by the JADEC specification is where the + * Manufacturer ID is in the returned result. After that each + * Manufacturer ~could~ encode things completely differently. + * Note, all Flash parts must have 64KB sectors. + */ manufacturer = flashid & 0xff; switch (manufacturer) { case 0x20: { /* Micron/Numonix */ @@ -4314,21 +4325,81 @@ int t4_get_flash_params(struct adapter *adapter) case 0x22: size = 1 << 28; /* 256MB */ break; - default: - dev_err(adapter, "Micron Flash Part has bad size, ID = %#x, Density code = %#x\n", - flashid, density); - return -EINVAL; } - - adapter->params.sf_size = size; - adapter->params.sf_nsec = size / SF_SEC_SIZE; break; } - default: - dev_err(adapter, "Unsupported Flash Part, ID = %#x\n", flashid); - return -EINVAL; + + case 0x9d: { /* ISSI -- Integrated Silicon Solution, Inc. */ + /** + * This Density -> Size decoding table is taken from ISSI + * Data Sheets. + */ + density = (flashid >> 16) & 0xff; + switch (density) { + case 0x16: + size = 1 << 25; /* 32MB */ + break; + case 0x17: + size = 1 << 26; /* 64MB */ + break; + } + break; } + case 0xc2: { /* Macronix */ + /** + * This Density -> Size decoding table is taken from Macronix + * Data Sheets. + */ + density = (flashid >> 16) & 0xff; + switch (density) { + case 0x17: + size = 1 << 23; /* 8MB */ + break; + case 0x18: + size = 1 << 24; /* 16MB */ + break; + } + break; + } + + case 0xef: { /* Winbond */ + /** + * This Density -> Size decoding table is taken from Winbond + * Data Sheets. + */ + density = (flashid >> 16) & 0xff; + switch (density) { + case 0x17: + size = 1 << 23; /* 8MB */ + break; + case 0x18: + size = 1 << 24; /* 16MB */ + break; + } + break; + } + } + + /* If we didn't recognize the FLASH part, that's no real issue: the + * Hardware/Software contract says that Hardware will _*ALWAYS*_ + * use a FLASH part which is at least 4MB in size and has 64KB + * sectors. The unrecognized FLASH part is likely to be much larger + * than 4MB, but that's all we really need. + */ + if (size == 0) { + dev_warn(adapter, + "Unknown Flash Part, ID = %#x, assuming 4MB\n", + flashid); + size = 1 << 22; + } + + /** + * Store decoded Flash size and fall through into vetting code. + */ + adapter->params.sf_size = size; + adapter->params.sf_nsec = size / SF_SEC_SIZE; + found: /* * We should reject adapters with FLASHes which are too small. So, emit diff --git a/dpdk/drivers/net/cxgbe/base/t4fw_interface.h b/dpdk/drivers/net/cxgbe/base/t4fw_interface.h index 6ca4f318..f0566e93 100644 --- a/dpdk/drivers/net/cxgbe/base/t4fw_interface.h +++ b/dpdk/drivers/net/cxgbe/base/t4fw_interface.h @@ -473,6 +473,11 @@ enum fw_iq_type { FW_IQ_TYPE_FL_INT_CAP, }; +enum fw_iq_iqtype { + FW_IQ_IQTYPE_NIC = 1, + FW_IQ_IQTYPE_OFLD, +}; + struct fw_iq_cmd { __be32 op_to_vfn; __be32 alloc_to_len16; @@ -606,6 +611,9 @@ struct fw_iq_cmd { (((x) >> S_FW_IQ_CMD_IQFLINTCONGEN) & M_FW_IQ_CMD_IQFLINTCONGEN) #define F_FW_IQ_CMD_IQFLINTCONGEN V_FW_IQ_CMD_IQFLINTCONGEN(1U) +#define S_FW_IQ_CMD_IQTYPE 24 +#define V_FW_IQ_CMD_IQTYPE(x) ((x) << S_FW_IQ_CMD_IQTYPE) + #define S_FW_IQ_CMD_FL0CNGCHMAP 20 #define M_FW_IQ_CMD_FL0CNGCHMAP 0xf #define V_FW_IQ_CMD_FL0CNGCHMAP(x) ((x) << S_FW_IQ_CMD_FL0CNGCHMAP) diff --git a/dpdk/drivers/net/cxgbe/cxgbe_compat.h b/dpdk/drivers/net/cxgbe/cxgbe_compat.h index 03bba9fe..c1cc936d 100644 --- a/dpdk/drivers/net/cxgbe/cxgbe_compat.h +++ b/dpdk/drivers/net/cxgbe/cxgbe_compat.h @@ -226,15 +226,6 @@ static inline int cxgbe_fls(int x) return x ? sizeof(x) * 8 - __builtin_clz(x) : 0; } -/** - * cxgbe_ffs - find first bit set - * @x: the word to search - */ -static inline int cxgbe_ffs(int x) -{ - return x ? __builtin_ffs(x) : 0; -} - static inline unsigned long ilog2(unsigned long n) { unsigned int e = 0; diff --git a/dpdk/drivers/net/cxgbe/sge.c b/dpdk/drivers/net/cxgbe/sge.c index fc10d958..51800845 100644 --- a/dpdk/drivers/net/cxgbe/sge.c +++ b/dpdk/drivers/net/cxgbe/sge.c @@ -1689,6 +1689,7 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, char z_name[RTE_MEMZONE_NAMESIZE]; char z_name_sw[RTE_MEMZONE_NAMESIZE]; unsigned int nb_refill; + u8 pciechan; /* Size needs to be multiple of 16, including status entry. */ iq->size = cxgbe_roundup(iq->size, 16); @@ -1708,6 +1709,9 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, c.op_to_vfn = htonl(V_FW_CMD_OP(FW_IQ_CMD) | F_FW_CMD_REQUEST | F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_IQ_CMD_PFN(adap->pf) | V_FW_IQ_CMD_VFN(0)); + + pciechan = pi->tx_chan; + c.alloc_to_len16 = htonl(F_FW_IQ_CMD_ALLOC | F_FW_IQ_CMD_IQSTART | (sizeof(c) / 16)); c.type_to_iqandstindex = @@ -1719,16 +1723,19 @@ int t4_sge_alloc_rxq(struct adapter *adap, struct sge_rspq *iq, bool fwevtq, V_FW_IQ_CMD_IQANDSTINDEX(intr_idx >= 0 ? intr_idx : -intr_idx - 1)); c.iqdroprss_to_iqesize = - htons(V_FW_IQ_CMD_IQPCIECH(cong > 0 ? cxgbe_ffs(cong) - 1 : - pi->tx_chan) | + htons(V_FW_IQ_CMD_IQPCIECH(pciechan) | F_FW_IQ_CMD_IQGTSMODE | V_FW_IQ_CMD_IQINTCNTTHRESH(iq->pktcnt_idx) | V_FW_IQ_CMD_IQESIZE(ilog2(iq->iqe_len) - 4)); c.iqsize = htons(iq->size); c.iqaddr = cpu_to_be64(iq->phys_addr); if (cong >= 0) - c.iqns_to_fl0congen = htonl(F_FW_IQ_CMD_IQFLINTCONGEN | - F_FW_IQ_CMD_IQRO); + c.iqns_to_fl0congen = + htonl(F_FW_IQ_CMD_IQFLINTCONGEN | + V_FW_IQ_CMD_IQTYPE(cong ? + FW_IQ_IQTYPE_NIC : + FW_IQ_IQTYPE_OFLD) | + F_FW_IQ_CMD_IQRO); if (fl) { struct sge_eth_rxq *rxq = container_of(fl, struct sge_eth_rxq, diff --git a/dpdk/drivers/net/dpaa/dpaa_ethdev.c b/dpdk/drivers/net/dpaa/dpaa_ethdev.c index e4375c37..02056bc6 100644 --- a/dpdk/drivers/net/dpaa/dpaa_ethdev.c +++ b/dpdk/drivers/net/dpaa/dpaa_ethdev.c @@ -324,10 +324,13 @@ dpaa_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, static int dpaa_xstats_get_names(__rte_unused struct rte_eth_dev *dev, struct rte_eth_xstat_name *xstats_names, - __rte_unused unsigned int limit) + unsigned int limit) { unsigned int i, stat_cnt = RTE_DIM(dpaa_xstats_strings); + if (limit < stat_cnt) + return stat_cnt; + if (xstats_names != NULL) for (i = 0; i < stat_cnt; i++) snprintf(xstats_names[i].name, @@ -355,7 +358,7 @@ dpaa_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids, return 0; fman_if_stats_get_all(dpaa_intf->fif, values_copy, - sizeof(struct dpaa_if_stats)); + sizeof(struct dpaa_if_stats) / 8); for (i = 0; i < stat_cnt; i++) values[i] = diff --git a/dpdk/drivers/net/dpaa2/dpaa2_ethdev.c b/dpdk/drivers/net/dpaa2/dpaa2_ethdev.c index 202f84f0..0711baf1 100644 --- a/dpdk/drivers/net/dpaa2/dpaa2_ethdev.c +++ b/dpdk/drivers/net/dpaa2/dpaa2_ethdev.c @@ -1144,12 +1144,12 @@ dpaa2_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats, union dpni_statistics value[3] = {}; unsigned int i = 0, num = RTE_DIM(dpaa2_xstats_strings); - if (xstats == NULL) - return 0; - if (n < num) return num; + if (xstats == NULL) + return 0; + /* Get Counters from page_0*/ retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token, 0, 0, &value[0]); @@ -1182,10 +1182,13 @@ err: static int dpaa2_xstats_get_names(__rte_unused struct rte_eth_dev *dev, struct rte_eth_xstat_name *xstats_names, - __rte_unused unsigned int limit) + unsigned int limit) { unsigned int i, stat_cnt = RTE_DIM(dpaa2_xstats_strings); + if (limit < stat_cnt) + return stat_cnt; + if (xstats_names != NULL) for (i = 0; i < stat_cnt; i++) snprintf(xstats_names[i].name, diff --git a/dpdk/drivers/net/dpaa2/mc/dpni.c b/dpdk/drivers/net/dpaa2/mc/dpni.c index 6f671fe0..fe2cb52e 100644 --- a/dpdk/drivers/net/dpaa2/mc/dpni.c +++ b/dpdk/drivers/net/dpaa2/mc/dpni.c @@ -231,7 +231,7 @@ int dpni_set_pools(struct fsl_mc_io *mc_io, token); cmd_params = (struct dpni_cmd_set_pools *)cmd.params; cmd_params->num_dpbp = cfg->num_dpbp; - for (i = 0; i < DPNI_MAX_DPBP; i++) { + for (i = 0; i < cmd_params->num_dpbp; i++) { cmd_params->pool[i].dpbp_id = cpu_to_le16(cfg->pools[i].dpbp_id); cmd_params->pool[i].priority_mask = diff --git a/dpdk/drivers/net/ena/base/ena_plat_dpdk.h b/dpdk/drivers/net/ena/base/ena_plat_dpdk.h index accecf51..ea78e8d7 100644 --- a/dpdk/drivers/net/ena/base/ena_plat_dpdk.h +++ b/dpdk/drivers/net/ena/base/ena_plat_dpdk.h @@ -116,11 +116,13 @@ typedef uint64_t dma_addr_t; #define ENA_MIN16(x, y) RTE_MIN((x), (y)) #define ENA_MIN8(x, y) RTE_MIN((x), (y)) +#define BITS_PER_LONG_LONG (__SIZEOF_LONG_LONG__ * 8) #define U64_C(x) x ## ULL #define BIT(nr) (1UL << (nr)) #define BITS_PER_LONG (__SIZEOF_LONG__ * 8) #define GENMASK(h, l) (((~0UL) << (l)) & (~0UL >> (BITS_PER_LONG - 1 - (h)))) -#define GENMASK_ULL(h, l) (((U64_C(1) << ((h) - (l) + 1)) - 1) << (l)) +#define GENMASK_ULL(h, l) (((~0ULL) - (1ULL << (l)) + 1) & \ + (~0ULL >> (BITS_PER_LONG_LONG - 1 - (h)))) #ifdef RTE_LIBRTE_ENA_COM_DEBUG #define ena_trc_dbg(format, arg...) \ @@ -189,10 +191,15 @@ typedef uint64_t dma_addr_t; snprintf(z_name, sizeof(z_name), \ "ena_alloc_%d", ena_alloc_cnt++); \ mz = rte_memzone_reserve(z_name, size, SOCKET_ID_ANY, 0); \ - memset(mz->addr, 0, size); \ - virt = mz->addr; \ - phys = mz->iova; \ handle = mz; \ + if (mz == NULL) { \ + virt = NULL; \ + phys = 0; \ + } else { \ + memset(mz->addr, 0, size); \ + virt = mz->addr; \ + phys = mz->iova; \ + } \ } while (0) #define ENA_MEM_FREE_COHERENT(dmadev, size, virt, phys, handle) \ ({ ENA_TOUCH(size); ENA_TOUCH(phys); \ @@ -207,21 +214,20 @@ typedef uint64_t dma_addr_t; snprintf(z_name, sizeof(z_name), \ "ena_alloc_%d", ena_alloc_cnt++); \ mz = rte_memzone_reserve(z_name, size, node, 0); \ - memset(mz->addr, 0, size); \ - virt = mz->addr; \ - phys = mz->iova; \ + if (mz == NULL) { \ + virt = NULL; \ + phys = 0; \ + } else { \ + memset(mz->addr, 0, size); \ + virt = mz->addr; \ + phys = mz->iova; \ + } \ } while (0) #define ENA_MEM_ALLOC_NODE(dmadev, size, virt, node, dev_node) \ do { \ - const struct rte_memzone *mz; \ - char z_name[RTE_MEMZONE_NAMESIZE]; \ ENA_TOUCH(dmadev); ENA_TOUCH(dev_node); \ - snprintf(z_name, sizeof(z_name), \ - "ena_alloc_%d", ena_alloc_cnt++); \ - mz = rte_memzone_reserve(z_name, size, node, 0); \ - memset(mz->addr, 0, size); \ - virt = mz->addr; \ + virt = rte_zmalloc_socket(NULL, size, 0, node); \ } while (0) #define ENA_MEM_ALLOC(dmadev, size) rte_zmalloc(NULL, size, 1) diff --git a/dpdk/drivers/net/ena/ena_ethdev.c b/dpdk/drivers/net/ena/ena_ethdev.c index 89dff543..4e526567 100644 --- a/dpdk/drivers/net/ena/ena_ethdev.c +++ b/dpdk/drivers/net/ena/ena_ethdev.c @@ -709,7 +709,7 @@ static int ena_link_update(struct rte_eth_dev *dev, struct rte_eth_link *link = &dev->data->dev_link; link->link_status = 1; - link->link_speed = ETH_SPEED_NUM_10G; + link->link_speed = ETH_SPEED_NUM_NONE; link->link_duplex = ETH_LINK_FULL_DUPLEX; return 0; @@ -907,7 +907,7 @@ static int ena_start(struct rte_eth_dev *dev) return rc; if (adapter->rte_dev->data->dev_conf.rxmode.mq_mode & - ETH_MQ_RX_RSS_FLAG) { + ETH_MQ_RX_RSS_FLAG && adapter->rte_dev->data->nb_rx_queues > 0) { rc = ena_rss_init_default(adapter); if (rc) return rc; @@ -1278,8 +1278,7 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) static int adapters_found; - // Temporary modification for multi process - //memset(adapter, 0, sizeof(struct ena_adapter)); + memset(adapter, 0, sizeof(struct ena_adapter)); ena_dev = &adapter->ena_dev; eth_dev->dev_ops = &ena_dev_ops; diff --git a/dpdk/drivers/net/enic/base/vnic_dev.c b/dpdk/drivers/net/enic/base/vnic_dev.c index 9b25d219..9e54ace3 100644 --- a/dpdk/drivers/net/enic/base/vnic_dev.c +++ b/dpdk/drivers/net/enic/base/vnic_dev.c @@ -627,17 +627,9 @@ int vnic_dev_stats_dump(struct vnic_dev *vdev, struct vnic_stats **stats) { u64 a0, a1; int wait = 1000; - static u32 instance; - char name[NAME_MAX]; - if (!vdev->stats) { - snprintf((char *)name, sizeof(name), - "vnic_stats-%u", instance++); - vdev->stats = vdev->alloc_consistent(vdev->priv, - sizeof(struct vnic_stats), &vdev->stats_pa, (u8 *)name); - if (!vdev->stats) - return -ENOMEM; - } + if (!vdev->stats) + return -ENOMEM; *stats = vdev->stats; a0 = vdev->stats_pa; @@ -962,6 +954,18 @@ u32 vnic_dev_get_intr_coal_timer_max(struct vnic_dev *vdev) return vdev->intr_coal_timer_info.max_usec; } +int vnic_dev_alloc_stats_mem(struct vnic_dev *vdev) +{ + char name[NAME_MAX]; + static u32 instance; + + snprintf((char *)name, sizeof(name), "vnic_stats-%u", instance++); + vdev->stats = vdev->alloc_consistent(vdev->priv, + sizeof(struct vnic_stats), + &vdev->stats_pa, (u8 *)name); + return vdev->stats == NULL ? -ENOMEM : 0; +} + void vnic_dev_unregister(struct vnic_dev *vdev) { if (vdev) { diff --git a/dpdk/drivers/net/enic/base/vnic_dev.h b/dpdk/drivers/net/enic/base/vnic_dev.h index c9ca25b3..94964e44 100644 --- a/dpdk/drivers/net/enic/base/vnic_dev.h +++ b/dpdk/drivers/net/enic/base/vnic_dev.h @@ -196,6 +196,7 @@ struct vnic_dev *vnic_dev_register(struct vnic_dev *vdev, void *priv, struct rte_pci_device *pdev, struct vnic_dev_bar *bar, unsigned int num_bars); struct rte_pci_device *vnic_dev_get_pdev(struct vnic_dev *vdev); +int vnic_dev_alloc_stats_mem(struct vnic_dev *vdev); int vnic_dev_cmd_init(struct vnic_dev *vdev, int fallback); int vnic_dev_get_size(void); int vnic_dev_int13(struct vnic_dev *vdev, u64 arg, u32 op); diff --git a/dpdk/drivers/net/enic/enic.h b/dpdk/drivers/net/enic/enic.h index a43fddc5..8bbff18c 100644 --- a/dpdk/drivers/net/enic/enic.h +++ b/dpdk/drivers/net/enic/enic.h @@ -139,6 +139,7 @@ struct enic { u8 adv_filters; u32 flow_filter_mode; u8 filter_tags; + uint8_t ig_vlan_rewrite_mode; /* devargs ig-vlan-rewrite */ unsigned int flags; unsigned int priv_flags; diff --git a/dpdk/drivers/net/enic/enic_ethdev.c b/dpdk/drivers/net/enic/enic_ethdev.c index 98391b00..46ea2cf4 100644 --- a/dpdk/drivers/net/enic/enic_ethdev.c +++ b/dpdk/drivers/net/enic/enic_ethdev.c @@ -40,6 +40,7 @@ #include #include #include +#include #include #include "vnic_intr.h" @@ -66,6 +67,8 @@ static const struct rte_pci_id pci_id_enic_map[] = { {.vendor_id = 0, /* sentinel */}, }; +#define ENIC_DEVARG_IG_VLAN_REWRITE "ig-vlan-rewrite" + static int enicpmd_fdir_ctrl_func(struct rte_eth_dev *eth_dev, enum rte_filter_op filter_op, void *arg) @@ -644,6 +647,64 @@ static const struct eth_dev_ops enicpmd_eth_dev_ops = { .filter_ctrl = enicpmd_dev_filter_ctrl, }; +static int enic_parse_ig_vlan_rewrite(__rte_unused const char *key, + const char *value, + void *opaque) +{ + struct enic *enic; + + enic = (struct enic *)opaque; + if (strcmp(value, "trunk") == 0) { + /* Trunk mode: always tag */ + enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_DEFAULT_TRUNK; + } else if (strcmp(value, "untag") == 0) { + /* Untag default VLAN mode: untag if VLAN = default VLAN */ + enic->ig_vlan_rewrite_mode = + IG_VLAN_REWRITE_MODE_UNTAG_DEFAULT_VLAN; + } else if (strcmp(value, "priority") == 0) { + /* + * Priority-tag default VLAN mode: priority tag (VLAN header + * with ID=0) if VLAN = default + */ + enic->ig_vlan_rewrite_mode = + IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN; + } else if (strcmp(value, "pass") == 0) { + /* Pass through mode: do not touch tags */ + enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PASS_THRU; + } else { + dev_err(enic, "Invalid value for " ENIC_DEVARG_IG_VLAN_REWRITE + ": expected=trunk|untag|priority|pass given=%s\n", + value); + return -EINVAL; + } + return 0; +} + +static int enic_check_devargs(struct rte_eth_dev *dev) +{ + static const char *const valid_keys[] = { + ENIC_DEVARG_IG_VLAN_REWRITE, + NULL}; + struct enic *enic = pmd_priv(dev); + struct rte_kvargs *kvlist; + + ENICPMD_FUNC_TRACE(); + + enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PASS_THRU; + if (!dev->device->devargs) + return 0; + kvlist = rte_kvargs_parse(dev->device->devargs->args, valid_keys); + if (!kvlist) + return -EINVAL; + if (rte_kvargs_process(kvlist, ENIC_DEVARG_IG_VLAN_REWRITE, + enic_parse_ig_vlan_rewrite, enic) < 0) { + rte_kvargs_free(kvlist); + return -EINVAL; + } + rte_kvargs_free(kvlist); + return 0; +} + struct enic *enicpmd_list_head = NULL; /* Initialize the driver * It returns 0 on success. @@ -653,6 +714,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) struct rte_pci_device *pdev; struct rte_pci_addr *addr; struct enic *enic = pmd_priv(eth_dev); + int err; ENICPMD_FUNC_TRACE(); @@ -670,6 +732,9 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) snprintf(enic->bdf_name, ENICPMD_BDF_LENGTH, "%04x:%02x:%02x.%x", addr->domain, addr->bus, addr->devid, addr->function); + err = enic_check_devargs(eth_dev); + if (err) + return err; return enic_probe(enic); } @@ -695,3 +760,5 @@ static struct rte_pci_driver rte_enic_pmd = { RTE_PMD_REGISTER_PCI(net_enic, rte_enic_pmd); RTE_PMD_REGISTER_PCI_TABLE(net_enic, pci_id_enic_map); RTE_PMD_REGISTER_KMOD_DEP(net_enic, "* igb_uio | uio_pci_generic | vfio-pci"); +RTE_PMD_REGISTER_PARAM_STRING(net_enic, + ENIC_DEVARG_IG_VLAN_REWRITE "=trunk|untag|priority|pass"); diff --git a/dpdk/drivers/net/enic/enic_main.c b/dpdk/drivers/net/enic/enic_main.c index 1694aed1..f946abf2 100644 --- a/dpdk/drivers/net/enic/enic_main.c +++ b/dpdk/drivers/net/enic/enic_main.c @@ -782,25 +782,23 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, static int instance; wq->socket_id = socket_id; - if (nb_desc) { - if (nb_desc > enic->config.wq_desc_count) { - dev_warning(enic, - "WQ %d - number of tx desc in cmd line (%d)"\ - "is greater than that in the UCSM/CIMC adapter"\ - "policy. Applying the value in the adapter "\ - "policy (%d)\n", - queue_idx, nb_desc, enic->config.wq_desc_count); - } else if (nb_desc != enic->config.wq_desc_count) { - enic->config.wq_desc_count = nb_desc; - dev_info(enic, - "TX Queues - effective number of descs:%d\n", - nb_desc); - } + if (nb_desc > enic->config.wq_desc_count) { + dev_warning(enic, + "WQ %d - number of tx desc in cmd line (%d) " + "is greater than that in the UCSM/CIMC adapter " + "policy. Applying the value in the adapter " + "policy (%d)\n", + queue_idx, nb_desc, enic->config.wq_desc_count); + nb_desc = enic->config.wq_desc_count; + } else if (nb_desc != enic->config.wq_desc_count) { + dev_info(enic, + "TX Queues - effective number of descs:%d\n", + nb_desc); } /* Allocate queue resources */ err = vnic_wq_alloc(enic->vdev, &enic->wq[queue_idx], queue_idx, - enic->config.wq_desc_count, + nb_desc, sizeof(struct wq_enet_desc)); if (err) { dev_err(enic, "error in allocation of wq\n"); @@ -808,7 +806,7 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx, } err = vnic_cq_alloc(enic->vdev, &enic->cq[cq_index], cq_index, - socket_id, enic->config.wq_desc_count, + socket_id, nb_desc, sizeof(struct cq_enet_wq_desc)); if (err) { vnic_wq_free(wq); @@ -1252,6 +1250,8 @@ int enic_set_mtu(struct enic *enic, uint16_t new_mtu) /* free and reallocate RQs with the new MTU */ for (rq_idx = 0; rq_idx < enic->rq_count; rq_idx++) { rq = &enic->rq[enic_rte_rq_idx_to_sop_idx(rq_idx)]; + if (!rq->in_use) + continue; enic_free_rq(rq); rc = enic_alloc_rq(enic, rq_idx, rq->socket_id, rq->mp, @@ -1383,6 +1383,15 @@ int enic_probe(struct enic *enic) enic_alloc_consistent, enic_free_consistent); + /* + * Allocate the consistent memory for stats upfront so both primary and + * secondary processes can dump stats. + */ + err = vnic_dev_alloc_stats_mem(enic->vdev); + if (err) { + dev_err(enic, "Failed to allocate cmd memory, aborting\n"); + goto err_out_unregister; + } /* Issue device open to get device in known state */ err = enic_dev_open(enic); if (err) { @@ -1391,8 +1400,10 @@ int enic_probe(struct enic *enic) } /* Set ingress vlan rewrite mode before vnic initialization */ + dev_debug(enic, "Set ig_vlan_rewrite_mode=%u\n", + enic->ig_vlan_rewrite_mode); err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, - IG_VLAN_REWRITE_MODE_PASS_THRU); + enic->ig_vlan_rewrite_mode); if (err) { dev_err(enic, "Failed to set ingress vlan rewrite mode, aborting.\n"); diff --git a/dpdk/drivers/net/failsafe/failsafe.c b/dpdk/drivers/net/failsafe/failsafe.c index 8336510f..41b4cb0b 100644 --- a/dpdk/drivers/net/failsafe/failsafe.c +++ b/dpdk/drivers/net/failsafe/failsafe.c @@ -210,7 +210,7 @@ fs_eth_dev_create(struct rte_vdev_device *vdev) mac); if (ret) { ERROR("Failed to set default MAC address"); - goto free_args; + goto cancel_alarm; } } } else { @@ -240,6 +240,8 @@ fs_eth_dev_create(struct rte_vdev_device *vdev) mac->addr_bytes[4], mac->addr_bytes[5]); dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC; return 0; +cancel_alarm: + failsafe_hotplug_alarm_cancel(dev); free_args: failsafe_args_free(dev); free_subs: diff --git a/dpdk/drivers/net/failsafe/failsafe_ether.c b/dpdk/drivers/net/failsafe/failsafe_ether.c index 21392e5a..5b5ac42b 100644 --- a/dpdk/drivers/net/failsafe/failsafe_ether.c +++ b/dpdk/drivers/net/failsafe/failsafe_ether.c @@ -287,6 +287,7 @@ fs_dev_remove(struct sub_device *sdev) sdev->state = DEV_ACTIVE; /* fallthrough */ case DEV_ACTIVE: + failsafe_eth_dev_unregister_callbacks(sdev); rte_eth_dev_close(PORT_ID(sdev)); sdev->state = DEV_PROBED; /* fallthrough */ @@ -346,6 +347,35 @@ fs_rxtx_clean(struct sub_device *sdev) return 1; } +void +failsafe_eth_dev_unregister_callbacks(struct sub_device *sdev) +{ + int ret; + + if (sdev == NULL) + return; + if (sdev->rmv_callback) { + ret = rte_eth_dev_callback_unregister(PORT_ID(sdev), + RTE_ETH_EVENT_INTR_RMV, + failsafe_eth_rmv_event_callback, + sdev); + if (ret) + WARN("Failed to unregister RMV callback for sub_device" + " %d", SUB_ID(sdev)); + sdev->rmv_callback = 0; + } + if (sdev->lsc_callback) { + ret = rte_eth_dev_callback_unregister(PORT_ID(sdev), + RTE_ETH_EVENT_INTR_LSC, + failsafe_eth_lsc_event_callback, + sdev); + if (ret) + WARN("Failed to unregister LSC callback for sub_device" + " %d", SUB_ID(sdev)); + sdev->lsc_callback = 0; + } +} + void failsafe_dev_remove(struct rte_eth_dev *dev) { diff --git a/dpdk/drivers/net/failsafe/failsafe_ops.c b/dpdk/drivers/net/failsafe/failsafe_ops.c index e16a5903..9a5d8733 100644 --- a/dpdk/drivers/net/failsafe/failsafe_ops.c +++ b/dpdk/drivers/net/failsafe/failsafe_ops.c @@ -124,7 +124,7 @@ fs_dev_configure(struct rte_eth_dev *dev) ERROR("Could not configure sub_device %d", i); return ret; } - if (rmv_interrupt) { + if (rmv_interrupt && sdev->rmv_callback == 0) { ret = rte_eth_dev_callback_register(PORT_ID(sdev), RTE_ETH_EVENT_INTR_RMV, failsafe_eth_rmv_event_callback, @@ -132,9 +132,11 @@ fs_dev_configure(struct rte_eth_dev *dev) if (ret) WARN("Failed to register RMV callback for sub_device %d", SUB_ID(sdev)); + else + sdev->rmv_callback = 1; } dev->data->dev_conf.intr_conf.rmv = 0; - if (lsc_interrupt) { + if (lsc_interrupt && sdev->lsc_callback == 0) { ret = rte_eth_dev_callback_register(PORT_ID(sdev), RTE_ETH_EVENT_INTR_LSC, failsafe_eth_lsc_event_callback, @@ -142,6 +144,8 @@ fs_dev_configure(struct rte_eth_dev *dev) if (ret) WARN("Failed to register LSC callback for sub_device %d", SUB_ID(sdev)); + else + sdev->lsc_callback = 1; } dev->data->dev_conf.intr_conf.lsc = lsc_enabled; sdev->state = DEV_ACTIVE; @@ -237,6 +241,7 @@ fs_dev_close(struct rte_eth_dev *dev) PRIV(dev)->state = DEV_ACTIVE - 1; FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { DEBUG("Closing sub_device %d", i); + failsafe_eth_dev_unregister_callbacks(sdev); rte_eth_dev_close(PORT_ID(sdev)); sdev->state = DEV_ACTIVE - 1; } diff --git a/dpdk/drivers/net/failsafe/failsafe_private.h b/dpdk/drivers/net/failsafe/failsafe_private.h index d81cc3ca..40eabb78 100644 --- a/dpdk/drivers/net/failsafe/failsafe_private.h +++ b/dpdk/drivers/net/failsafe/failsafe_private.h @@ -117,6 +117,10 @@ struct sub_device { volatile unsigned int remove:1; /* flow isolation state */ int flow_isolated:1; + /* RMV callback registration state */ + unsigned int rmv_callback:1; + /* LSC callback registration state */ + unsigned int lsc_callback:1; }; struct fs_priv { @@ -187,6 +191,7 @@ int failsafe_eal_uninit(struct rte_eth_dev *dev); /* ETH_DEV */ int failsafe_eth_dev_state_sync(struct rte_eth_dev *dev); +void failsafe_eth_dev_unregister_callbacks(struct sub_device *sdev); void failsafe_dev_remove(struct rte_eth_dev *dev); void failsafe_stats_increment(struct rte_eth_stats *to, struct rte_eth_stats *from); diff --git a/dpdk/drivers/net/i40e/base/i40e_register.h b/dpdk/drivers/net/i40e/base/i40e_register.h index a482ab90..df66e76a 100644 --- a/dpdk/drivers/net/i40e/base/i40e_register.h +++ b/dpdk/drivers/net/i40e/base/i40e_register.h @@ -90,7 +90,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_PF_ARQLEN_ARQCRIT_SHIFT 30 #define I40E_PF_ARQLEN_ARQCRIT_MASK I40E_MASK(0x1, I40E_PF_ARQLEN_ARQCRIT_SHIFT) #define I40E_PF_ARQLEN_ARQENABLE_SHIFT 31 -#define I40E_PF_ARQLEN_ARQENABLE_MASK I40E_MASK(0x1, I40E_PF_ARQLEN_ARQENABLE_SHIFT) +#define I40E_PF_ARQLEN_ARQENABLE_MASK I40E_MASK(0x1u, I40E_PF_ARQLEN_ARQENABLE_SHIFT) #define I40E_PF_ARQT 0x00080480 /* Reset: EMPR */ #define I40E_PF_ARQT_ARQT_SHIFT 0 #define I40E_PF_ARQT_ARQT_MASK I40E_MASK(0x3FF, I40E_PF_ARQT_ARQT_SHIFT) @@ -113,7 +113,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_PF_ATQLEN_ATQCRIT_SHIFT 30 #define I40E_PF_ATQLEN_ATQCRIT_MASK I40E_MASK(0x1, I40E_PF_ATQLEN_ATQCRIT_SHIFT) #define I40E_PF_ATQLEN_ATQENABLE_SHIFT 31 -#define I40E_PF_ATQLEN_ATQENABLE_MASK I40E_MASK(0x1, I40E_PF_ATQLEN_ATQENABLE_SHIFT) +#define I40E_PF_ATQLEN_ATQENABLE_MASK I40E_MASK(0x1u, I40E_PF_ATQLEN_ATQENABLE_SHIFT) #define I40E_PF_ATQT 0x00080400 /* Reset: EMPR */ #define I40E_PF_ATQT_ATQT_SHIFT 0 #define I40E_PF_ATQT_ATQT_MASK I40E_MASK(0x3FF, I40E_PF_ATQT_ATQT_SHIFT) @@ -140,7 +140,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_VF_ARQLEN_ARQCRIT_SHIFT 30 #define I40E_VF_ARQLEN_ARQCRIT_MASK I40E_MASK(0x1, I40E_VF_ARQLEN_ARQCRIT_SHIFT) #define I40E_VF_ARQLEN_ARQENABLE_SHIFT 31 -#define I40E_VF_ARQLEN_ARQENABLE_MASK I40E_MASK(0x1, I40E_VF_ARQLEN_ARQENABLE_SHIFT) +#define I40E_VF_ARQLEN_ARQENABLE_MASK I40E_MASK(0x1u, I40E_VF_ARQLEN_ARQENABLE_SHIFT) #define I40E_VF_ARQT(_VF) (0x00082C00 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: EMPR */ #define I40E_VF_ARQT_MAX_INDEX 127 #define I40E_VF_ARQT_ARQT_SHIFT 0 @@ -168,7 +168,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_VF_ATQLEN_ATQCRIT_SHIFT 30 #define I40E_VF_ATQLEN_ATQCRIT_MASK I40E_MASK(0x1, I40E_VF_ATQLEN_ATQCRIT_SHIFT) #define I40E_VF_ATQLEN_ATQENABLE_SHIFT 31 -#define I40E_VF_ATQLEN_ATQENABLE_MASK I40E_MASK(0x1, I40E_VF_ATQLEN_ATQENABLE_SHIFT) +#define I40E_VF_ATQLEN_ATQENABLE_MASK I40E_MASK(0x1u, I40E_VF_ATQLEN_ATQENABLE_SHIFT) #define I40E_VF_ATQT(_VF) (0x00082800 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: EMPR */ #define I40E_VF_ATQT_MAX_INDEX 127 #define I40E_VF_ATQT_ATQT_SHIFT 0 @@ -291,7 +291,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_PRTDCB_RETSTCC_UPINTC_MODE_SHIFT 30 #define I40E_PRTDCB_RETSTCC_UPINTC_MODE_MASK I40E_MASK(0x1, I40E_PRTDCB_RETSTCC_UPINTC_MODE_SHIFT) #define I40E_PRTDCB_RETSTCC_ETSTC_SHIFT 31 -#define I40E_PRTDCB_RETSTCC_ETSTC_MASK I40E_MASK(0x1, I40E_PRTDCB_RETSTCC_ETSTC_SHIFT) +#define I40E_PRTDCB_RETSTCC_ETSTC_MASK I40E_MASK(0x1u, I40E_PRTDCB_RETSTCC_ETSTC_SHIFT) #define I40E_PRTDCB_RPPMC 0x001223A0 /* Reset: CORER */ #define I40E_PRTDCB_RPPMC_LANRPPM_SHIFT 0 #define I40E_PRTDCB_RPPMC_LANRPPM_MASK I40E_MASK(0xFF, I40E_PRTDCB_RPPMC_LANRPPM_SHIFT) @@ -535,7 +535,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_GLGEN_MSCA_MDICMD_SHIFT 30 #define I40E_GLGEN_MSCA_MDICMD_MASK I40E_MASK(0x1, I40E_GLGEN_MSCA_MDICMD_SHIFT) #define I40E_GLGEN_MSCA_MDIINPROGEN_SHIFT 31 -#define I40E_GLGEN_MSCA_MDIINPROGEN_MASK I40E_MASK(0x1, I40E_GLGEN_MSCA_MDIINPROGEN_SHIFT) +#define I40E_GLGEN_MSCA_MDIINPROGEN_MASK I40E_MASK(0x1u, I40E_GLGEN_MSCA_MDIINPROGEN_SHIFT) #define I40E_GLGEN_MSRWD(_i) (0x0008819C + ((_i) * 4)) /* _i=0...3 */ /* Reset: POR */ #define I40E_GLGEN_MSRWD_MAX_INDEX 3 #define I40E_GLGEN_MSRWD_MDIWRDATA_SHIFT 0 @@ -1274,14 +1274,14 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_GLLAN_TXPRE_QDIS_SET_QDIS_SHIFT 30 #define I40E_GLLAN_TXPRE_QDIS_SET_QDIS_MASK I40E_MASK(0x1, I40E_GLLAN_TXPRE_QDIS_SET_QDIS_SHIFT) #define I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_SHIFT 31 -#define I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK I40E_MASK(0x1, I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_SHIFT) +#define I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_MASK I40E_MASK(0x1u, I40E_GLLAN_TXPRE_QDIS_CLEAR_QDIS_SHIFT) #define I40E_PFLAN_QALLOC 0x001C0400 /* Reset: CORER */ #define I40E_PFLAN_QALLOC_FIRSTQ_SHIFT 0 #define I40E_PFLAN_QALLOC_FIRSTQ_MASK I40E_MASK(0x7FF, I40E_PFLAN_QALLOC_FIRSTQ_SHIFT) #define I40E_PFLAN_QALLOC_LASTQ_SHIFT 16 #define I40E_PFLAN_QALLOC_LASTQ_MASK I40E_MASK(0x7FF, I40E_PFLAN_QALLOC_LASTQ_SHIFT) #define I40E_PFLAN_QALLOC_VALID_SHIFT 31 -#define I40E_PFLAN_QALLOC_VALID_MASK I40E_MASK(0x1, I40E_PFLAN_QALLOC_VALID_SHIFT) +#define I40E_PFLAN_QALLOC_VALID_MASK I40E_MASK(0x1u, I40E_PFLAN_QALLOC_VALID_SHIFT) #define I40E_QRX_ENA(_Q) (0x00120000 + ((_Q) * 4)) /* _i=0...1535 */ /* Reset: PFR */ #define I40E_QRX_ENA_MAX_INDEX 1535 #define I40E_QRX_ENA_QENA_REQ_SHIFT 0 @@ -1692,7 +1692,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_GLNVM_SRCTL_START_SHIFT 30 #define I40E_GLNVM_SRCTL_START_MASK I40E_MASK(0x1, I40E_GLNVM_SRCTL_START_SHIFT) #define I40E_GLNVM_SRCTL_DONE_SHIFT 31 -#define I40E_GLNVM_SRCTL_DONE_MASK I40E_MASK(0x1, I40E_GLNVM_SRCTL_DONE_SHIFT) +#define I40E_GLNVM_SRCTL_DONE_MASK I40E_MASK(0x1u, I40E_GLNVM_SRCTL_DONE_SHIFT) #define I40E_GLNVM_SRDATA 0x000B6114 /* Reset: POR */ #define I40E_GLNVM_SRDATA_WRDATA_SHIFT 0 #define I40E_GLNVM_SRDATA_WRDATA_MASK I40E_MASK(0xFFFF, I40E_GLNVM_SRDATA_WRDATA_SHIFT) @@ -3059,7 +3059,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_PF_VT_PFALLOC_LASTVF_SHIFT 8 #define I40E_PF_VT_PFALLOC_LASTVF_MASK I40E_MASK(0xFF, I40E_PF_VT_PFALLOC_LASTVF_SHIFT) #define I40E_PF_VT_PFALLOC_VALID_SHIFT 31 -#define I40E_PF_VT_PFALLOC_VALID_MASK I40E_MASK(0x1, I40E_PF_VT_PFALLOC_VALID_SHIFT) +#define I40E_PF_VT_PFALLOC_VALID_MASK I40E_MASK(0x1u, I40E_PF_VT_PFALLOC_VALID_SHIFT) #define I40E_VP_MDET_RX(_VF) (0x0012A000 + ((_VF) * 4)) /* _i=0...127 */ /* Reset: CORER */ #define I40E_VP_MDET_RX_MAX_INDEX 127 #define I40E_VP_MDET_RX_VALID_SHIFT 0 @@ -3196,7 +3196,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_VF_ARQLEN1_ARQCRIT_SHIFT 30 #define I40E_VF_ARQLEN1_ARQCRIT_MASK I40E_MASK(0x1, I40E_VF_ARQLEN1_ARQCRIT_SHIFT) #define I40E_VF_ARQLEN1_ARQENABLE_SHIFT 31 -#define I40E_VF_ARQLEN1_ARQENABLE_MASK I40E_MASK(0x1, I40E_VF_ARQLEN1_ARQENABLE_SHIFT) +#define I40E_VF_ARQLEN1_ARQENABLE_MASK I40E_MASK(0x1u, I40E_VF_ARQLEN1_ARQENABLE_SHIFT) #define I40E_VF_ARQT1 0x00007000 /* Reset: EMPR */ #define I40E_VF_ARQT1_ARQT_SHIFT 0 #define I40E_VF_ARQT1_ARQT_MASK I40E_MASK(0x3FF, I40E_VF_ARQT1_ARQT_SHIFT) @@ -3219,7 +3219,7 @@ POSSIBILITY OF SUCH DAMAGE. #define I40E_VF_ATQLEN1_ATQCRIT_SHIFT 30 #define I40E_VF_ATQLEN1_ATQCRIT_MASK I40E_MASK(0x1, I40E_VF_ATQLEN1_ATQCRIT_SHIFT) #define I40E_VF_ATQLEN1_ATQENABLE_SHIFT 31 -#define I40E_VF_ATQLEN1_ATQENABLE_MASK I40E_MASK(0x1, I40E_VF_ATQLEN1_ATQENABLE_SHIFT) +#define I40E_VF_ATQLEN1_ATQENABLE_MASK I40E_MASK(0x1u, I40E_VF_ATQLEN1_ATQENABLE_SHIFT) #define I40E_VF_ATQT1 0x00008400 /* Reset: EMPR */ #define I40E_VF_ATQT1_ATQT_SHIFT 0 #define I40E_VF_ATQT1_ATQT_MASK I40E_MASK(0x3FF, I40E_VF_ATQT1_ATQT_SHIFT) diff --git a/dpdk/drivers/net/i40e/i40e_ethdev.c b/dpdk/drivers/net/i40e/i40e_ethdev.c index 290ef249..711c6e7b 100644 --- a/dpdk/drivers/net/i40e/i40e_ethdev.c +++ b/dpdk/drivers/net/i40e/i40e_ethdev.c @@ -1211,6 +1211,13 @@ eth_i40e_dev_init(struct rte_eth_dev *dev) hw->bus.func = pci_dev->addr.function; hw->adapter_stopped = 0; + /* + * Switch Tag value should not be identical to either the First Tag + * or Second Tag values. So set something other than common Ethertype + * for internal switching. + */ + hw->switch_tag = 0xffff; + /* Check if need to support multi-driver */ i40e_support_multi_driver(dev); @@ -1554,6 +1561,7 @@ eth_i40e_dev_uninit(struct rte_eth_dev *dev) struct rte_flow *p_flow; int ret; uint8_t aq_fail = 0; + int retries = 0; PMD_INIT_FUNC_TRACE(); @@ -1595,9 +1603,20 @@ eth_i40e_dev_uninit(struct rte_eth_dev *dev) /* disable uio intr before callback unregister */ rte_intr_disable(intr_handle); - /* register callback func to eal lib */ - rte_intr_callback_unregister(intr_handle, - i40e_dev_interrupt_handler, dev); + /* unregister callback func to eal lib */ + do { + ret = rte_intr_callback_unregister(intr_handle, + i40e_dev_interrupt_handler, dev); + if (ret >= 0) { + break; + } else if (ret != -EAGAIN) { + PMD_INIT_LOG(ERR, + "intr callback unregister failed: %d", + ret); + return ret; + } + i40e_msec_delay(500); + } while (retries++ < 5); i40e_rm_ethtype_filter_list(pf); i40e_rm_tunnel_filter_list(pf); @@ -1964,27 +1983,40 @@ i40e_phy_conf_link(struct i40e_hw *hw, struct i40e_aq_get_phy_abilities_resp phy_ab; struct i40e_aq_set_phy_config phy_conf; enum i40e_aq_phy_type cnt; + uint8_t avail_speed; uint32_t phy_type_mask = 0; const uint8_t mask = I40E_AQ_PHY_FLAG_PAUSE_TX | I40E_AQ_PHY_FLAG_PAUSE_RX | I40E_AQ_PHY_FLAG_PAUSE_RX | I40E_AQ_PHY_FLAG_LOW_POWER; - const uint8_t advt = I40E_LINK_SPEED_40GB | - I40E_LINK_SPEED_25GB | - I40E_LINK_SPEED_10GB | - I40E_LINK_SPEED_1GB | - I40E_LINK_SPEED_100MB; int ret = -ENOTSUP; + /* To get phy capabilities of available speeds. */ + status = i40e_aq_get_phy_capabilities(hw, false, true, &phy_ab, + NULL); + if (status) { + PMD_DRV_LOG(ERR, "Failed to get PHY capabilities: %d\n", + status); + return ret; + } + avail_speed = phy_ab.link_speed; + /* To get the current phy config. */ status = i40e_aq_get_phy_capabilities(hw, false, false, &phy_ab, NULL); - if (status) + if (status) { + PMD_DRV_LOG(ERR, "Failed to get the current PHY config: %d\n", + status); return ret; + } - /* If link already up, no need to set up again */ - if (is_up && phy_ab.phy_type != 0) + /* If link needs to go up and it is in autoneg mode the speed is OK, + * no need to set up again. + */ + if (is_up && phy_ab.phy_type != 0 && + abilities & I40E_AQ_PHY_AN_ENABLED && + phy_ab.link_speed != 0) return I40E_SUCCESS; memset(&phy_conf, 0, sizeof(phy_conf)); @@ -1993,18 +2025,20 @@ i40e_phy_conf_link(struct i40e_hw *hw, abilities &= ~mask; abilities |= phy_ab.abilities & mask; - /* update ablities and speed */ - if (abilities & I40E_AQ_PHY_AN_ENABLED) - phy_conf.link_speed = advt; - else - phy_conf.link_speed = is_up ? force_speed : phy_ab.link_speed; - phy_conf.abilities = abilities; + /* If link needs to go up, but the force speed is not supported, + * Warn users and config the default available speeds. + */ + if (is_up && !(force_speed & avail_speed)) { + PMD_DRV_LOG(WARNING, "Invalid speed setting, set to default!\n"); + phy_conf.link_speed = avail_speed; + } else { + phy_conf.link_speed = is_up ? force_speed : avail_speed; + } - - /* To enable link, phy_type mask needs to include each type */ - for (cnt = I40E_PHY_TYPE_SGMII; cnt < I40E_PHY_TYPE_MAX; cnt++) + /* PHY type mask needs to include each type except PHY type extension */ + for (cnt = I40E_PHY_TYPE_SGMII; cnt < I40E_PHY_TYPE_25GBASE_KR; cnt++) phy_type_mask |= 1 << cnt; /* use get_phy_abilities_resp value for the rest */ @@ -2037,11 +2071,18 @@ i40e_apply_link_speed(struct rte_eth_dev *dev) struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); struct rte_eth_conf *conf = &dev->data->dev_conf; + if (conf->link_speeds == ETH_LINK_SPEED_AUTONEG) { + conf->link_speeds = ETH_LINK_SPEED_40G | + ETH_LINK_SPEED_25G | + ETH_LINK_SPEED_20G | + ETH_LINK_SPEED_10G | + ETH_LINK_SPEED_1G | + ETH_LINK_SPEED_100M; + } speed = i40e_parse_link_speeds(conf->link_speeds); - abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK; - if (!(conf->link_speeds & ETH_LINK_SPEED_FIXED)) - abilities |= I40E_AQ_PHY_AN_ENABLED; - abilities |= I40E_AQ_PHY_LINK_ENABLED; + abilities |= I40E_AQ_PHY_ENABLE_ATOMIC_LINK | + I40E_AQ_PHY_AN_ENABLED | + I40E_AQ_PHY_LINK_ENABLED; return i40e_phy_conf_link(hw, abilities, speed, true); } @@ -2148,13 +2189,6 @@ i40e_dev_start(struct rte_eth_dev *dev) } /* Apply link configure */ - if (dev->data->dev_conf.link_speeds & ~(ETH_LINK_SPEED_100M | - ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G | - ETH_LINK_SPEED_20G | ETH_LINK_SPEED_25G | - ETH_LINK_SPEED_40G)) { - PMD_DRV_LOG(ERR, "Invalid link setting"); - goto err_up; - } ret = i40e_apply_link_speed(dev); if (I40E_SUCCESS != ret) { PMD_DRV_LOG(ERR, "Fail to apply link setting"); @@ -2297,6 +2331,8 @@ i40e_dev_close(struct rte_eth_dev *dev) i40e_pf_disable_irq0(hw); rte_intr_disable(intr_handle); + i40e_fdir_teardown(pf); + /* shutdown and destroy the HMC */ i40e_shutdown_lan_hmc(hw); @@ -2308,7 +2344,6 @@ i40e_dev_close(struct rte_eth_dev *dev) pf->vmdq = NULL; /* release all the existing VSIs and VEBs */ - i40e_fdir_teardown(pf); i40e_vsi_release(pf->main_vsi); /* shutdown the adminq */ @@ -2444,77 +2479,139 @@ i40e_dev_set_link_down(struct rte_eth_dev *dev) return i40e_phy_conf_link(hw, abilities, speed, false); } -int -i40e_dev_link_update(struct rte_eth_dev *dev, - int wait_to_complete) +static __rte_always_inline void +update_link_no_wait(struct i40e_hw *hw, struct rte_eth_link *link) { -#define CHECK_INTERVAL 100 /* 100ms */ -#define MAX_REPEAT_TIME 10 /* 1s (10 * 100ms) in total */ - struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); - struct i40e_link_status link_status; - struct rte_eth_link link, old; - int status; - unsigned rep_cnt = MAX_REPEAT_TIME; - bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false; +/* Link status registers and values*/ +#define I40E_PRTMAC_LINKSTA 0x001E2420 +#define I40E_REG_LINK_UP 0x40000080 +#define I40E_PRTMAC_MACC 0x001E24E0 +#define I40E_REG_MACC_25GB 0x00020000 +#define I40E_REG_SPEED_MASK 0x38000000 +#define I40E_REG_SPEED_100MB 0x00000000 +#define I40E_REG_SPEED_1GB 0x08000000 +#define I40E_REG_SPEED_10GB 0x10000000 +#define I40E_REG_SPEED_20GB 0x20000000 +#define I40E_REG_SPEED_25_40GB 0x18000000 + uint32_t link_speed; + uint32_t reg_val; + + reg_val = I40E_READ_REG(hw, I40E_PRTMAC_LINKSTA); + link_speed = reg_val & I40E_REG_SPEED_MASK; + reg_val &= I40E_REG_LINK_UP; + link->link_status = (reg_val == I40E_REG_LINK_UP) ? 1 : 0; + + if (unlikely(link->link_status == 0)) + return; + + /* Parse the link status */ + switch (link_speed) { + case I40E_REG_SPEED_100MB: + link->link_speed = ETH_SPEED_NUM_100M; + break; + case I40E_REG_SPEED_1GB: + link->link_speed = ETH_SPEED_NUM_1G; + break; + case I40E_REG_SPEED_10GB: + link->link_speed = ETH_SPEED_NUM_10G; + break; + case I40E_REG_SPEED_20GB: + link->link_speed = ETH_SPEED_NUM_20G; + break; + case I40E_REG_SPEED_25_40GB: + reg_val = I40E_READ_REG(hw, I40E_PRTMAC_MACC); + + if (reg_val & I40E_REG_MACC_25GB) + link->link_speed = ETH_SPEED_NUM_25G; + else + link->link_speed = ETH_SPEED_NUM_40G; + + break; + default: + PMD_DRV_LOG(ERR, "Unknown link speed info %u", link_speed); + break; + } +} + +static __rte_always_inline void +update_link_wait(struct i40e_hw *hw, struct rte_eth_link *link, + bool enable_lse) +{ +#define CHECK_INTERVAL 100 /* 100ms */ +#define MAX_REPEAT_TIME 10 /* 1s (10 * 100ms) in total */ + uint32_t rep_cnt = MAX_REPEAT_TIME; + struct i40e_link_status link_status; + int status; - memset(&link, 0, sizeof(link)); - memset(&old, 0, sizeof(old)); memset(&link_status, 0, sizeof(link_status)); - rte_i40e_dev_atomic_read_link_status(dev, &old); do { /* Get link status information from hardware */ status = i40e_aq_get_link_info(hw, enable_lse, &link_status, NULL); - if (status != I40E_SUCCESS) { - link.link_speed = ETH_SPEED_NUM_100M; - link.link_duplex = ETH_LINK_FULL_DUPLEX; + if (unlikely(status != I40E_SUCCESS)) { + link->link_speed = ETH_SPEED_NUM_100M; + link->link_duplex = ETH_LINK_FULL_DUPLEX; PMD_DRV_LOG(ERR, "Failed to get link info"); - goto out; + return; } - link.link_status = link_status.link_info & I40E_AQ_LINK_UP; - if (!wait_to_complete || link.link_status) + link->link_status = link_status.link_info & I40E_AQ_LINK_UP; + if (unlikely(link->link_status != 0)) break; rte_delay_ms(CHECK_INTERVAL); } while (--rep_cnt); - if (!link.link_status) - goto out; - - /* i40e uses full duplex only */ - link.link_duplex = ETH_LINK_FULL_DUPLEX; - /* Parse the link status */ switch (link_status.link_speed) { case I40E_LINK_SPEED_100MB: - link.link_speed = ETH_SPEED_NUM_100M; + link->link_speed = ETH_SPEED_NUM_100M; break; case I40E_LINK_SPEED_1GB: - link.link_speed = ETH_SPEED_NUM_1G; + link->link_speed = ETH_SPEED_NUM_1G; break; case I40E_LINK_SPEED_10GB: - link.link_speed = ETH_SPEED_NUM_10G; + link->link_speed = ETH_SPEED_NUM_10G; break; case I40E_LINK_SPEED_20GB: - link.link_speed = ETH_SPEED_NUM_20G; + link->link_speed = ETH_SPEED_NUM_20G; break; case I40E_LINK_SPEED_25GB: - link.link_speed = ETH_SPEED_NUM_25G; + link->link_speed = ETH_SPEED_NUM_25G; break; case I40E_LINK_SPEED_40GB: - link.link_speed = ETH_SPEED_NUM_40G; + link->link_speed = ETH_SPEED_NUM_40G; break; default: - link.link_speed = ETH_SPEED_NUM_100M; + link->link_speed = ETH_SPEED_NUM_100M; break; } +} +int +i40e_dev_link_update(struct rte_eth_dev *dev, + int wait_to_complete) +{ + struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private); + struct rte_eth_link link, old; + bool enable_lse = dev->data->dev_conf.intr_conf.lsc ? true : false; + + memset(&link, 0, sizeof(link)); + memset(&old, 0, sizeof(old)); + + rte_i40e_dev_atomic_read_link_status(dev, &old); + + /* i40e uses full duplex only */ + link.link_duplex = ETH_LINK_FULL_DUPLEX; link.link_autoneg = !(dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED); -out: + if (!wait_to_complete) + update_link_no_wait(hw, &link); + else + update_link_wait(hw, &link, enable_lse); + rte_i40e_dev_atomic_write_link_status(dev, &link); if (link.link_status == old.link_status) return -1; @@ -9693,6 +9790,60 @@ i40e_pctype_to_flowtype(const struct i40e_adapter *adapter, #define I40E_GL_SWR_PM_UP_THR_SF_VALUE 0x06060606 #define I40E_GL_SWR_PM_UP_THR 0x269FBC +/* + * GL_SWR_PM_UP_THR: + * The value is not impacted from the link speed, its value is set according + * to the total number of ports for a better pipe-monitor configuration. + */ +static bool +i40e_get_swr_pm_cfg(struct i40e_hw *hw, uint32_t *value) +{ +#define I40E_GL_SWR_PM_EF_DEVICE(dev) \ + .device_id = (dev), \ + .val = I40E_GL_SWR_PM_UP_THR_EF_VALUE + +#define I40E_GL_SWR_PM_SF_DEVICE(dev) \ + .device_id = (dev), \ + .val = I40E_GL_SWR_PM_UP_THR_SF_VALUE + + static const struct { + uint16_t device_id; + uint32_t val; + } swr_pm_table[] = { + { I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_SFP_XL710) }, + { I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_KX_C) }, + { I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_10G_BASE_T) }, + { I40E_GL_SWR_PM_EF_DEVICE(I40E_DEV_ID_10G_BASE_T4) }, + + { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_KX_B) }, + { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_QSFP_A) }, + { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_QSFP_B) }, + { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_20G_KR2) }, + { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_20G_KR2_A) }, + { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_25G_B) }, + { I40E_GL_SWR_PM_SF_DEVICE(I40E_DEV_ID_25G_SFP28) }, + }; + uint32_t i; + + if (value == NULL) { + PMD_DRV_LOG(ERR, "value is NULL"); + return false; + } + + for (i = 0; i < RTE_DIM(swr_pm_table); i++) { + if (hw->device_id == swr_pm_table[i].device_id) { + *value = swr_pm_table[i].val; + + PMD_DRV_LOG(DEBUG, "Device 0x%x with GL_SWR_PM_UP_THR " + "value - 0x%08x", + hw->device_id, *value); + return true; + } + } + + return false; +} + static int i40e_dev_sync_phy_type(struct i40e_hw *hw) { @@ -9757,13 +9908,16 @@ i40e_configure_registers(struct i40e_hw *hw) } if (reg_table[i].addr == I40E_GL_SWR_PM_UP_THR) { - if (I40E_PHY_TYPE_SUPPORT_40G(hw->phy.phy_types) || /* For XL710 */ - I40E_PHY_TYPE_SUPPORT_25G(hw->phy.phy_types)) /* For XXV710 */ - reg_table[i].val = - I40E_GL_SWR_PM_UP_THR_SF_VALUE; - else /* For X710 */ - reg_table[i].val = - I40E_GL_SWR_PM_UP_THR_EF_VALUE; + uint32_t cfg_val; + + if (!i40e_get_swr_pm_cfg(hw, &cfg_val)) { + PMD_DRV_LOG(DEBUG, "Device 0x%x skips " + "GL_SWR_PM_UP_THR value fixup", + hw->device_id); + continue; + } + + reg_table[i].val = cfg_val; } ret = i40e_aq_debug_read_register(hw, reg_table[i].addr, @@ -11329,7 +11483,8 @@ i40e_find_customized_pctype(struct i40e_pf *pf, uint8_t index) static int i40e_update_customized_pctype(struct rte_eth_dev *dev, uint8_t *pkg, uint32_t pkg_size, uint32_t proto_num, - struct rte_pmd_i40e_proto_info *proto) + struct rte_pmd_i40e_proto_info *proto, + enum rte_pmd_i40e_package_op op) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); uint32_t pctype_num; @@ -11342,6 +11497,12 @@ i40e_update_customized_pctype(struct rte_eth_dev *dev, uint8_t *pkg, uint32_t i, j, n; int ret; + if (op != RTE_PMD_I40E_PKG_OP_WR_ADD && + op != RTE_PMD_I40E_PKG_OP_WR_DEL) { + PMD_DRV_LOG(ERR, "Unsupported operation."); + return -1; + } + ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, (uint8_t *)&pctype_num, sizeof(pctype_num), RTE_PMD_I40E_PKG_INFO_PCTYPE_NUM); @@ -11404,8 +11565,13 @@ i40e_update_customized_pctype(struct rte_eth_dev *dev, uint8_t *pkg, i40e_find_customized_pctype(pf, I40E_CUSTOMIZED_GTPU); if (new_pctype) { - new_pctype->pctype = pctype_value; - new_pctype->valid = true; + if (op == RTE_PMD_I40E_PKG_OP_WR_ADD) { + new_pctype->pctype = pctype_value; + new_pctype->valid = true; + } else { + new_pctype->pctype = I40E_FILTER_PCTYPE_INVALID; + new_pctype->valid = false; + } } } @@ -11415,8 +11581,9 @@ i40e_update_customized_pctype(struct rte_eth_dev *dev, uint8_t *pkg, static int i40e_update_customized_ptype(struct rte_eth_dev *dev, uint8_t *pkg, - uint32_t pkg_size, uint32_t proto_num, - struct rte_pmd_i40e_proto_info *proto) + uint32_t pkg_size, uint32_t proto_num, + struct rte_pmd_i40e_proto_info *proto, + enum rte_pmd_i40e_package_op op) { struct rte_pmd_i40e_ptype_mapping *ptype_mapping; uint16_t port_id = dev->data->port_id; @@ -11429,6 +11596,17 @@ i40e_update_customized_ptype(struct rte_eth_dev *dev, uint8_t *pkg, bool inner_ip; int ret; + if (op != RTE_PMD_I40E_PKG_OP_WR_ADD && + op != RTE_PMD_I40E_PKG_OP_WR_DEL) { + PMD_DRV_LOG(ERR, "Unsupported operation."); + return -1; + } + + if (op == RTE_PMD_I40E_PKG_OP_WR_DEL) { + rte_pmd_i40e_ptype_mapping_reset(port_id); + return 0; + } + /* get information about new ptype num */ ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, (uint8_t *)&ptype_num, sizeof(ptype_num), @@ -11547,7 +11725,7 @@ i40e_update_customized_ptype(struct rte_eth_dev *dev, uint8_t *pkg, void i40e_update_customized_info(struct rte_eth_dev *dev, uint8_t *pkg, - uint32_t pkg_size) + uint32_t pkg_size, enum rte_pmd_i40e_package_op op) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); uint32_t proto_num; @@ -11556,6 +11734,12 @@ i40e_update_customized_info(struct rte_eth_dev *dev, uint8_t *pkg, uint32_t i; int ret; + if (op != RTE_PMD_I40E_PKG_OP_WR_ADD && + op != RTE_PMD_I40E_PKG_OP_WR_DEL) { + PMD_DRV_LOG(ERR, "Unsupported operation."); + return; + } + /* get information about protocol number */ ret = rte_pmd_i40e_get_ddp_info(pkg, pkg_size, (uint8_t *)&proto_num, sizeof(proto_num), @@ -11589,20 +11773,23 @@ i40e_update_customized_info(struct rte_eth_dev *dev, uint8_t *pkg, /* Check if GTP is supported. */ for (i = 0; i < proto_num; i++) { if (!strncmp(proto[i].name, "GTP", 3)) { - pf->gtp_support = true; + if (op == RTE_PMD_I40E_PKG_OP_WR_ADD) + pf->gtp_support = true; + else + pf->gtp_support = false; break; } } /* Update customized pctype info */ ret = i40e_update_customized_pctype(dev, pkg, pkg_size, - proto_num, proto); + proto_num, proto, op); if (ret) PMD_DRV_LOG(INFO, "No pctype is updated."); /* Update customized ptype info */ ret = i40e_update_customized_ptype(dev, pkg, pkg_size, - proto_num, proto); + proto_num, proto, op); if (ret) PMD_DRV_LOG(INFO, "No ptype is updated."); diff --git a/dpdk/drivers/net/i40e/i40e_ethdev.h b/dpdk/drivers/net/i40e/i40e_ethdev.h index 229c974e..2c107e2b 100644 --- a/dpdk/drivers/net/i40e/i40e_ethdev.h +++ b/dpdk/drivers/net/i40e/i40e_ethdev.h @@ -40,6 +40,7 @@ #include #include #include +#include "rte_pmd_i40e.h" #define I40E_VLAN_TAG_SIZE 4 @@ -1221,7 +1222,8 @@ void i40e_tm_conf_uninit(struct rte_eth_dev *dev); struct i40e_customized_pctype* i40e_find_customized_pctype(struct i40e_pf *pf, uint8_t index); void i40e_update_customized_info(struct rte_eth_dev *dev, uint8_t *pkg, - uint32_t pkg_size); + uint32_t pkg_size, + enum rte_pmd_i40e_package_op op); int i40e_dcb_init_configure(struct rte_eth_dev *dev, bool sw_dcb); int i40e_flush_queue_region_all_conf(struct rte_eth_dev *dev, struct i40e_hw *hw, struct i40e_pf *pf, uint16_t on); diff --git a/dpdk/drivers/net/i40e/i40e_flow.c b/dpdk/drivers/net/i40e/i40e_flow.c index 37380e6e..4ebf925a 100644 --- a/dpdk/drivers/net/i40e/i40e_flow.c +++ b/dpdk/drivers/net/i40e/i40e_flow.c @@ -2418,7 +2418,7 @@ i40e_flow_fdir_get_pctype_value(struct i40e_pf *pf, break; } - if (cus_pctype) + if (cus_pctype && cus_pctype->valid) return cus_pctype->pctype; return I40E_FILTER_PCTYPE_INVALID; diff --git a/dpdk/drivers/net/i40e/rte_pmd_i40e.c b/dpdk/drivers/net/i40e/rte_pmd_i40e.c index f726a9cc..2d25873d 100644 --- a/dpdk/drivers/net/i40e/rte_pmd_i40e.c +++ b/dpdk/drivers/net/i40e/rte_pmd_i40e.c @@ -1632,8 +1632,6 @@ rte_pmd_i40e_process_ddp_package(uint16_t port, uint8_t *buff, return -EINVAL; } - i40e_update_customized_info(dev, buff, size); - /* Find metadata segment */ metadata_seg_hdr = i40e_find_segment_in_package(SEGMENT_TYPE_METADATA, pkg_hdr); @@ -1690,6 +1688,7 @@ rte_pmd_i40e_process_ddp_package(uint16_t port, uint8_t *buff, PMD_DRV_LOG(ERR, "Profile of group 0 already exists."); else if (is_exist == 3) PMD_DRV_LOG(ERR, "Profile of different group already exists"); + i40e_update_customized_info(dev, buff, size, op); rte_free(profile_info_sec); return -EEXIST; } @@ -1737,6 +1736,10 @@ rte_pmd_i40e_process_ddp_package(uint16_t port, uint8_t *buff, } } + if (op == RTE_PMD_I40E_PKG_OP_WR_ADD || + op == RTE_PMD_I40E_PKG_OP_WR_DEL) + i40e_update_customized_info(dev, buff, size, op); + rte_free(profile_info_sec); return status; } diff --git a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c index f2198660..d7eb4580 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c +++ b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.c @@ -1366,6 +1366,8 @@ eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct rte_intr_handle *intr_handle = &pci_dev->intr_handle; struct ixgbe_hw *hw; + int retries = 0; + int ret; PMD_INIT_FUNC_TRACE(); @@ -1386,8 +1388,20 @@ eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev) /* disable uio intr before callback unregister */ rte_intr_disable(intr_handle); - rte_intr_callback_unregister(intr_handle, - ixgbe_dev_interrupt_handler, eth_dev); + + do { + ret = rte_intr_callback_unregister(intr_handle, + ixgbe_dev_interrupt_handler, eth_dev); + if (ret >= 0) { + break; + } else if (ret != -EAGAIN) { + PMD_INIT_LOG(ERR, + "intr callback unregister failed: %d", + ret); + return ret; + } + rte_delay_ms(100); + } while (retries++ < (10 + IXGBE_LINK_UP_TIME)); /* uninitialize PF if max_vfs not zero */ ixgbe_pf_host_uninit(eth_dev); @@ -2316,11 +2330,6 @@ ixgbe_check_mq_mode(struct rte_eth_dev *dev) if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) { const struct rte_eth_dcb_rx_conf *conf; - if (nb_rx_q != IXGBE_DCB_NB_QUEUES) { - PMD_INIT_LOG(ERR, "DCB selected, nb_rx_q != %d.", - IXGBE_DCB_NB_QUEUES); - return -EINVAL; - } conf = &dev_conf->rx_adv_conf.dcb_rx_conf; if (!(conf->nb_tcs == ETH_4_TCS || conf->nb_tcs == ETH_8_TCS)) { @@ -2334,11 +2343,6 @@ ixgbe_check_mq_mode(struct rte_eth_dev *dev) if (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB) { const struct rte_eth_dcb_tx_conf *conf; - if (nb_tx_q != IXGBE_DCB_NB_QUEUES) { - PMD_INIT_LOG(ERR, "DCB, nb_tx_q != %d.", - IXGBE_DCB_NB_QUEUES); - return -EINVAL; - } conf = &dev_conf->tx_adv_conf.dcb_tx_conf; if (!(conf->nb_tcs == ETH_4_TCS || conf->nb_tcs == ETH_8_TCS)) { @@ -3886,7 +3890,7 @@ ixgbevf_check_link(struct ixgbe_hw *hw, ixgbe_link_speed *speed, /* for SFP+ modules and DA cables on 82599 it can take up to 500usecs * before the link status is correct */ - if (mac->type == ixgbe_mac_82599_vf) { + if (mac->type == ixgbe_mac_82599_vf && wait_to_complete) { int i; for (i = 0; i < 5; i++) { @@ -5822,8 +5826,12 @@ ixgbe_configure_msix(struct rte_eth_dev *dev) /* won't configure msix register if no mapping is done * between intr vector and event fd + * but if misx has been enabled already, need to configure + * auto clean, auto mask and throttling. */ - if (!rte_intr_dp_is_en(intr_handle)) + gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); + if (!rte_intr_dp_is_en(intr_handle) && + !(gpie & (IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT))) return; if (rte_intr_allow_others(intr_handle)) @@ -5847,27 +5855,30 @@ ixgbe_configure_msix(struct rte_eth_dev *dev) /* Populate the IVAR table and set the ITR values to the * corresponding register. */ - for (queue_id = 0; queue_id < dev->data->nb_rx_queues; - queue_id++) { - /* by default, 1:1 mapping */ - ixgbe_set_ivar_map(hw, 0, queue_id, vec); - intr_handle->intr_vec[queue_id] = vec; - if (vec < base + intr_handle->nb_efd - 1) - vec++; - } + if (rte_intr_dp_is_en(intr_handle)) { + for (queue_id = 0; queue_id < dev->data->nb_rx_queues; + queue_id++) { + /* by default, 1:1 mapping */ + ixgbe_set_ivar_map(hw, 0, queue_id, vec); + intr_handle->intr_vec[queue_id] = vec; + if (vec < base + intr_handle->nb_efd - 1) + vec++; + } - switch (hw->mac.type) { - case ixgbe_mac_82598EB: - ixgbe_set_ivar_map(hw, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX, - IXGBE_MISC_VEC_ID); - break; - case ixgbe_mac_82599EB: - case ixgbe_mac_X540: - case ixgbe_mac_X550: - ixgbe_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID); - break; - default: - break; + switch (hw->mac.type) { + case ixgbe_mac_82598EB: + ixgbe_set_ivar_map(hw, -1, + IXGBE_IVAR_OTHER_CAUSES_INDEX, + IXGBE_MISC_VEC_ID); + break; + case ixgbe_mac_82599EB: + case ixgbe_mac_X540: + case ixgbe_mac_X550: + ixgbe_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID); + break; + default: + break; + } } IXGBE_WRITE_REG(hw, IXGBE_EITR(IXGBE_MISC_VEC_ID), IXGBE_MIN_INTER_INTERRUPT_INTERVAL_DEFAULT & 0xFFF); diff --git a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.h b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.h index 51ddcfd4..762f1ad3 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_ethdev.h +++ b/dpdk/drivers/net/ixgbe/ixgbe_ethdev.h @@ -123,6 +123,11 @@ #define IXGBE_5TUPLE_MAX_PRI 7 #define IXGBE_5TUPLE_MIN_PRI 1 +/* bit of VXLAN tunnel type | 7 bits of zeros | 8 bits of zeros*/ +#define IXGBE_FDIR_VXLAN_TUNNEL_TYPE 0x8000 +/* bit of NVGRE tunnel type | 7 bits of zeros | 8 bits of zeros*/ +#define IXGBE_FDIR_NVGRE_TUNNEL_TYPE 0x0 + #define IXGBE_RSS_OFFLOAD_ALL ( \ ETH_RSS_IPV4 | \ ETH_RSS_NONFRAG_IPV4_TCP | \ diff --git a/dpdk/drivers/net/ixgbe/ixgbe_fdir.c b/dpdk/drivers/net/ixgbe/ixgbe_fdir.c index c1176472..02adfa43 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_fdir.c +++ b/dpdk/drivers/net/ixgbe/ixgbe_fdir.c @@ -423,9 +423,12 @@ fdir_set_input_mask_x550(struct rte_eth_dev *dev) IXGBE_FDIRIP6M_TNI_VNI; if (mode == RTE_FDIR_MODE_PERFECT_TUNNEL) { - mac_mask = info->mask.mac_addr_byte_mask; - fdiripv6m |= (mac_mask << IXGBE_FDIRIP6M_INNER_MAC_SHIFT) - & IXGBE_FDIRIP6M_INNER_MAC; + fdiripv6m |= IXGBE_FDIRIP6M_INNER_MAC; + mac_mask = info->mask.mac_addr_byte_mask & + (IXGBE_FDIRIP6M_INNER_MAC >> + IXGBE_FDIRIP6M_INNER_MAC_SHIFT); + fdiripv6m &= ~((mac_mask << IXGBE_FDIRIP6M_INNER_MAC_SHIFT) & + IXGBE_FDIRIP6M_INNER_MAC); switch (info->mask.tunnel_type_mask) { case 0: @@ -800,10 +803,19 @@ ixgbe_fdir_filter_to_atr_input(const struct rte_eth_fdir_filter *fdir_filter, input->formatted.inner_mac, fdir_filter->input.flow.tunnel_flow.mac_addr.addr_bytes, sizeof(input->formatted.inner_mac)); - input->formatted.tunnel_type = - fdir_filter->input.flow.tunnel_flow.tunnel_type; + if (fdir_filter->input.flow.tunnel_flow.tunnel_type == + RTE_FDIR_TUNNEL_TYPE_VXLAN) + input->formatted.tunnel_type = + IXGBE_FDIR_VXLAN_TUNNEL_TYPE; + else if (fdir_filter->input.flow.tunnel_flow.tunnel_type == + RTE_FDIR_TUNNEL_TYPE_NVGRE) + input->formatted.tunnel_type = + IXGBE_FDIR_NVGRE_TUNNEL_TYPE; + else + PMD_DRV_LOG(ERR, " invalid tunnel type arguments."); + input->formatted.tni_vni = - fdir_filter->input.flow.tunnel_flow.tunnel_id; + fdir_filter->input.flow.tunnel_flow.tunnel_id >> 8; } return 0; @@ -1030,8 +1042,7 @@ fdir_write_perfect_filter_82599(struct ixgbe_hw *hw, IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(2), 0); } else { /* tunnel mode */ - if (input->formatted.tunnel_type != - RTE_FDIR_TUNNEL_TYPE_NVGRE) + if (input->formatted.tunnel_type) tunnel_type = 0x80000000; tunnel_type |= addr_high; IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(0), addr_low); @@ -1039,6 +1050,9 @@ fdir_write_perfect_filter_82599(struct ixgbe_hw *hw, IXGBE_WRITE_REG(hw, IXGBE_FDIRSIPv6(2), input->formatted.tni_vni); } + IXGBE_WRITE_REG(hw, IXGBE_FDIRIPSA, 0); + IXGBE_WRITE_REG(hw, IXGBE_FDIRIPDA, 0); + IXGBE_WRITE_REG(hw, IXGBE_FDIRPORT, 0); } /* record vlan (little-endian) and flex_bytes(big-endian) */ diff --git a/dpdk/drivers/net/ixgbe/ixgbe_flow.c b/dpdk/drivers/net/ixgbe/ixgbe_flow.c index 07abb343..e60ecce7 100644 --- a/dpdk/drivers/net/ixgbe/ixgbe_flow.c +++ b/dpdk/drivers/net/ixgbe/ixgbe_flow.c @@ -1665,7 +1665,8 @@ ixgbe_parse_fdir_filter_normal(struct rte_eth_dev *dev, return -rte_errno; } } else { - if (item->type != RTE_FLOW_ITEM_TYPE_IPV4) { + if (item->type != RTE_FLOW_ITEM_TYPE_IPV4 && + item->type != RTE_FLOW_ITEM_TYPE_VLAN) { memset(rule, 0, sizeof(struct ixgbe_fdir_rule)); rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, @@ -2370,7 +2371,7 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, /* Get the VxLAN info */ if (item->type == RTE_FLOW_ITEM_TYPE_VXLAN) { rule->ixgbe_fdir.formatted.tunnel_type = - RTE_FDIR_TUNNEL_TYPE_VXLAN; + IXGBE_FDIR_VXLAN_TUNNEL_TYPE; /* Only care about VNI, others should be masked. */ if (!item->mask) { @@ -2422,17 +2423,15 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, vxlan_spec = (const struct rte_flow_item_vxlan *) item->spec; rte_memcpy(((uint8_t *) - &rule->ixgbe_fdir.formatted.tni_vni + 1), + &rule->ixgbe_fdir.formatted.tni_vni), vxlan_spec->vni, RTE_DIM(vxlan_spec->vni)); - rule->ixgbe_fdir.formatted.tni_vni = rte_be_to_cpu_32( - rule->ixgbe_fdir.formatted.tni_vni); } } /* Get the NVGRE info */ if (item->type == RTE_FLOW_ITEM_TYPE_NVGRE) { rule->ixgbe_fdir.formatted.tunnel_type = - RTE_FDIR_TUNNEL_TYPE_NVGRE; + IXGBE_FDIR_NVGRE_TUNNEL_TYPE; /** * Only care about flags0, flags1, protocol and TNI, @@ -2524,7 +2523,6 @@ ixgbe_parse_fdir_filter_tunnel(const struct rte_flow_attr *attr, /* tni is a 24-bits bit field */ rte_memcpy(&rule->ixgbe_fdir.formatted.tni_vni, nvgre_spec->tni, RTE_DIM(nvgre_spec->tni)); - rule->ixgbe_fdir.formatted.tni_vni <<= 8; } } diff --git a/dpdk/drivers/net/kni/rte_eth_kni.c b/dpdk/drivers/net/kni/rte_eth_kni.c index c1a2ea51..5c795011 100644 --- a/dpdk/drivers/net/kni/rte_eth_kni.c +++ b/dpdk/drivers/net/kni/rte_eth_kni.c @@ -90,7 +90,7 @@ static const struct rte_eth_link pmd_link = { .link_speed = ETH_SPEED_NUM_10G, .link_duplex = ETH_LINK_FULL_DUPLEX, .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_AUTONEG, + .link_autoneg = ETH_LINK_FIXED, }; static int is_kni_initialized; diff --git a/dpdk/drivers/net/liquidio/lio_ethdev.c b/dpdk/drivers/net/liquidio/lio_ethdev.c index 84b8a328..0ca491b0 100644 --- a/dpdk/drivers/net/liquidio/lio_ethdev.c +++ b/dpdk/drivers/net/liquidio/lio_ethdev.c @@ -1479,6 +1479,11 @@ lio_dev_start(struct rte_eth_dev *eth_dev) /* Configure RSS if device configured with multiple RX queues. */ lio_dev_mq_rx_configure(eth_dev); + /* Before update the link info, + * must set linfo.link.link_status64 to 0. + */ + lio_dev->linfo.link.link_status64 = 0; + /* start polling for lsc */ ret = rte_eal_alarm_set(LIO_LSC_TIMEOUT, lio_sync_link_state_check, diff --git a/dpdk/drivers/net/mlx4/mlx4.c b/dpdk/drivers/net/mlx4/mlx4.c index 97dac643..d39ad3e0 100644 --- a/dpdk/drivers/net/mlx4/mlx4.c +++ b/dpdk/drivers/net/mlx4/mlx4.c @@ -85,6 +85,8 @@ const char *pmd_mlx4_init_params[] = { NULL, }; +static void mlx4_dev_stop(struct rte_eth_dev *dev); + /** * DPDK callback for Ethernet device configuration. * @@ -108,7 +110,13 @@ mlx4_dev_configure(struct rte_eth_dev *dev) " flow error type %d, cause %p, message: %s", -ret, strerror(-ret), error.type, error.cause, error.message ? error.message : "(unspecified)"); + goto exit; } + ret = mlx4_intr_install(priv); + if (ret) + ERROR("%p: interrupt handler installation failed", + (void *)dev); +exit: return ret; } @@ -141,7 +149,7 @@ mlx4_dev_start(struct rte_eth_dev *dev) (void *)dev, strerror(-ret)); goto err; } - ret = mlx4_intr_install(priv); + ret = mlx4_rxq_intr_enable(priv); if (ret) { ERROR("%p: interrupt handler installation failed", (void *)dev); @@ -161,8 +169,7 @@ mlx4_dev_start(struct rte_eth_dev *dev) dev->rx_pkt_burst = mlx4_rx_burst; return 0; err: - /* Rollback. */ - priv->started = 0; + mlx4_dev_stop(dev); return ret; } @@ -187,7 +194,7 @@ mlx4_dev_stop(struct rte_eth_dev *dev) dev->rx_pkt_burst = mlx4_rx_burst_removed; rte_wmb(); mlx4_flow_sync(priv, NULL); - mlx4_intr_uninstall(priv); + mlx4_rxq_intr_disable(priv); mlx4_rss_deinit(priv); } @@ -212,6 +219,7 @@ mlx4_dev_close(struct rte_eth_dev *dev) dev->tx_pkt_burst = mlx4_tx_burst_removed; rte_wmb(); mlx4_flow_clean(priv); + mlx4_rss_deinit(priv); for (i = 0; i != dev->data->nb_rx_queues; ++i) mlx4_rx_queue_release(dev->data->rx_queues[i]); for (i = 0; i != dev->data->nb_tx_queues; ++i) @@ -336,7 +344,7 @@ mlx4_arg_parse(const char *key, const char *val, struct mlx4_conf *conf) return -rte_errno; } if (strcmp(MLX4_PMD_PORT_KVARG, key) == 0) { - uint32_t ports = rte_log2_u32(conf->ports.present); + uint32_t ports = rte_log2_u32(conf->ports.present + 1); if (tmp >= ports) { ERROR("port index %lu outside range [0,%" PRIu32 ")", @@ -426,6 +434,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) int err = 0; struct ibv_context *attr_ctx = NULL; struct ibv_device_attr device_attr; + struct ibv_device_attr_ex device_attr_ex; struct mlx4_conf conf = { .ports.present = 0, }; @@ -486,19 +495,24 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) ibv_dev = list[i]; DEBUG("device opened"); if (ibv_query_device(attr_ctx, &device_attr)) { - rte_errno = ENODEV; + err = ENODEV; goto error; } INFO("%u port(s) detected", device_attr.phys_port_cnt); conf.ports.present |= (UINT64_C(1) << device_attr.phys_port_cnt) - 1; if (mlx4_args(pci_dev->device.devargs, &conf)) { ERROR("failed to process device arguments"); - rte_errno = EINVAL; + err = EINVAL; goto error; } /* Use all ports when none are defined */ if (!conf.ports.enabled) conf.ports.enabled = conf.ports.present; + /* Retrieve extended device attributes. */ + if (ibv_query_device_ex(attr_ctx, NULL, &device_attr_ex)) { + err = ENODEV; + goto error; + } for (i = 0; i < device_attr.phys_port_cnt; i++) { uint32_t port = i + 1; /* ports are indexed from one */ struct ibv_context *ctx = NULL; @@ -514,18 +528,18 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) DEBUG("using port %u", port); ctx = ibv_open_device(ibv_dev); if (ctx == NULL) { - rte_errno = ENODEV; + err = ENODEV; goto port_error; } /* Check port status. */ err = ibv_query_port(ctx, port, &port_attr); if (err) { - rte_errno = err; - ERROR("port query failed: %s", strerror(rte_errno)); + err = ENODEV; + ERROR("port query failed: %s", strerror(err)); goto port_error; } if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { - rte_errno = ENOTSUP; + err = ENOTSUP; ERROR("port %d is not configured in Ethernet mode", port); goto port_error; @@ -535,15 +549,16 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) port, ibv_port_state_str(port_attr.state), port_attr.state); /* Make asynchronous FD non-blocking to handle interrupts. */ - if (mlx4_fd_set_non_blocking(ctx->async_fd) < 0) { + err = mlx4_fd_set_non_blocking(ctx->async_fd); + if (err) { ERROR("cannot make asynchronous FD non-blocking: %s", - strerror(rte_errno)); + strerror(err)); goto port_error; } /* Allocate protection domain. */ pd = ibv_alloc_pd(ctx); if (pd == NULL) { - rte_errno = ENOMEM; + err = ENOMEM; ERROR("PD allocation failure"); goto port_error; } @@ -552,7 +567,7 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) sizeof(*priv), RTE_CACHE_LINE_SIZE); if (priv == NULL) { - rte_errno = ENOMEM; + err = ENOMEM; ERROR("priv allocation failure"); goto port_error; } @@ -573,10 +588,14 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) PCI_DEVICE_ID_MELLANOX_CONNECTX3PRO); DEBUG("L2 tunnel checksum offloads are %ssupported", (priv->hw_csum_l2tun ? "" : "not ")); + priv->hw_rss_max_qps = + device_attr_ex.rss_caps.max_rwq_indirection_table_size; + DEBUG("MAX RSS queues %d", priv->hw_rss_max_qps); /* Configure the first MAC address by default. */ - if (mlx4_get_mac(priv, &mac.addr_bytes)) { + err = mlx4_get_mac(priv, &mac.addr_bytes); + if (err) { ERROR("cannot get MAC address, is mlx4_en loaded?" - " (rte_errno: %s)", strerror(rte_errno)); + " (error: %s)", strerror(err)); goto port_error; } INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", @@ -609,8 +628,8 @@ mlx4_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) eth_dev = rte_eth_dev_allocate(name); } if (eth_dev == NULL) { + err = ENOMEM; ERROR("can not allocate rte ethdev"); - rte_errno = ENOMEM; goto port_error; } eth_dev->data->dev_private = priv; @@ -655,8 +674,6 @@ port_error: rte_eth_dev_release_port(eth_dev); break; } - if (i == device_attr.phys_port_cnt) - return 0; /* * XXX if something went wrong in the loop above, there is a resource * leak (ctx, pd, priv, dpdk ethdev) but we can do nothing about it as @@ -668,8 +685,9 @@ error: claim_zero(ibv_close_device(attr_ctx)); if (list) ibv_free_device_list(list); - assert(rte_errno >= 0); - return -rte_errno; + if (err) + rte_errno = err; + return -err; } static const struct rte_pci_id mlx4_pci_id_map[] = { diff --git a/dpdk/drivers/net/mlx4/mlx4.h b/dpdk/drivers/net/mlx4/mlx4.h index 3aeef87e..30f12eba 100644 --- a/dpdk/drivers/net/mlx4/mlx4.h +++ b/dpdk/drivers/net/mlx4/mlx4.h @@ -126,8 +126,10 @@ struct priv { uint32_t vf:1; /**< This is a VF device. */ uint32_t intr_alarm:1; /**< An interrupt alarm is scheduled. */ uint32_t isolated:1; /**< Toggle isolated mode. */ + uint32_t rss_init:1; /**< Common RSS context is initialized. */ uint32_t hw_csum:1; /* Checksum offload is supported. */ uint32_t hw_csum_l2tun:1; /* Checksum support for L2 tunnels. */ + uint32_t hw_rss_max_qps; /**< Max Rx Queues supported by RSS. */ struct rte_intr_handle intr_handle; /**< Port interrupt handle. */ struct mlx4_drop *drop; /**< Shared resources for drop flow rules. */ LIST_HEAD(, mlx4_rss) rss; /**< Shared targets for Rx flow rules. */ @@ -170,6 +172,8 @@ const uint32_t *mlx4_dev_supported_ptypes_get(struct rte_eth_dev *dev); int mlx4_intr_uninstall(struct priv *priv); int mlx4_intr_install(struct priv *priv); +int mlx4_rxq_intr_enable(struct priv *priv); +void mlx4_rxq_intr_disable(struct priv *priv); int mlx4_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx); int mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx); diff --git a/dpdk/drivers/net/mlx4/mlx4_ethdev.c b/dpdk/drivers/net/mlx4/mlx4_ethdev.c index 2f69e7d4..89f552c8 100644 --- a/dpdk/drivers/net/mlx4/mlx4_ethdev.c +++ b/dpdk/drivers/net/mlx4/mlx4_ethdev.c @@ -158,167 +158,6 @@ try_dev_id: return 0; } -/** - * Read from sysfs entry. - * - * @param[in] priv - * Pointer to private structure. - * @param[in] entry - * Entry name relative to sysfs path. - * @param[out] buf - * Data output buffer. - * @param size - * Buffer size. - * - * @return - * Number of bytes read on success, negative errno value otherwise and - * rte_errno is set. - */ -static int -mlx4_sysfs_read(const struct priv *priv, const char *entry, - char *buf, size_t size) -{ - char ifname[IF_NAMESIZE]; - FILE *file; - int ret; - - ret = mlx4_get_ifname(priv, &ifname); - if (ret) - return ret; - - MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, - ifname, entry); - - file = fopen(path, "rb"); - if (file == NULL) { - rte_errno = errno; - return -rte_errno; - } - ret = fread(buf, 1, size, file); - if ((size_t)ret < size && ferror(file)) { - rte_errno = EIO; - ret = -rte_errno; - } else { - ret = size; - } - fclose(file); - return ret; -} - -/** - * Write to sysfs entry. - * - * @param[in] priv - * Pointer to private structure. - * @param[in] entry - * Entry name relative to sysfs path. - * @param[in] buf - * Data buffer. - * @param size - * Buffer size. - * - * @return - * Number of bytes written on success, negative errno value otherwise and - * rte_errno is set. - */ -static int -mlx4_sysfs_write(const struct priv *priv, const char *entry, - char *buf, size_t size) -{ - char ifname[IF_NAMESIZE]; - FILE *file; - int ret; - - ret = mlx4_get_ifname(priv, &ifname); - if (ret) - return ret; - - MKSTR(path, "%s/device/net/%s/%s", priv->ctx->device->ibdev_path, - ifname, entry); - - file = fopen(path, "wb"); - if (file == NULL) { - rte_errno = errno; - return -rte_errno; - } - ret = fwrite(buf, 1, size, file); - if ((size_t)ret < size || ferror(file)) { - rte_errno = EIO; - ret = -rte_errno; - } else { - ret = size; - } - fclose(file); - return ret; -} - -/** - * Get unsigned long sysfs property. - * - * @param priv - * Pointer to private structure. - * @param[in] name - * Entry name relative to sysfs path. - * @param[out] value - * Value output buffer. - * - * @return - * 0 on success, negative errno value otherwise and rte_errno is set. - */ -static int -mlx4_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value) -{ - int ret; - unsigned long value_ret; - char value_str[32]; - - ret = mlx4_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1)); - if (ret < 0) { - DEBUG("cannot read %s value from sysfs: %s", - name, strerror(rte_errno)); - return ret; - } - value_str[ret] = '\0'; - errno = 0; - value_ret = strtoul(value_str, NULL, 0); - if (errno) { - rte_errno = errno; - DEBUG("invalid %s value `%s': %s", name, value_str, - strerror(rte_errno)); - return -rte_errno; - } - *value = value_ret; - return 0; -} - -/** - * Set unsigned long sysfs property. - * - * @param priv - * Pointer to private structure. - * @param[in] name - * Entry name relative to sysfs path. - * @param value - * Value to set. - * - * @return - * 0 on success, negative errno value otherwise and rte_errno is set. - */ -static int -mlx4_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) -{ - int ret; - MKSTR(value_str, "%lu", value); - - ret = mlx4_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1)); - if (ret < 0) { - DEBUG("cannot write %s `%s' (%lu) to sysfs: %s", - name, value_str, value, strerror(rte_errno)); - return ret; - } - return 0; -} - /** * Perform ifreq ioctl() on associated Ethernet device. * @@ -388,12 +227,12 @@ mlx4_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN]) int mlx4_mtu_get(struct priv *priv, uint16_t *mtu) { - unsigned long ulong_mtu = 0; - int ret = mlx4_get_sysfs_ulong(priv, "mtu", &ulong_mtu); + struct ifreq request; + int ret = mlx4_ifreq(priv, SIOCGIFMTU, &request); if (ret) return ret; - *mtu = ulong_mtu; + *mtu = request.ifr_mtu; return 0; } @@ -412,20 +251,13 @@ int mlx4_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) { struct priv *priv = dev->data->dev_private; - uint16_t new_mtu; - int ret = mlx4_set_sysfs_ulong(priv, "mtu", mtu); + struct ifreq request = { .ifr_mtu = mtu, }; + int ret = mlx4_ifreq(priv, SIOCSIFMTU, &request); if (ret) return ret; - ret = mlx4_mtu_get(priv, &new_mtu); - if (ret) - return ret; - if (new_mtu == mtu) { - priv->mtu = mtu; - return 0; - } - rte_errno = EINVAL; - return -rte_errno; + priv->mtu = mtu; + return 0; } /** @@ -444,14 +276,14 @@ mlx4_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) static int mlx4_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) { - unsigned long tmp = 0; - int ret = mlx4_get_sysfs_ulong(priv, "flags", &tmp); + struct ifreq request; + int ret = mlx4_ifreq(priv, SIOCGIFFLAGS, &request); if (ret) return ret; - tmp &= keep; - tmp |= (flags & (~keep)); - return mlx4_set_sysfs_ulong(priv, "flags", tmp); + request.ifr_flags &= keep; + request.ifr_flags |= flags & ~keep; + return mlx4_ifreq(priv, SIOCSIFFLAGS, &request); } /** diff --git a/dpdk/drivers/net/mlx4/mlx4_flow.c b/dpdk/drivers/net/mlx4/mlx4_flow.c index e81e24db..fc0f0614 100644 --- a/dpdk/drivers/net/mlx4/mlx4_flow.c +++ b/dpdk/drivers/net/mlx4/mlx4_flow.c @@ -116,7 +116,7 @@ static uint64_t mlx4_conv_rss_hf(uint64_t rss_hf) { enum { IPV4, IPV6, TCP, UDP, }; - const uint64_t in[] = { + static const uint64_t in[] = { [IPV4] = (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4 | ETH_RSS_NONFRAG_IPV4_TCP | @@ -139,7 +139,7 @@ mlx4_conv_rss_hf(uint64_t rss_hf) */ [UDP] = 0, }; - const uint64_t out[RTE_DIM(in)] = { + static const uint64_t out[RTE_DIM(in)] = { [IPV4] = IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4, [IPV6] = IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6, [TCP] = IBV_RX_HASH_SRC_PORT_TCP | IBV_RX_HASH_DST_PORT_TCP, @@ -379,6 +379,9 @@ error: * Additional mlx4-specific constraints on supported fields: * * - No support for partial masks. + * - Due to HW/FW limitation, flow rule priority is not taken into account + * when matching UDP destination ports, doing is therefore only supported + * at the highest priority level (0). * * @param[in, out] flow * Flow rule handle to update. @@ -410,6 +413,11 @@ mlx4_flow_merge_udp(struct rte_flow *flow, msg = "mlx4 does not support matching partial UDP fields"; goto error; } + if (mask && mask->hdr.dst_port && flow->priority) { + msg = "combining UDP destination port matching with a nonzero" + " priority level is not supported"; + goto error; + } if (!flow->ibv_attr) return 0; ++flow->ibv_attr->num_of_specs; @@ -674,6 +682,7 @@ mlx4_flow_prepare(struct priv *priv, NULL, "only ingress is supported"); fill: proc = mlx4_flow_proc_item_list; + flow->priority = attr->priority; /* Go over pattern. */ for (item = pattern; item->type; ++item) { const struct mlx4_flow_proc_item *next = NULL; @@ -839,11 +848,14 @@ fill: }, }; - if (!mlx4_zmallocv(__func__, vec, RTE_DIM(vec))) + if (!mlx4_zmallocv(__func__, vec, RTE_DIM(vec))) { + if (temp.rss) + mlx4_rss_put(temp.rss); return rte_flow_error_set (error, -rte_errno, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "flow rule handle allocation failure"); + } /* Most fields will be updated by second pass. */ *flow = (struct rte_flow){ .ibv_attr = temp.ibv_attr, @@ -1217,9 +1229,12 @@ mlx4_flow_internal_next_vlan(struct priv *priv, uint16_t vlan) * * Various flow rules are created depending on the mode the device is in: * - * 1. Promiscuous: port MAC + catch-all (VLAN filtering is ignored). - * 2. All multicast: port MAC/VLAN + catch-all multicast. - * 3. Otherwise: port MAC/VLAN + broadcast MAC/VLAN. + * 1. Promiscuous: + * port MAC + broadcast + catch-all (VLAN filtering is ignored). + * 2. All multicast: + * port MAC/VLAN + broadcast + catch-all multicast. + * 3. Otherwise: + * port MAC/VLAN + broadcast MAC/VLAN. * * About MAC flow rules: * @@ -1298,9 +1313,6 @@ mlx4_flow_internal(struct priv *priv, struct rte_flow_error *error) !priv->dev->data->promiscuous ? &vlan_spec.tci : NULL; - int broadcast = - !priv->dev->data->promiscuous && - !priv->dev->data->all_multicast; uint16_t vlan = 0; struct rte_flow *flow; unsigned int i; @@ -1334,7 +1346,7 @@ next_vlan: rule_vlan = NULL; } } - for (i = 0; i != RTE_DIM(priv->mac) + broadcast; ++i) { + for (i = 0; i != RTE_DIM(priv->mac) + 1; ++i) { const struct ether_addr *mac; /* Broadcasts are handled by an extra iteration. */ @@ -1398,7 +1410,7 @@ next_vlan: goto next_vlan; } /* Take care of promiscuous and all multicast flow rules. */ - if (!broadcast) { + if (priv->dev->data->promiscuous || priv->dev->data->all_multicast) { for (flow = LIST_FIRST(&priv->flows); flow && flow->internal; flow = LIST_NEXT(flow, next)) { diff --git a/dpdk/drivers/net/mlx4/mlx4_flow.h b/dpdk/drivers/net/mlx4/mlx4_flow.h index 651fd37b..5e1f9ea7 100644 --- a/dpdk/drivers/net/mlx4/mlx4_flow.h +++ b/dpdk/drivers/net/mlx4/mlx4_flow.h @@ -70,6 +70,7 @@ struct rte_flow { uint32_t promisc:1; /**< This rule matches everything. */ uint32_t allmulti:1; /**< This rule matches all multicast traffic. */ uint32_t drop:1; /**< This rule drops packets. */ + uint32_t priority; /**< Flow rule priority. */ struct mlx4_rss *rss; /**< Rx target. */ }; diff --git a/dpdk/drivers/net/mlx4/mlx4_intr.c b/dpdk/drivers/net/mlx4/mlx4_intr.c index 50d19769..2364cb20 100644 --- a/dpdk/drivers/net/mlx4/mlx4_intr.c +++ b/dpdk/drivers/net/mlx4/mlx4_intr.c @@ -291,7 +291,7 @@ mlx4_intr_uninstall(struct priv *priv) } rte_eal_alarm_cancel((void (*)(void *))mlx4_link_status_alarm, priv); priv->intr_alarm = 0; - mlx4_rx_intr_vec_disable(priv); + mlx4_rxq_intr_disable(priv); rte_errno = err; return 0; } @@ -313,8 +313,6 @@ mlx4_intr_install(struct priv *priv) int rc; mlx4_intr_uninstall(priv); - if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0) - goto error; if (intr_conf->lsc | intr_conf->rmv) { priv->intr_handle.fd = priv->ctx->async_fd; rc = rte_intr_callback_register(&priv->intr_handle, @@ -395,3 +393,40 @@ mlx4_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx) } return -ret; } + +/** + * Enable datapath interrupts. + * + * @param priv + * Pointer to private structure. + * + * @return + * 0 on success, negative errno value otherwise and rte_errno is set. + */ +int +mlx4_rxq_intr_enable(struct priv *priv) +{ + const struct rte_intr_conf *const intr_conf = + &priv->dev->data->dev_conf.intr_conf; + + if (intr_conf->rxq && mlx4_rx_intr_vec_enable(priv) < 0) + goto error; + return 0; +error: + return -rte_errno; +} + +/** + * Disable datapath interrupts, keeping other interrupts intact. + * + * @param priv + * Pointer to private structure. + */ +void +mlx4_rxq_intr_disable(struct priv *priv) +{ + int err = rte_errno; /* Make sure rte_errno remains unchanged. */ + + mlx4_rx_intr_vec_disable(priv); + rte_errno = err; +} diff --git a/dpdk/drivers/net/mlx4/mlx4_rxq.c b/dpdk/drivers/net/mlx4/mlx4_rxq.c index 53313c56..b3f70d6c 100644 --- a/dpdk/drivers/net/mlx4/mlx4_rxq.c +++ b/dpdk/drivers/net/mlx4/mlx4_rxq.c @@ -363,6 +363,14 @@ mlx4_rss_init(struct priv *priv) unsigned int i; int ret; + if (priv->rss_init) + return 0; + if (priv->dev->data->nb_rx_queues > priv->hw_rss_max_qps) { + ERROR("RSS does not support more than %d queues", + priv->hw_rss_max_qps); + rte_errno = EINVAL; + return -rte_errno; + } /* Prepare range for RSS contexts before creating the first WQ. */ ret = mlx4dv_set_context_attr(priv->ctx, MLX4DV_SET_CTX_ATTR_LOG_WQS_RANGE_SZ, @@ -444,6 +452,7 @@ wq_num_check: } wq_num_prev = wq_num; } + priv->rss_init = 1; return 0; error: ERROR("cannot initialize common RSS resources (queue %u): %s: %s", @@ -472,6 +481,8 @@ mlx4_rss_deinit(struct priv *priv) { unsigned int i; + if (!priv->rss_init) + return; for (i = 0; i != priv->dev->data->nb_rx_queues; ++i) { struct rxq *rxq = priv->dev->data->rx_queues[i]; @@ -480,6 +491,7 @@ mlx4_rss_deinit(struct priv *priv) mlx4_rxq_detach(rxq); } } + priv->rss_init = 0; } /** @@ -622,6 +634,7 @@ error: claim_zero(ibv_destroy_wq(wq)); if (cq) claim_zero(ibv_destroy_cq(cq)); + --rxq->usecnt; rte_errno = ret; ERROR("error while attaching Rx queue %p: %s: %s", (void *)rxq, msg, strerror(ret)); diff --git a/dpdk/drivers/net/mlx4/mlx4_rxtx.c b/dpdk/drivers/net/mlx4/mlx4_rxtx.c index 92b62577..05c48920 100644 --- a/dpdk/drivers/net/mlx4/mlx4_rxtx.c +++ b/dpdk/drivers/net/mlx4/mlx4_rxtx.c @@ -961,7 +961,8 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) /* Update packet information. */ pkt->packet_type = rxq_cq_to_pkt_type(cqe, rxq->l2tun_offload); - pkt->ol_flags = 0; + pkt->ol_flags = PKT_RX_RSS_HASH; + pkt->hash.rss = cqe->immed_rss_invalid; pkt->pkt_len = len; if (rxq->csum | rxq->csum_l2tun) { uint32_t flags = diff --git a/dpdk/drivers/net/mlx4/mlx4_rxtx.h b/dpdk/drivers/net/mlx4/mlx4_rxtx.h index 463df2b0..5a390e85 100644 --- a/dpdk/drivers/net/mlx4/mlx4_rxtx.h +++ b/dpdk/drivers/net/mlx4/mlx4_rxtx.h @@ -107,7 +107,7 @@ struct txq_elt { struct rte_mbuf *buf; /**< Buffer. */ }; -/** Rx queue counters. */ +/** Tx queue counters. */ struct mlx4_txq_stats { unsigned int idx; /**< Mapping index. */ uint64_t opackets; /**< Total of successfully sent packets. */ diff --git a/dpdk/drivers/net/mlx5/Makefile b/dpdk/drivers/net/mlx5/Makefile index a3984eb9..c62ad118 100644 --- a/dpdk/drivers/net/mlx5/Makefile +++ b/dpdk/drivers/net/mlx5/Makefile @@ -145,7 +145,52 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh $Q sh -- '$<' '$@' \ HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT \ infiniband/verbs.h \ - enum IBV_FLOW_SPEC_ACTION_COUNT \ + type 'struct ibv_counter_set_init_attr' \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_SUPPORTED_40000baseKR4_Full \ + /usr/include/linux/ethtool.h \ + define SUPPORTED_40000baseKR4_Full \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_SUPPORTED_40000baseCR4_Full \ + /usr/include/linux/ethtool.h \ + define SUPPORTED_40000baseCR4_Full \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_SUPPORTED_40000baseSR4_Full \ + /usr/include/linux/ethtool.h \ + define SUPPORTED_40000baseSR4_Full \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_SUPPORTED_40000baseLR4_Full \ + /usr/include/linux/ethtool.h \ + define SUPPORTED_40000baseLR4_Full \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_SUPPORTED_56000baseKR4_Full \ + /usr/include/linux/ethtool.h \ + define SUPPORTED_56000baseKR4_Full \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_SUPPORTED_56000baseCR4_Full \ + /usr/include/linux/ethtool.h \ + define SUPPORTED_56000baseCR4_Full \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_SUPPORTED_56000baseSR4_Full \ + /usr/include/linux/ethtool.h \ + define SUPPORTED_56000baseSR4_Full \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_SUPPORTED_56000baseLR4_Full \ + /usr/include/linux/ethtool.h \ + define SUPPORTED_56000baseLR4_Full \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_STATIC_ASSERT \ + /usr/include/assert.h \ + define static_assert \ $(AUTOCONF_OUTPUT) # Create mlx5_autoconf.h or update it in case it differs from the new one. diff --git a/dpdk/drivers/net/mlx5/mlx5.c b/dpdk/drivers/net/mlx5/mlx5.c index 45e0e8db..36f3a056 100644 --- a/dpdk/drivers/net/mlx5/mlx5.c +++ b/dpdk/drivers/net/mlx5/mlx5.c @@ -39,6 +39,7 @@ #include #include #include +#include /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ @@ -56,6 +57,7 @@ #include #include #include +#include #include #include "mlx5.h" @@ -117,6 +119,10 @@ struct mlx5_args { int tx_vec_en; int rx_vec_en; }; + +/** Driver-specific log messages type. */ +int mlx5_logtype; + /** * Retrieve integer value from environment variable. * @@ -148,7 +154,7 @@ mlx5_getenv_int(const char *name) * A pointer to the callback data. * * @return - * a pointer to the allocate space. + * Allocated buffer, NULL otherwise and rte_errno is set. */ static void * mlx5_alloc_verbs_buf(size_t size, void *data) @@ -156,11 +162,22 @@ mlx5_alloc_verbs_buf(size_t size, void *data) struct priv *priv = data; void *ret; size_t alignment = sysconf(_SC_PAGESIZE); + unsigned int socket = SOCKET_ID_ANY; + if (priv->verbs_alloc_ctx.type == MLX5_VERBS_ALLOC_TYPE_TX_QUEUE) { + const struct mlx5_txq_ctrl *ctrl = priv->verbs_alloc_ctx.obj; + + socket = ctrl->socket; + } else if (priv->verbs_alloc_ctx.type == + MLX5_VERBS_ALLOC_TYPE_RX_QUEUE) { + const struct mlx5_rxq_ctrl *ctrl = priv->verbs_alloc_ctx.obj; + + socket = ctrl->socket; + } assert(data != NULL); - ret = rte_malloc_socket(__func__, size, alignment, - priv->dev->device->numa_node); - DEBUG("Extern alloc size: %lu, align: %lu: %p", size, alignment, ret); + ret = rte_malloc_socket(__func__, size, alignment, socket); + if (!ret && size) + rte_errno = ENOMEM; return ret; } @@ -176,7 +193,6 @@ static void mlx5_free_verbs_buf(void *ptr, void *data __rte_unused) { assert(data != NULL); - DEBUG("Extern free request: %p", ptr); rte_free(ptr); } @@ -191,17 +207,16 @@ mlx5_free_verbs_buf(void *ptr, void *data __rte_unused) static void mlx5_dev_close(struct rte_eth_dev *dev) { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; unsigned int i; int ret; - priv_lock(priv); - DEBUG("%p: closing device \"%s\"", - (void *)dev, - ((priv->ctx != NULL) ? priv->ctx->device->name : "")); + DRV_LOG(DEBUG, "port %u closing device \"%s\"", + dev->data->port_id, + ((priv->ctx != NULL) ? priv->ctx->device->name : "")); /* In case mlx5_dev_stop() has not been called. */ - priv_dev_interrupt_handler_uninstall(priv, dev); - priv_dev_traffic_disable(priv, dev); + mlx5_dev_interrupt_handler_uninstall(dev); + mlx5_traffic_disable(dev); /* Prevent crashes when queues are still in use. */ dev->rx_pkt_burst = removed_rx_burst; dev->tx_pkt_burst = removed_tx_burst; @@ -209,7 +224,7 @@ mlx5_dev_close(struct rte_eth_dev *dev) /* XXX race condition if mlx5_rx_burst() is still running. */ usleep(1000); for (i = 0; (i != priv->rxqs_n); ++i) - mlx5_priv_rxq_release(priv, i); + mlx5_rxq_release(dev, i); priv->rxqs_n = 0; priv->rxqs = NULL; } @@ -217,7 +232,7 @@ mlx5_dev_close(struct rte_eth_dev *dev) /* XXX race condition if mlx5_tx_burst() is still running. */ usleep(1000); for (i = 0; (i != priv->txqs_n); ++i) - mlx5_priv_txq_release(priv, i); + mlx5_txq_release(dev, i); priv->txqs_n = 0; priv->txqs = NULL; } @@ -231,32 +246,40 @@ mlx5_dev_close(struct rte_eth_dev *dev) rte_free(priv->rss_conf.rss_key); if (priv->reta_idx != NULL) rte_free(priv->reta_idx); - priv_socket_uninit(priv); - ret = mlx5_priv_hrxq_ibv_verify(priv); + if (priv->primary_socket) + mlx5_socket_uninit(dev); + ret = mlx5_hrxq_ibv_verify(dev); if (ret) - WARN("%p: some Hash Rx queue still remain", (void *)priv); - ret = mlx5_priv_ind_table_ibv_verify(priv); + DRV_LOG(WARNING, "port %u some hash Rx queue still remain", + dev->data->port_id); + ret = mlx5_ind_table_ibv_verify(dev); if (ret) - WARN("%p: some Indirection table still remain", (void *)priv); - ret = mlx5_priv_rxq_ibv_verify(priv); + DRV_LOG(WARNING, "port %u some indirection table still remain", + dev->data->port_id); + ret = mlx5_rxq_ibv_verify(dev); if (ret) - WARN("%p: some Verbs Rx queue still remain", (void *)priv); - ret = mlx5_priv_rxq_verify(priv); + DRV_LOG(WARNING, "port %u some Verbs Rx queue still remain", + dev->data->port_id); + ret = mlx5_rxq_verify(dev); if (ret) - WARN("%p: some Rx Queues still remain", (void *)priv); - ret = mlx5_priv_txq_ibv_verify(priv); + DRV_LOG(WARNING, "port %u some Rx queues still remain", + dev->data->port_id); + ret = mlx5_txq_ibv_verify(dev); if (ret) - WARN("%p: some Verbs Tx queue still remain", (void *)priv); - ret = mlx5_priv_txq_verify(priv); + DRV_LOG(WARNING, "port %u some Verbs Tx queue still remain", + dev->data->port_id); + ret = mlx5_txq_verify(dev); if (ret) - WARN("%p: some Tx Queues still remain", (void *)priv); - ret = priv_flow_verify(priv); + DRV_LOG(WARNING, "port %u some Tx queues still remain", + dev->data->port_id); + ret = mlx5_flow_verify(dev); if (ret) - WARN("%p: some flows still remain", (void *)priv); - ret = priv_mr_verify(priv); + DRV_LOG(WARNING, "port %u some flows still remain", + dev->data->port_id); + ret = mlx5_mr_verify(dev); if (ret) - WARN("%p: some Memory Region still remain", (void *)priv); - priv_unlock(priv); + DRV_LOG(WARNING, "port %u some memory region still remain", + dev->data->port_id); memset(priv, 0, sizeof(*priv)); } @@ -322,6 +345,10 @@ const struct eth_dev_ops mlx5_dev_ops_isolate = { .dev_set_link_down = mlx5_set_link_down, .dev_set_link_up = mlx5_set_link_up, .dev_close = mlx5_dev_close, + .promiscuous_enable = mlx5_promiscuous_enable, + .promiscuous_disable = mlx5_promiscuous_disable, + .allmulticast_enable = mlx5_allmulticast_enable, + .allmulticast_disable = mlx5_allmulticast_disable, .link_update = mlx5_link_update, .stats_get = mlx5_stats_get, .stats_reset = mlx5_stats_reset, @@ -394,7 +421,7 @@ mlx5_dev_idx(struct rte_pci_addr *pci_addr) * User data. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_args_check(const char *key, const char *val, void *opaque) @@ -405,8 +432,9 @@ mlx5_args_check(const char *key, const char *val, void *opaque) errno = 0; tmp = strtoul(val, NULL, 0); if (errno) { - WARN("%s: \"%s\" is not a valid integer", key, val); - return errno; + rte_errno = errno; + DRV_LOG(WARNING, "%s: \"%s\" is not a valid integer", key, val); + return -rte_errno; } if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) { args->cqe_comp = !!tmp; @@ -427,8 +455,9 @@ mlx5_args_check(const char *key, const char *val, void *opaque) } else if (strcmp(MLX5_RX_VEC_EN, key) == 0) { args->rx_vec_en = !!tmp; } else { - WARN("%s: unknown parameter", key); - return -EINVAL; + DRV_LOG(WARNING, "%s: unknown parameter", key); + rte_errno = EINVAL; + return -rte_errno; } return 0; } @@ -442,7 +471,7 @@ mlx5_args_check(const char *key, const char *val, void *opaque) * Device arguments structure. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_args(struct mlx5_args *args, struct rte_devargs *devargs) @@ -474,9 +503,10 @@ mlx5_args(struct mlx5_args *args, struct rte_devargs *devargs) if (rte_kvargs_count(kvlist, params[i])) { ret = rte_kvargs_process(kvlist, params[i], mlx5_args_check, args); - if (ret != 0) { + if (ret) { + rte_errno = EINVAL; rte_kvargs_free(kvlist); - return ret; + return -rte_errno; } } } @@ -486,6 +516,112 @@ mlx5_args(struct mlx5_args *args, struct rte_devargs *devargs) static struct rte_pci_driver mlx5_driver; +/* + * Reserved UAR address space for TXQ UAR(hw doorbell) mapping, process + * local resource used by both primary and secondary to avoid duplicate + * reservation. + * The space has to be available on both primary and secondary process, + * TXQ UAR maps to this area using fixed mmap w/o double check. + */ +static void *uar_base; + +/** + * Reserve UAR address space for primary process. + * + * @param[in] dev + * Pointer to Ethernet device. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_uar_init_primary(struct rte_eth_dev *dev) +{ + struct priv *priv = dev->data->dev_private; + void *addr = (void *)0; + int i; + const struct rte_mem_config *mcfg; + + if (uar_base) { /* UAR address space mapped. */ + priv->uar_base = uar_base; + return 0; + } + /* find out lower bound of hugepage segments */ + mcfg = rte_eal_get_configuration()->mem_config; + for (i = 0; i < RTE_MAX_MEMSEG && mcfg->memseg[i].addr; i++) { + if (addr) + addr = RTE_MIN(addr, mcfg->memseg[i].addr); + else + addr = mcfg->memseg[i].addr; + } + /* keep distance to hugepages to minimize potential conflicts. */ + addr = RTE_PTR_SUB(addr, MLX5_UAR_OFFSET + MLX5_UAR_SIZE); + /* anonymous mmap, no real memory consumption. */ + addr = mmap(addr, MLX5_UAR_SIZE, + PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) { + DRV_LOG(ERR, + "port %u failed to reserve UAR address space, please" + " adjust MLX5_UAR_SIZE or try --base-virtaddr", + dev->data->port_id); + rte_errno = ENOMEM; + return -rte_errno; + } + /* Accept either same addr or a new addr returned from mmap if target + * range occupied. + */ + DRV_LOG(INFO, "port %u reserved UAR address space: %p", + dev->data->port_id, addr); + priv->uar_base = addr; /* for primary and secondary UAR re-mmap. */ + uar_base = addr; /* process local, don't reserve again. */ + return 0; +} + +/** + * Reserve UAR address space for secondary process, align with + * primary process. + * + * @param[in] dev + * Pointer to Ethernet device. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_uar_init_secondary(struct rte_eth_dev *dev) +{ + struct priv *priv = dev->data->dev_private; + void *addr; + + assert(priv->uar_base); + if (uar_base) { /* already reserved. */ + assert(uar_base == priv->uar_base); + return 0; + } + /* anonymous mmap, no real memory consumption. */ + addr = mmap(priv->uar_base, MLX5_UAR_SIZE, + PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (addr == MAP_FAILED) { + DRV_LOG(ERR, "port %u UAR mmap failed: %p size: %llu", + dev->data->port_id, priv->uar_base, MLX5_UAR_SIZE); + rte_errno = ENXIO; + return -rte_errno; + } + if (priv->uar_base != addr) { + DRV_LOG(ERR, + "port %u UAR address %p size %llu occupied, please" + " adjust MLX5_UAR_OFFSET or try EAL parameter" + " --base-virtaddr", + dev->data->port_id, priv->uar_base, MLX5_UAR_SIZE); + rte_errno = ENXIO; + return -rte_errno; + } + uar_base = addr; /* process local, don't reserve again */ + DRV_LOG(INFO, "port %u reserved UAR address space: %p", + dev->data->port_id, addr); + return 0; +} + /** * Assign parameters from args into priv, only non default * values are considered. @@ -530,17 +666,17 @@ mlx5_args_assign(struct priv *priv, struct mlx5_args *args) * PCI device information. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) +mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, + struct rte_pci_device *pci_dev) { - struct ibv_device **list; + struct ibv_device **list = NULL; struct ibv_device *ibv_dev; int err = 0; struct ibv_context *attr_ctx = NULL; struct ibv_device_attr_ex device_attr; - unsigned int sriov; unsigned int mps; unsigned int cqe_comp; unsigned int tunnel_en = 0; @@ -548,27 +684,28 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) int i; struct mlx5dv_context attrs_out; #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT - struct ibv_counter_set_description cs_desc; + struct ibv_counter_set_description cs_desc = { .counter_type = 0 }; #endif - (void)pci_drv; assert(pci_drv == &mlx5_driver); /* Get mlx5_dev[] index. */ idx = mlx5_dev_idx(&pci_dev->addr); if (idx == -1) { - ERROR("this driver cannot support any more adapters"); - return -ENOMEM; + DRV_LOG(ERR, "this driver cannot support any more adapters"); + err = ENOMEM; + goto error; } - DEBUG("using driver device index %d", idx); - + DRV_LOG(DEBUG, "using driver device index %d", idx); /* Save PCI address. */ mlx5_dev[idx].pci_addr = pci_dev->addr; list = ibv_get_device_list(&i); if (list == NULL) { assert(errno); + err = errno; if (errno == ENOSYS) - ERROR("cannot list devices, is ib_uverbs loaded?"); - return -errno; + DRV_LOG(ERR, + "cannot list devices, is ib_uverbs loaded?"); + goto error; } assert(i >= 0); /* @@ -579,7 +716,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) struct rte_pci_addr pci_addr; --i; - DEBUG("checking device \"%s\"", list[i]->name); + DRV_LOG(DEBUG, "checking device \"%s\"", list[i]->name); if (mlx5_ibv_device_to_pci_addr(list[i], &pci_addr)) continue; if ((pci_dev->addr.domain != pci_addr.domain) || @@ -587,14 +724,6 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) (pci_dev->addr.devid != pci_addr.devid) || (pci_dev->addr.function != pci_addr.function)) continue; - sriov = ((pci_dev->id.device_id == - PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) || - (pci_dev->id.device_id == - PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF) || - (pci_dev->id.device_id == - PCI_DEVICE_ID_MELLANOX_CONNECTX5VF) || - (pci_dev->id.device_id == - PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF)); switch (pci_dev->id.device_id) { case PCI_DEVICE_ID_MELLANOX_CONNECTX4: tunnel_en = 1; @@ -609,30 +738,29 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) default: break; } - INFO("PCI information matches, using device \"%s\"" - " (SR-IOV: %s)", - list[i]->name, - sriov ? "true" : "false"); + DRV_LOG(INFO, "PCI information matches, using device \"%s\"", + list[i]->name); attr_ctx = ibv_open_device(list[i]); - err = errno; + rte_errno = errno; + err = rte_errno; break; } if (attr_ctx == NULL) { - ibv_free_device_list(list); switch (err) { case 0: - ERROR("cannot access device, is mlx5_ib loaded?"); - return -ENODEV; + DRV_LOG(ERR, + "cannot access device, is mlx5_ib loaded?"); + err = ENODEV; + break; case EINVAL: - ERROR("cannot use device, are drivers up to date?"); - return -EINVAL; + DRV_LOG(ERR, + "cannot use device, are drivers up to date?"); + break; } - assert(err > 0); - return -err; + goto error; } ibv_dev = list[i]; - - DEBUG("device opened"); + DRV_LOG(DEBUG, "device opened"); /* * Multi-packet send is supported by ConnectX-4 Lx PF as well * as all ConnectX-5 devices. @@ -640,14 +768,14 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) mlx5dv_query_device(attr_ctx, &attrs_out); if (attrs_out.flags & MLX5DV_CONTEXT_FLAGS_MPW_ALLOWED) { if (attrs_out.flags & MLX5DV_CONTEXT_FLAGS_ENHANCED_MPW) { - DEBUG("Enhanced MPW is supported"); + DRV_LOG(DEBUG, "enhanced MPW is supported"); mps = MLX5_MPW_ENHANCED; } else { - DEBUG("MPW is supported"); + DRV_LOG(DEBUG, "MPW is supported"); mps = MLX5_MPW; } } else { - DEBUG("MPW isn't supported"); + DRV_LOG(DEBUG, "MPW isn't supported"); mps = MLX5_MPW_DISABLED; } if (RTE_CACHE_LINE_SIZE == 128 && @@ -655,10 +783,13 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) cqe_comp = 0; else cqe_comp = 1; - if (ibv_query_device_ex(attr_ctx, NULL, &device_attr)) + err = ibv_query_device_ex(attr_ctx, NULL, &device_attr); + if (err) { + DEBUG("ibv_query_device_ex() failed"); goto error; - INFO("%u port(s) detected", device_attr.orig_attr.phys_port_cnt); - + } + DRV_LOG(INFO, "%u port(s) detected", + device_attr.orig_attr.phys_port_cnt); for (i = 0; i < device_attr.orig_attr.phys_port_cnt; i++) { char name[RTE_ETH_NAME_MAX_LEN]; uint32_t port = i + 1; /* ports are indexed from one */ @@ -667,11 +798,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) struct ibv_port_attr port_attr; struct ibv_pd *pd = NULL; struct priv *priv = NULL; - struct rte_eth_dev *eth_dev; + struct rte_eth_dev *eth_dev = NULL; struct ibv_device_attr_ex device_attr_ex; struct ether_addr mac; - uint16_t num_vfs = 0; - struct ibv_device_attr_ex device_attr; struct mlx5_args args = { .cqe_comp = MLX5_ARG_UNSET, .txq_inline = MLX5_ARG_UNSET, @@ -687,84 +816,85 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) snprintf(name, sizeof(name), PCI_PRI_FMT, pci_dev->addr.domain, pci_dev->addr.bus, pci_dev->addr.devid, pci_dev->addr.function); - mlx5_dev[idx].ports |= test; - if (rte_eal_process_type() == RTE_PROC_SECONDARY) { eth_dev = rte_eth_dev_attach_secondary(name); if (eth_dev == NULL) { - ERROR("can not attach rte ethdev"); - err = ENOMEM; + DRV_LOG(ERR, "can not attach rte ethdev"); + rte_errno = ENOMEM; + err = rte_errno; goto error; } eth_dev->device = &pci_dev->device; eth_dev->dev_ops = &mlx5_dev_sec_ops; - priv = eth_dev->data->dev_private; + err = mlx5_uar_init_secondary(eth_dev); + if (err) { + err = rte_errno; + goto error; + } /* Receive command fd from primary process */ - err = priv_socket_connect(priv); + err = mlx5_socket_connect(eth_dev); if (err < 0) { - err = -err; + err = rte_errno; goto error; } /* Remap UAR for Tx queues. */ - err = priv_tx_uar_remap(priv, err); - if (err < 0) { - err = -err; + err = mlx5_tx_uar_remap(eth_dev, err); + if (err) { + err = rte_errno; goto error; } - priv_dev_select_rx_function(priv, eth_dev); - priv_dev_select_tx_function(priv, eth_dev); + /* + * Ethdev pointer is still required as input since + * the primary device is not accessible from the + * secondary process. + */ + eth_dev->rx_pkt_burst = + mlx5_select_rx_function(eth_dev); + eth_dev->tx_pkt_burst = + mlx5_select_tx_function(eth_dev); continue; } - - DEBUG("using port %u (%08" PRIx32 ")", port, test); - + DRV_LOG(DEBUG, "using port %u (%08" PRIx32 ")", port, test); ctx = ibv_open_device(ibv_dev); if (ctx == NULL) { err = ENODEV; goto port_error; } - - ibv_query_device_ex(ctx, NULL, &device_attr); /* Check port status. */ err = ibv_query_port(ctx, port, &port_attr); if (err) { - ERROR("port query failed: %s", strerror(err)); + DRV_LOG(ERR, "port query failed: %s", strerror(err)); goto port_error; } - if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) { - ERROR("port %d is not configured in Ethernet mode", - port); + DRV_LOG(ERR, + "port %d is not configured in Ethernet mode", + port); err = EINVAL; goto port_error; } - if (port_attr.state != IBV_PORT_ACTIVE) - DEBUG("port %d is not active: \"%s\" (%d)", - port, ibv_port_state_str(port_attr.state), - port_attr.state); - + DRV_LOG(DEBUG, "port %d is not active: \"%s\" (%d)", + port, ibv_port_state_str(port_attr.state), + port_attr.state); /* Allocate protection domain. */ pd = ibv_alloc_pd(ctx); if (pd == NULL) { - ERROR("PD allocation failure"); + DRV_LOG(ERR, "PD allocation failure"); err = ENOMEM; goto port_error; } - mlx5_dev[idx].ports |= test; - /* from rte_ethdev.c */ priv = rte_zmalloc("ethdev private structure", sizeof(*priv), RTE_CACHE_LINE_SIZE); if (priv == NULL) { - ERROR("priv allocation failure"); + DRV_LOG(ERR, "priv allocation failure"); err = ENOMEM; goto port_error; } - priv->ctx = ctx; strncpy(priv->ibdev_path, priv->ctx->device->ibdev_path, sizeof(priv->ibdev_path)); @@ -780,35 +910,37 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) priv->rx_vec_en = 1; err = mlx5_args(&args, pci_dev->device.devargs); if (err) { - ERROR("failed to process device arguments: %s", - strerror(err)); + DRV_LOG(ERR, "failed to process device arguments: %s", + strerror(err)); + err = rte_errno; goto port_error; } mlx5_args_assign(priv, &args); - if (ibv_query_device_ex(ctx, NULL, &device_attr_ex)) { - ERROR("ibv_query_device_ex() failed"); + err = ibv_query_device_ex(ctx, NULL, &device_attr_ex); + if (err) { + DRV_LOG(ERR, "ibv_query_device_ex() failed"); goto port_error; } - priv->hw_csum = !!(device_attr_ex.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM); - DEBUG("checksum offloading is %ssupported", - (priv->hw_csum ? "" : "not ")); + DRV_LOG(DEBUG, "checksum offloading is %ssupported", + (priv->hw_csum ? "" : "not ")); #ifdef HAVE_IBV_DEVICE_VXLAN_SUPPORT priv->hw_csum_l2tun = !!(exp_device_attr.exp_device_cap_flags & IBV_DEVICE_VXLAN_SUPPORT); #endif - DEBUG("Rx L2 tunnel checksum offloads are %ssupported", - (priv->hw_csum_l2tun ? "" : "not ")); + DRV_LOG(DEBUG, "Rx L2 tunnel checksum offloads are %ssupported", + (priv->hw_csum_l2tun ? "" : "not ")); #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT priv->counter_set_supported = !!(device_attr.max_counter_sets); ibv_describe_counter_set(ctx, 0, &cs_desc); - DEBUG("counter type = %d, num of cs = %ld, attributes = %d", - cs_desc.counter_type, cs_desc.num_of_cs, - cs_desc.attributes); + DRV_LOG(DEBUG, + "counter type = %d, num of cs = %ld, attributes = %d", + cs_desc.counter_type, cs_desc.num_of_cs, + cs_desc.attributes); #endif priv->ind_table_max_size = device_attr_ex.rss_caps.max_rwq_indirection_table_size; @@ -817,27 +949,24 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) if (priv->ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512) priv->ind_table_max_size = ETH_RSS_RETA_SIZE_512; - DEBUG("maximum RX indirection table size is %u", - priv->ind_table_max_size); + DRV_LOG(DEBUG, "maximum Rx indirection table size is %u", + priv->ind_table_max_size); priv->hw_vlan_strip = !!(device_attr_ex.raw_packet_caps & IBV_RAW_PACKET_CAP_CVLAN_STRIPPING); - DEBUG("VLAN stripping is %ssupported", - (priv->hw_vlan_strip ? "" : "not ")); + DRV_LOG(DEBUG, "VLAN stripping is %ssupported", + (priv->hw_vlan_strip ? "" : "not ")); - priv->hw_fcs_strip = - !!(device_attr_ex.orig_attr.device_cap_flags & - IBV_WQ_FLAGS_SCATTER_FCS); - DEBUG("FCS stripping configuration is %ssupported", - (priv->hw_fcs_strip ? "" : "not ")); + priv->hw_fcs_strip = !!(device_attr_ex.raw_packet_caps & + IBV_RAW_PACKET_CAP_SCATTER_FCS); + DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported", + (priv->hw_fcs_strip ? "" : "not ")); #ifdef HAVE_IBV_WQ_FLAG_RX_END_PADDING priv->hw_padding = !!device_attr_ex.rx_pad_end_addr_align; #endif - DEBUG("hardware RX end alignment padding is %ssupported", - (priv->hw_padding ? "" : "not ")); - - priv_get_num_vfs(priv, &num_vfs); - priv->sriov = (num_vfs || sriov); + DRV_LOG(DEBUG, + "hardware Rx end alignment padding is %ssupported", + (priv->hw_padding ? "" : "not ")); priv->tso = ((priv->tso) && (device_attr_ex.tso_caps.max_tso > 0) && (device_attr_ex.tso_caps.supported_qpts & @@ -846,18 +975,21 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) priv->max_tso_payload_sz = device_attr_ex.tso_caps.max_tso; if (priv->mps && !mps) { - ERROR("multi-packet send not supported on this device" - " (" MLX5_TXQ_MPW_EN ")"); + DRV_LOG(ERR, + "multi-packet send not supported on this device" + " (" MLX5_TXQ_MPW_EN ")"); err = ENOTSUP; goto port_error; } else if (priv->mps && priv->tso) { - WARN("multi-packet send not supported in conjunction " - "with TSO. MPS disabled"); + DRV_LOG(WARNING, + "multi-packet send not supported in conjunction" + " with TSO. MPS disabled"); priv->mps = 0; } - INFO("%sMPS is %s", - priv->mps == MLX5_MPW_ENHANCED ? "Enhanced " : "", - priv->mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); + DRV_LOG(INFO, "%s MPS is %s", + priv->mps == MLX5_MPW_ENHANCED ? "enhanced " : "", + priv->mps != MLX5_MPW_DISABLED ? "enabled" : + "disabled"); /* Set default values for Enhanced MPW, a.k.a MPWv2. */ if (priv->mps == MLX5_MPW_ENHANCED) { if (args.txqs_inline == MLX5_ARG_UNSET) @@ -870,59 +1002,71 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) MLX5_WQE_SIZE; } if (priv->cqe_comp && !cqe_comp) { - WARN("Rx CQE compression isn't supported"); + DRV_LOG(WARNING, "Rx CQE compression isn't supported"); priv->cqe_comp = 0; } - /* Configure the first MAC address by default. */ - if (priv_get_mac(priv, &mac.addr_bytes)) { - ERROR("cannot get MAC address, is mlx5_en loaded?" - " (errno: %s)", strerror(errno)); - err = ENODEV; - goto port_error; - } - INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", - priv->port, - mac.addr_bytes[0], mac.addr_bytes[1], - mac.addr_bytes[2], mac.addr_bytes[3], - mac.addr_bytes[4], mac.addr_bytes[5]); -#ifndef NDEBUG - { - char ifname[IF_NAMESIZE]; - - if (priv_get_ifname(priv, &ifname) == 0) - DEBUG("port %u ifname is \"%s\"", - priv->port, ifname); - else - DEBUG("port %u ifname is unknown", priv->port); - } -#endif - /* Get actual MTU if possible. */ - priv_get_mtu(priv, &priv->mtu); - DEBUG("port %u MTU is %u", priv->port, priv->mtu); - eth_dev = rte_eth_dev_allocate(name); if (eth_dev == NULL) { - ERROR("can not allocate rte ethdev"); + DRV_LOG(ERR, "can not allocate rte ethdev"); err = ENOMEM; goto port_error; } eth_dev->data->dev_private = priv; + priv->dev_data = eth_dev->data; eth_dev->data->mac_addrs = priv->mac; eth_dev->device = &pci_dev->device; rte_eth_copy_pci_info(eth_dev, pci_dev); eth_dev->device->driver = &mlx5_driver.driver; + err = mlx5_uar_init_primary(eth_dev); + if (err) { + err = rte_errno; + goto port_error; + } + /* Configure the first MAC address by default. */ + if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) { + DRV_LOG(ERR, + "port %u cannot get MAC address, is mlx5_en" + " loaded? (errno: %s)", + eth_dev->data->port_id, strerror(errno)); + err = ENODEV; + goto port_error; + } + DRV_LOG(INFO, + "port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", + eth_dev->data->port_id, + mac.addr_bytes[0], mac.addr_bytes[1], + mac.addr_bytes[2], mac.addr_bytes[3], + mac.addr_bytes[4], mac.addr_bytes[5]); +#ifndef NDEBUG + { + char ifname[IF_NAMESIZE]; + + if (mlx5_get_ifname(eth_dev, &ifname) == 0) + DRV_LOG(DEBUG, "port %u ifname is \"%s\"", + eth_dev->data->port_id, ifname); + else + DRV_LOG(DEBUG, "port %u ifname is unknown", + eth_dev->data->port_id); + } +#endif + /* Get actual MTU if possible. */ + err = mlx5_get_mtu(eth_dev, &priv->mtu); + if (err) { + err = rte_errno; + goto port_error; + } + DRV_LOG(DEBUG, "port %u MTU is %u", eth_dev->data->port_id, + priv->mtu); /* * Initialize burst functions to prevent crashes before link-up. */ eth_dev->rx_pkt_burst = removed_rx_burst; eth_dev->tx_pkt_burst = removed_tx_burst; - priv->dev = eth_dev; eth_dev->dev_ops = &mlx5_dev_ops; /* Register MAC address. */ claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0)); TAILQ_INIT(&priv->flows); TAILQ_INIT(&priv->ctrl_flows); - /* Hint libmlx5 to use PMD allocator for data plane resources */ struct mlx5dv_ctx_allocators alctr = { .alloc = &mlx5_alloc_verbs_buf, @@ -931,12 +1075,17 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) }; mlx5dv_set_context_attr(ctx, MLX5DV_CTX_ATTR_BUF_ALLOCATORS, (void *)((uintptr_t)&alctr)); - /* Bring Ethernet device up. */ - DEBUG("forcing Ethernet interface up"); - priv_set_flags(priv, ~IFF_UP, IFF_UP); + DRV_LOG(DEBUG, "port %u forcing Ethernet interface up", + eth_dev->data->port_id); + mlx5_set_link_up(eth_dev); + /* + * Even though the interrupt handler is not installed yet, + * interrupts will still trigger on the asyn_fd from + * Verbs context returned by ibv_open_device(). + */ + mlx5_link_update(eth_dev, 0); continue; - port_error: if (priv) rte_free(priv); @@ -944,29 +1093,31 @@ port_error: claim_zero(ibv_dealloc_pd(pd)); if (ctx) claim_zero(ibv_close_device(ctx)); + if (eth_dev && rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_eth_dev_release_port(eth_dev); break; } - /* * XXX if something went wrong in the loop above, there is a resource * leak (ctx, pd, priv, dpdk ethdev) but we can do nothing about it as * long as the dpdk does not provide a way to deallocate a ethdev and a * way to enumerate the registered ethdevs to free the previous ones. */ - /* no port found, complain */ if (!mlx5_dev[idx].ports) { - err = ENODEV; - goto error; + rte_errno = ENODEV; + err = rte_errno; } - error: if (attr_ctx) claim_zero(ibv_close_device(attr_ctx)); if (list) ibv_free_device_list(list); - assert(err >= 0); - return -err; + if (err) { + rte_errno = err; + return -rte_errno; + } + return 0; } static const struct rte_pci_id mlx5_pci_id_map[] = { @@ -1042,3 +1193,11 @@ rte_mlx5_pmd_init(void) RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__); RTE_PMD_REGISTER_PCI_TABLE(net_mlx5, mlx5_pci_id_map); RTE_PMD_REGISTER_KMOD_DEP(net_mlx5, "* ib_uverbs & mlx5_core & mlx5_ib"); + +/** Initialize driver log type. */ +RTE_INIT(vdev_netvsc_init_log) +{ + mlx5_logtype = rte_log_register("pmd.net.mlx5"); + if (mlx5_logtype >= 0) + rte_log_set_level(mlx5_logtype, RTE_LOG_NOTICE); +} diff --git a/dpdk/drivers/net/mlx5/mlx5.h b/dpdk/drivers/net/mlx5/mlx5.h index d49595bc..5e6027b8 100644 --- a/dpdk/drivers/net/mlx5/mlx5.h +++ b/dpdk/drivers/net/mlx5/mlx5.h @@ -90,8 +90,26 @@ struct mlx5_xstats_ctrl { /* Flow list . */ TAILQ_HEAD(mlx5_flows, rte_flow); +/** + * Type of objet being allocated. + */ +enum mlx5_verbs_alloc_type { + MLX5_VERBS_ALLOC_TYPE_NONE, + MLX5_VERBS_ALLOC_TYPE_TX_QUEUE, + MLX5_VERBS_ALLOC_TYPE_RX_QUEUE, +}; + +/** + * Verbs allocator needs a context to know in the callback which kind of + * resources it is allocating. + */ +struct mlx5_verbs_alloc_ctx { + enum mlx5_verbs_alloc_type type; /* Kind of object being allocated. */ + const void *obj; /* Pointer to the DPDK object. */ +}; + struct priv { - struct rte_eth_dev *dev; /* Ethernet device of master process. */ + struct rte_eth_dev_data *dev_data; /* Pointer to device data. */ struct ibv_context *ctx; /* Verbs context. */ struct ibv_device_attr_ex device_attr; /* Device properties. */ struct ibv_pd *pd; /* Protection Domain. */ @@ -107,11 +125,9 @@ struct priv { unsigned int hw_vlan_strip:1; /* VLAN stripping is supported. */ unsigned int hw_fcs_strip:1; /* FCS stripping is supported. */ unsigned int hw_padding:1; /* End alignment padding is supported. */ - unsigned int sriov:1; /* This is a VF or PF with VF devices. */ unsigned int mps:2; /* Multi-packet send mode (0: disabled). */ unsigned int mpw_hdr_dseg:1; /* Enable DSEGs in the title WQEBB. */ unsigned int cqe_comp:1; /* Whether CQE compression is enabled. */ - unsigned int pending_alarm:1; /* An alarm is pending. */ unsigned int tso:1; /* Whether TSO is supported. */ unsigned int tunnel_en:1; unsigned int isolated:1; /* Whether isolated mode is enabled. */ @@ -146,51 +162,16 @@ struct priv { LIST_HEAD(ind_tables, mlx5_ind_table_ibv) ind_tbls; uint32_t link_speed_capa; /* Link speed capabilities. */ struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */ - rte_spinlock_t lock; /* Lock for control functions. */ + rte_spinlock_t mr_lock; /* MR Lock. */ int primary_socket; /* Unix socket for primary process. */ + void *uar_base; /* Reserved address space for UAR mapping */ struct rte_intr_handle intr_handle_socket; /* Interrupt handler. */ + struct mlx5_verbs_alloc_ctx verbs_alloc_ctx; + /* Context for Verbs allocator. */ }; -/** - * Lock private structure to protect it from concurrent access in the - * control path. - * - * @param priv - * Pointer to private structure. - */ -static inline void -priv_lock(struct priv *priv) -{ - rte_spinlock_lock(&priv->lock); -} - -/** - * Try to lock private structure to protect it from concurrent access in the - * control path. - * - * @param priv - * Pointer to private structure. - * - * @return - * 1 if the lock is successfully taken; 0 otherwise. - */ -static inline int -priv_trylock(struct priv *priv) -{ - return rte_spinlock_trylock(&priv->lock); -} - -/** - * Unlock private structure. - * - * @param priv - * Pointer to private structure. - */ -static inline void -priv_unlock(struct priv *priv) -{ - rte_spinlock_unlock(&priv->lock); -} +#define PORT_ID(priv) ((priv)->dev_data->port_id) +#define ETH_DEV(priv) (&rte_eth_devices[PORT_ID(priv)]) /* mlx5.c */ @@ -198,130 +179,138 @@ int mlx5_getenv_int(const char *); /* mlx5_ethdev.c */ -struct priv *mlx5_get_priv(struct rte_eth_dev *dev); -int mlx5_is_secondary(void); -int priv_get_ifname(const struct priv *, char (*)[IF_NAMESIZE]); -int priv_ifreq(const struct priv *, int req, struct ifreq *); -int priv_is_ib_cntr(const char *); -int priv_get_cntr_sysfs(struct priv *, const char *, uint64_t *); -int priv_get_num_vfs(struct priv *, uint16_t *); -int priv_get_mtu(struct priv *, uint16_t *); -int priv_set_flags(struct priv *, unsigned int, unsigned int); -int mlx5_dev_configure(struct rte_eth_dev *); -void mlx5_dev_infos_get(struct rte_eth_dev *, struct rte_eth_dev_info *); +int mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]); +int mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr); +int mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu); +int mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, + unsigned int flags); +int mlx5_dev_configure(struct rte_eth_dev *dev); +void mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info); const uint32_t *mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev); -int priv_link_update(struct priv *, int); -int priv_force_link_status_change(struct priv *, int); -int mlx5_link_update(struct rte_eth_dev *, int); -int mlx5_dev_set_mtu(struct rte_eth_dev *, uint16_t); -int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *, struct rte_eth_fc_conf *); -int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *, struct rte_eth_fc_conf *); -int mlx5_ibv_device_to_pci_addr(const struct ibv_device *, - struct rte_pci_addr *); -void mlx5_dev_link_status_handler(void *); -void mlx5_dev_interrupt_handler(void *); -void priv_dev_interrupt_handler_uninstall(struct priv *, struct rte_eth_dev *); -void priv_dev_interrupt_handler_install(struct priv *, struct rte_eth_dev *); +int mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete); +int mlx5_force_link_status_change(struct rte_eth_dev *dev, int status); +int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu); +int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf); +int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, + struct rte_eth_fc_conf *fc_conf); +int mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, + struct rte_pci_addr *pci_addr); +void mlx5_dev_link_status_handler(void *arg); +void mlx5_dev_interrupt_handler(void *arg); +void mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev); +void mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev); int mlx5_set_link_down(struct rte_eth_dev *dev); int mlx5_set_link_up(struct rte_eth_dev *dev); -void priv_dev_select_tx_function(struct priv *priv, struct rte_eth_dev *dev); -void priv_dev_select_rx_function(struct priv *priv, struct rte_eth_dev *dev); +eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev); +eth_rx_burst_t mlx5_select_rx_function(struct rte_eth_dev *dev); /* mlx5_mac.c */ -int priv_get_mac(struct priv *, uint8_t (*)[ETHER_ADDR_LEN]); -void mlx5_mac_addr_remove(struct rte_eth_dev *, uint32_t); -int mlx5_mac_addr_add(struct rte_eth_dev *, struct ether_addr *, uint32_t, - uint32_t); -void mlx5_mac_addr_set(struct rte_eth_dev *, struct ether_addr *); +int mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN]); +void mlx5_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index); +int mlx5_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac, + uint32_t index, uint32_t vmdq); +void mlx5_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr); /* mlx5_rss.c */ -int mlx5_rss_hash_update(struct rte_eth_dev *, struct rte_eth_rss_conf *); -int mlx5_rss_hash_conf_get(struct rte_eth_dev *, struct rte_eth_rss_conf *); -int priv_rss_reta_index_resize(struct priv *, unsigned int); -int mlx5_dev_rss_reta_query(struct rte_eth_dev *, - struct rte_eth_rss_reta_entry64 *, uint16_t); -int mlx5_dev_rss_reta_update(struct rte_eth_dev *, - struct rte_eth_rss_reta_entry64 *, uint16_t); +int mlx5_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf); +int mlx5_rss_hash_conf_get(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf); +int mlx5_rss_reta_index_resize(struct rte_eth_dev *dev, unsigned int reta_size); +int mlx5_dev_rss_reta_query(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size); +int mlx5_dev_rss_reta_update(struct rte_eth_dev *dev, + struct rte_eth_rss_reta_entry64 *reta_conf, + uint16_t reta_size); /* mlx5_rxmode.c */ -void mlx5_promiscuous_enable(struct rte_eth_dev *); -void mlx5_promiscuous_disable(struct rte_eth_dev *); -void mlx5_allmulticast_enable(struct rte_eth_dev *); -void mlx5_allmulticast_disable(struct rte_eth_dev *); +void mlx5_promiscuous_enable(struct rte_eth_dev *dev); +void mlx5_promiscuous_disable(struct rte_eth_dev *dev); +void mlx5_allmulticast_enable(struct rte_eth_dev *dev); +void mlx5_allmulticast_disable(struct rte_eth_dev *dev); /* mlx5_stats.c */ -void priv_xstats_init(struct priv *); -int mlx5_stats_get(struct rte_eth_dev *, struct rte_eth_stats *); -void mlx5_stats_reset(struct rte_eth_dev *); -int mlx5_xstats_get(struct rte_eth_dev *, - struct rte_eth_xstat *, unsigned int); -void mlx5_xstats_reset(struct rte_eth_dev *); -int mlx5_xstats_get_names(struct rte_eth_dev *, - struct rte_eth_xstat_name *, unsigned int); +void mlx5_xstats_init(struct rte_eth_dev *dev); +int mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats); +void mlx5_stats_reset(struct rte_eth_dev *dev); +int mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats, + unsigned int n); +void mlx5_xstats_reset(struct rte_eth_dev *dev); +int mlx5_xstats_get_names(struct rte_eth_dev *dev __rte_unused, + struct rte_eth_xstat_name *xstats_names, + unsigned int n); /* mlx5_vlan.c */ -int mlx5_vlan_filter_set(struct rte_eth_dev *, uint16_t, int); -int mlx5_vlan_offload_set(struct rte_eth_dev *, int); -void mlx5_vlan_strip_queue_set(struct rte_eth_dev *, uint16_t, int); +int mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on); +void mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on); +int mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask); /* mlx5_trigger.c */ -int mlx5_dev_start(struct rte_eth_dev *); -void mlx5_dev_stop(struct rte_eth_dev *); -int priv_dev_traffic_enable(struct priv *, struct rte_eth_dev *); -int priv_dev_traffic_disable(struct priv *, struct rte_eth_dev *); -int priv_dev_traffic_restart(struct priv *, struct rte_eth_dev *); -int mlx5_traffic_restart(struct rte_eth_dev *); +int mlx5_dev_start(struct rte_eth_dev *dev); +void mlx5_dev_stop(struct rte_eth_dev *dev); +int mlx5_traffic_enable(struct rte_eth_dev *dev); +void mlx5_traffic_disable(struct rte_eth_dev *dev); +int mlx5_traffic_restart(struct rte_eth_dev *dev); /* mlx5_flow.c */ -int mlx5_dev_filter_ctrl(struct rte_eth_dev *, enum rte_filter_type, - enum rte_filter_op, void *); -int mlx5_flow_validate(struct rte_eth_dev *, const struct rte_flow_attr *, - const struct rte_flow_item [], - const struct rte_flow_action [], - struct rte_flow_error *); -struct rte_flow *mlx5_flow_create(struct rte_eth_dev *, - const struct rte_flow_attr *, - const struct rte_flow_item [], - const struct rte_flow_action [], - struct rte_flow_error *); -int mlx5_flow_destroy(struct rte_eth_dev *, struct rte_flow *, - struct rte_flow_error *); -void priv_flow_flush(struct priv *, struct mlx5_flows *); -int mlx5_flow_flush(struct rte_eth_dev *, struct rte_flow_error *); -int mlx5_flow_query(struct rte_eth_dev *, struct rte_flow *, - enum rte_flow_action_type, void *, - struct rte_flow_error *); -int mlx5_flow_isolate(struct rte_eth_dev *, int, struct rte_flow_error *); -int priv_flow_start(struct priv *, struct mlx5_flows *); -void priv_flow_stop(struct priv *, struct mlx5_flows *); -int priv_flow_verify(struct priv *); -int mlx5_ctrl_flow_vlan(struct rte_eth_dev *, struct rte_flow_item_eth *, - struct rte_flow_item_eth *, struct rte_flow_item_vlan *, - struct rte_flow_item_vlan *); -int mlx5_ctrl_flow(struct rte_eth_dev *, struct rte_flow_item_eth *, - struct rte_flow_item_eth *); -int priv_flow_create_drop_queue(struct priv *); -void priv_flow_delete_drop_queue(struct priv *); +int mlx5_flow_validate(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); +struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error); +int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, + struct rte_flow_error *error); +void mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list); +int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error); +int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow, + enum rte_flow_action_type action, void *data, + struct rte_flow_error *error); +int mlx5_flow_isolate(struct rte_eth_dev *dev, int enable, + struct rte_flow_error *error); +int mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, + enum rte_filter_type filter_type, + enum rte_filter_op filter_op, + void *arg); +int mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list); +void mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list); +int mlx5_flow_verify(struct rte_eth_dev *dev); +int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, + struct rte_flow_item_eth *eth_spec, + struct rte_flow_item_eth *eth_mask, + struct rte_flow_item_vlan *vlan_spec, + struct rte_flow_item_vlan *vlan_mask); +int mlx5_ctrl_flow(struct rte_eth_dev *dev, + struct rte_flow_item_eth *eth_spec, + struct rte_flow_item_eth *eth_mask); +int mlx5_flow_create_drop_queue(struct rte_eth_dev *dev); +void mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev); /* mlx5_socket.c */ -int priv_socket_init(struct priv *priv); -int priv_socket_uninit(struct priv *priv); -void priv_socket_handle(struct priv *priv); -int priv_socket_connect(struct priv *priv); +int mlx5_socket_init(struct rte_eth_dev *priv); +void mlx5_socket_uninit(struct rte_eth_dev *priv); +void mlx5_socket_handle(struct rte_eth_dev *priv); +int mlx5_socket_connect(struct rte_eth_dev *priv); /* mlx5_mr.c */ -struct mlx5_mr *priv_mr_new(struct priv *, struct rte_mempool *); -struct mlx5_mr *priv_mr_get(struct priv *, struct rte_mempool *); -int priv_mr_release(struct priv *, struct mlx5_mr *); -int priv_mr_verify(struct priv *); +struct mlx5_mr *mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp); +struct mlx5_mr *mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp); +int mlx5_mr_release(struct mlx5_mr *mr); +int mlx5_mr_verify(struct rte_eth_dev *dev); #endif /* RTE_PMD_MLX5_H_ */ diff --git a/dpdk/drivers/net/mlx5/mlx5_defs.h b/dpdk/drivers/net/mlx5/mlx5_defs.h index 24caf7e7..9c64bb33 100644 --- a/dpdk/drivers/net/mlx5/mlx5_defs.h +++ b/dpdk/drivers/net/mlx5/mlx5_defs.h @@ -92,10 +92,11 @@ #define MLX5_VPMD_MIN_TXQS 4 /* Threshold of buffer replenishment for vectorized Rx. */ -#define MLX5_VPMD_RXQ_RPLNSH_THRESH 64U +#define MLX5_VPMD_RXQ_RPLNSH_THRESH(n) \ + (RTE_MIN(MLX5_VPMD_RX_MAX_BURST, (unsigned int)(n) >> 2)) /* Maximum size of burst for vectorized Rx. */ -#define MLX5_VPMD_RX_MAX_BURST MLX5_VPMD_RXQ_RPLNSH_THRESH +#define MLX5_VPMD_RX_MAX_BURST 64U /* * Maximum size of burst for vectorized Tx. This is related to the maximum size @@ -110,7 +111,22 @@ /* Supported RSS */ #define MLX5_RSS_HF_MASK (~(ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP)) -/* Maximum number of attempts to query link status before giving up. */ -#define MLX5_MAX_LINK_QUERY_ATTEMPTS 5 +/* Timeout in seconds to get a valid link status. */ +#define MLX5_LINK_STATUS_TIMEOUT 10 + +/* Reserved address space for UAR mapping. */ +#define MLX5_UAR_SIZE (1ULL << 32) + +/* Offset of reserved UAR address space to hugepage memory. Offset is used here + * to minimize possibility of address next to hugepage being used by other code + * in either primary or secondary process, failing to map TX UAR would make TX + * packets invisible to HW. + */ +#define MLX5_UAR_OFFSET (1ULL << 32) + +/* Definition of static_assert found in /usr/include/assert.h */ +#ifndef HAVE_STATIC_ASSERT +#define static_assert _Static_assert +#endif #endif /* RTE_PMD_MLX5_DEFS_H_ */ diff --git a/dpdk/drivers/net/mlx5/mlx5_ethdev.c b/dpdk/drivers/net/mlx5/mlx5_ethdev.c index ffe1cdd6..e441483a 100644 --- a/dpdk/drivers/net/mlx5/mlx5_ethdev.c +++ b/dpdk/drivers/net/mlx5/mlx5_ethdev.c @@ -35,6 +35,7 @@ #include #include +#include #include #include #include @@ -45,14 +46,13 @@ #include #include #include -#include #include #include #include -#include #include #include #include +#include #include #include @@ -60,13 +60,38 @@ #include #include #include -#include #include #include "mlx5.h" #include "mlx5_rxtx.h" #include "mlx5_utils.h" +/* Supported speed values found in /usr/include/linux/ethtool.h */ +#ifndef HAVE_SUPPORTED_40000baseKR4_Full +#define SUPPORTED_40000baseKR4_Full (1 << 23) +#endif +#ifndef HAVE_SUPPORTED_40000baseCR4_Full +#define SUPPORTED_40000baseCR4_Full (1 << 24) +#endif +#ifndef HAVE_SUPPORTED_40000baseSR4_Full +#define SUPPORTED_40000baseSR4_Full (1 << 25) +#endif +#ifndef HAVE_SUPPORTED_40000baseLR4_Full +#define SUPPORTED_40000baseLR4_Full (1 << 26) +#endif +#ifndef HAVE_SUPPORTED_56000baseKR4_Full +#define SUPPORTED_56000baseKR4_Full (1 << 27) +#endif +#ifndef HAVE_SUPPORTED_56000baseCR4_Full +#define SUPPORTED_56000baseCR4_Full (1 << 28) +#endif +#ifndef HAVE_SUPPORTED_56000baseSR4_Full +#define SUPPORTED_56000baseSR4_Full (1 << 29) +#endif +#ifndef HAVE_SUPPORTED_56000baseLR4_Full +#define SUPPORTED_56000baseLR4_Full (1 << 30) +#endif + /* Add defines in case the running kernel is not the same as user headers. */ #ifndef ETHTOOL_GLINKSETTINGS struct ethtool_link_settings { @@ -118,35 +143,21 @@ struct ethtool_link_settings { #define ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT 39 #endif -/** - * Return private structure associated with an Ethernet device. - * - * @param dev - * Pointer to Ethernet device structure. - * - * @return - * Pointer to private structure. - */ -struct priv * -mlx5_get_priv(struct rte_eth_dev *dev) -{ - return dev->data->dev_private; -} - /** * Get interface name from private structure. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * @param[out] ifname * Interface name output buffer. * * @return - * 0 on success, -1 on failure and errno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) +mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]) { + struct priv *priv = dev->data->dev_private; DIR *dir; struct dirent *dent; unsigned int dev_type = 0; @@ -157,8 +168,10 @@ priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]) MKSTR(path, "%s/device/net", priv->ibdev_path); dir = opendir(path); - if (dir == NULL) - return -1; + if (dir == NULL) { + rte_errno = errno; + return -rte_errno; + } } while ((dent = readdir(dir)) != NULL) { char *name = dent->d_name; @@ -208,355 +221,131 @@ try_dev_id: snprintf(match, sizeof(match), "%s", name); } closedir(dir); - if (match[0] == '\0') - return -1; + if (match[0] == '\0') { + rte_errno = ENOENT; + return -rte_errno; + } strncpy(*ifname, match, sizeof(*ifname)); return 0; } -/** - * Check if the counter is located on ib counters file. - * - * @param[in] cntr - * Counter name. - * - * @return - * 1 if counter is located on ib counters file , 0 otherwise. - */ -int -priv_is_ib_cntr(const char *cntr) -{ - if (!strcmp(cntr, "out_of_buffer")) - return 1; - return 0; -} - -/** - * Read from sysfs entry. - * - * @param[in] priv - * Pointer to private structure. - * @param[in] entry - * Entry name relative to sysfs path. - * @param[out] buf - * Data output buffer. - * @param size - * Buffer size. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_sysfs_read(const struct priv *priv, const char *entry, - char *buf, size_t size) -{ - char ifname[IF_NAMESIZE]; - FILE *file; - int ret; - int err; - - if (priv_get_ifname(priv, &ifname)) - return -1; - - if (priv_is_ib_cntr(entry)) { - MKSTR(path, "%s/ports/1/hw_counters/%s", - priv->ibdev_path, entry); - file = fopen(path, "rb"); - } else { - MKSTR(path, "%s/device/net/%s/%s", - priv->ibdev_path, ifname, entry); - file = fopen(path, "rb"); - } - if (file == NULL) - return -1; - ret = fread(buf, 1, size, file); - err = errno; - if (((size_t)ret < size) && (ferror(file))) - ret = -1; - else - ret = size; - fclose(file); - errno = err; - return ret; -} - -/** - * Write to sysfs entry. - * - * @param[in] priv - * Pointer to private structure. - * @param[in] entry - * Entry name relative to sysfs path. - * @param[in] buf - * Data buffer. - * @param size - * Buffer size. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_sysfs_write(const struct priv *priv, const char *entry, - char *buf, size_t size) -{ - char ifname[IF_NAMESIZE]; - FILE *file; - int ret; - int err; - - if (priv_get_ifname(priv, &ifname)) - return -1; - - MKSTR(path, "%s/device/net/%s/%s", priv->ibdev_path, ifname, entry); - - file = fopen(path, "wb"); - if (file == NULL) - return -1; - ret = fwrite(buf, 1, size, file); - err = errno; - if (((size_t)ret < size) || (ferror(file))) - ret = -1; - else - ret = size; - fclose(file); - errno = err; - return ret; -} - -/** - * Get unsigned long sysfs property. - * - * @param priv - * Pointer to private structure. - * @param[in] name - * Entry name relative to sysfs path. - * @param[out] value - * Value output buffer. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_get_sysfs_ulong(struct priv *priv, const char *name, unsigned long *value) -{ - int ret; - unsigned long value_ret; - char value_str[32]; - - ret = priv_sysfs_read(priv, name, value_str, (sizeof(value_str) - 1)); - if (ret == -1) { - DEBUG("cannot read %s value from sysfs: %s", - name, strerror(errno)); - return -1; - } - value_str[ret] = '\0'; - errno = 0; - value_ret = strtoul(value_str, NULL, 0); - if (errno) { - DEBUG("invalid %s value `%s': %s", name, value_str, - strerror(errno)); - return -1; - } - *value = value_ret; - return 0; -} - -/** - * Set unsigned long sysfs property. - * - * @param priv - * Pointer to private structure. - * @param[in] name - * Entry name relative to sysfs path. - * @param value - * Value to set. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -static int -priv_set_sysfs_ulong(struct priv *priv, const char *name, unsigned long value) -{ - int ret; - MKSTR(value_str, "%lu", value); - - ret = priv_sysfs_write(priv, name, value_str, (sizeof(value_str) - 1)); - if (ret == -1) { - DEBUG("cannot write %s `%s' (%lu) to sysfs: %s", - name, value_str, value, strerror(errno)); - return -1; - } - return 0; -} - /** * Perform ifreq ioctl() on associated Ethernet device. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * @param req * Request number to pass to ioctl(). * @param[out] ifr * Interface request structure output buffer. * * @return - * 0 on success, -1 on failure and errno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr) +mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr) { int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP); - int ret = -1; + int ret = 0; - if (sock == -1) - return ret; - if (priv_get_ifname(priv, &ifr->ifr_name) == 0) - ret = ioctl(sock, req, ifr); + if (sock == -1) { + rte_errno = errno; + return -rte_errno; + } + ret = mlx5_get_ifname(dev, &ifr->ifr_name); + if (ret) + goto error; + ret = ioctl(sock, req, ifr); + if (ret == -1) { + rte_errno = errno; + goto error; + } close(sock); - return ret; -} - -/** - * Return the number of active VFs for the current device. - * - * @param[in] priv - * Pointer to private structure. - * @param[out] num_vfs - * Number of active VFs. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -int -priv_get_num_vfs(struct priv *priv, uint16_t *num_vfs) -{ - /* The sysfs entry name depends on the operating system. */ - const char **name = (const char *[]){ - "device/sriov_numvfs", - "device/mlx5_num_vfs", - NULL, - }; - int ret; - - do { - unsigned long ulong_num_vfs; - - ret = priv_get_sysfs_ulong(priv, *name, &ulong_num_vfs); - if (!ret) - *num_vfs = ulong_num_vfs; - } while (*(++name) && ret); - return ret; + return 0; +error: + close(sock); + return -rte_errno; } /** * Get device MTU. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param[out] mtu * MTU value output buffer. * * @return - * 0 on success, -1 on failure and errno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_get_mtu(struct priv *priv, uint16_t *mtu) +mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu) { - unsigned long ulong_mtu; + struct ifreq request; + int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request); - if (priv_get_sysfs_ulong(priv, "mtu", &ulong_mtu) == -1) - return -1; - *mtu = ulong_mtu; - return 0; -} - -/** - * Read device counter from sysfs. - * - * @param priv - * Pointer to private structure. - * @param name - * Counter name. - * @param[out] cntr - * Counter output buffer. - * - * @return - * 0 on success, -1 on failure and errno is set. - */ -int -priv_get_cntr_sysfs(struct priv *priv, const char *name, uint64_t *cntr) -{ - unsigned long ulong_ctr; - - if (priv_get_sysfs_ulong(priv, name, &ulong_ctr) == -1) - return -1; - *cntr = ulong_ctr; + if (ret) + return ret; + *mtu = request.ifr_mtu; return 0; } /** * Set device MTU. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param mtu * MTU value to set. * * @return - * 0 on success, -1 on failure and errno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_set_mtu(struct priv *priv, uint16_t mtu) +mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { - uint16_t new_mtu; + struct ifreq request = { .ifr_mtu = mtu, }; - if (priv_set_sysfs_ulong(priv, "mtu", mtu) || - priv_get_mtu(priv, &new_mtu)) - return -1; - if (new_mtu == mtu) - return 0; - errno = EINVAL; - return -1; + return mlx5_ifreq(dev, SIOCSIFMTU, &request); } /** * Set device flags. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param keep * Bitmask for flags that must remain untouched. * @param flags * Bitmask for flags to modify. * * @return - * 0 on success, -1 on failure and errno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags) +mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags) { - unsigned long tmp; + struct ifreq request; + int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request); - if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1) - return -1; - tmp &= keep; - tmp |= (flags & (~keep)); - return priv_set_sysfs_ulong(priv, "flags", tmp); + if (ret) + return ret; + request.ifr_flags &= keep; + request.ifr_flags |= flags & ~keep; + return mlx5_ifreq(dev, SIOCSIFFLAGS, &request); } /** - * Ethernet device configuration. - * - * Prepare the driver for a given number of TX and RX queues. + * DPDK callback for Ethernet device configuration. * * @param dev * Pointer to Ethernet device structure. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ -static int -dev_configure(struct rte_eth_dev *dev) +int +mlx5_dev_configure(struct rte_eth_dev *dev) { struct priv *priv = dev->data->dev_private; unsigned int rxqs_n = dev->data->nb_rx_queues; @@ -566,19 +355,24 @@ dev_configure(struct rte_eth_dev *dev) unsigned int reta_idx_n; const uint8_t use_app_rss_key = !!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key; + int ret = 0; if (use_app_rss_key && (dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key_len != rss_hash_default_key_len)) { - /* MLX5 RSS only support 40bytes key. */ - return EINVAL; + DRV_LOG(ERR, "port %u RSS key len must be %zu Bytes long", + dev->data->port_id, rss_hash_default_key_len); + rte_errno = EINVAL; + return -rte_errno; } priv->rss_conf.rss_key = rte_realloc(priv->rss_conf.rss_key, rss_hash_default_key_len, 0); if (!priv->rss_conf.rss_key) { - ERROR("cannot allocate RSS hash key memory (%u)", rxqs_n); - return ENOMEM; + DRV_LOG(ERR, "port %u cannot allocate RSS hash key memory (%u)", + dev->data->port_id, rxqs_n); + rte_errno = ENOMEM; + return -rte_errno; } memcpy(priv->rss_conf.rss_key, use_app_rss_key ? @@ -590,18 +384,20 @@ dev_configure(struct rte_eth_dev *dev) priv->rxqs = (void *)dev->data->rx_queues; priv->txqs = (void *)dev->data->tx_queues; if (txqs_n != priv->txqs_n) { - INFO("%p: TX queues number update: %u -> %u", - (void *)dev, priv->txqs_n, txqs_n); + DRV_LOG(INFO, "port %u Tx queues number update: %u -> %u", + dev->data->port_id, priv->txqs_n, txqs_n); priv->txqs_n = txqs_n; } if (rxqs_n > priv->ind_table_max_size) { - ERROR("cannot handle this many RX queues (%u)", rxqs_n); - return EINVAL; + DRV_LOG(ERR, "port %u cannot handle this many Rx queues (%u)", + dev->data->port_id, rxqs_n); + rte_errno = EINVAL; + return -rte_errno; } if (rxqs_n == priv->rxqs_n) return 0; - INFO("%p: RX queues number update: %u -> %u", - (void *)dev, priv->rxqs_n, rxqs_n); + DRV_LOG(INFO, "port %u Rx queues number update: %u -> %u", + dev->data->port_id, priv->rxqs_n, rxqs_n); priv->rxqs_n = rxqs_n; /* If the requested number of RX queues is not a power of two, use the * maximum indirection table size for better balancing. @@ -609,8 +405,9 @@ dev_configure(struct rte_eth_dev *dev) reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ? priv->ind_table_max_size : rxqs_n)); - if (priv_rss_reta_index_resize(priv, reta_idx_n)) - return ENOMEM; + ret = mlx5_rss_reta_index_resize(dev, reta_idx_n); + if (ret) + return ret; /* When the number of RX queues is not a power of two, the remaining * table entries are padded with reused WQs and hashes are not spread * uniformly. */ @@ -622,28 +419,6 @@ dev_configure(struct rte_eth_dev *dev) return 0; } -/** - * DPDK callback for Ethernet device configuration. - * - * @param dev - * Pointer to Ethernet device structure. - * - * @return - * 0 on success, negative errno value on failure. - */ -int -mlx5_dev_configure(struct rte_eth_dev *dev) -{ - struct priv *priv = dev->data->dev_private; - int ret; - - priv_lock(priv); - ret = dev_configure(dev); - assert(ret >= 0); - priv_unlock(priv); - return -ret; -} - /** * DPDK callback to get information about the device. * @@ -655,13 +430,11 @@ mlx5_dev_configure(struct rte_eth_dev *dev) void mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; unsigned int max; char ifname[IF_NAMESIZE]; info->pci_dev = RTE_ETH_DEV_TO_PCI(dev); - - priv_lock(priv); /* FIXME: we should ask the device for these values. */ info->min_rx_bufsize = 32; info->max_rx_pktlen = 65536; @@ -699,16 +472,24 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) info->tx_offload_capa |= (DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | DEV_TX_OFFLOAD_GRE_TNL_TSO); - if (priv_get_ifname(priv, &ifname) == 0) + if (mlx5_get_ifname(dev, &ifname) == 0) info->if_index = if_nametoindex(ifname); info->reta_size = priv->reta_idx_n ? priv->reta_idx_n : priv->ind_table_max_size; - info->hash_key_size = priv->rss_conf.rss_key_len; + info->hash_key_size = rss_hash_default_key_len; info->speed_capa = priv->link_speed_capa; info->flow_type_rss_offloads = ~MLX5_RSS_HF_MASK; - priv_unlock(priv); } +/** + * Get supported packet types. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * A pointer to the supported Packet types array. + */ const uint32_t * mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) { @@ -741,35 +522,41 @@ mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev) * * @param dev * Pointer to Ethernet device structure. - * @param wait_to_complete - * Wait for request completion (ignored). + * @param[out] link + * Storage for current link status. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) +mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, + struct rte_eth_link *link) { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; struct ethtool_cmd edata = { .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */ }; struct ifreq ifr; struct rte_eth_link dev_link; int link_speed = 0; + int ret; - /* priv_lock() is not taken to allow concurrent calls. */ - - (void)wait_to_complete; - if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { - WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); - return -1; + ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr); + if (ret) { + DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s", + dev->data->port_id, strerror(rte_errno)); + return ret; } memset(&dev_link, 0, sizeof(dev_link)); dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING)); ifr.ifr_data = (void *)&edata; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { - WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", - strerror(errno)); - return -1; + ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); + if (ret) { + DRV_LOG(WARNING, + "port %u ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", + dev->data->port_id, strerror(rte_errno)); + return ret; } link_speed = ethtool_cmd_speed(&edata); if (link_speed == -1) @@ -793,13 +580,13 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED); - if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { - /* Link status changed. */ - dev->data->dev_link = dev_link; - return 0; + if ((dev_link.link_speed && !dev_link.link_status) || + (!dev_link.link_speed && dev_link.link_status)) { + rte_errno = EAGAIN; + return -rte_errno; } - /* Link status is still the same. */ - return -1; + *link = dev_link; + return 0; } /** @@ -807,31 +594,41 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete) * * @param dev * Pointer to Ethernet device structure. - * @param wait_to_complete - * Wait for request completion (ignored). + * @param[out] link + * Storage for current link status. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) +mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, + struct rte_eth_link *link) + { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; struct ethtool_link_settings gcmd = { .cmd = ETHTOOL_GLINKSETTINGS }; struct ifreq ifr; struct rte_eth_link dev_link; uint64_t sc; + int ret; - (void)wait_to_complete; - if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { - WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); - return -1; + ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr); + if (ret) { + DRV_LOG(WARNING, "port %u ioctl(SIOCGIFFLAGS) failed: %s", + dev->data->port_id, strerror(rte_errno)); + return ret; } memset(&dev_link, 0, sizeof(dev_link)); dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && (ifr.ifr_flags & IFF_RUNNING)); ifr.ifr_data = (void *)&gcmd; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { - DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", - strerror(errno)); - return -1; + ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); + if (ret) { + DRV_LOG(DEBUG, + "port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)" + " failed: %s", + dev->data->port_id, strerror(rte_errno)); + return ret; } gcmd.link_mode_masks_nwords = -gcmd.link_mode_masks_nwords; @@ -842,10 +639,13 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) *ecmd = gcmd; ifr.ifr_data = (void *)ecmd; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { - DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s", - strerror(errno)); - return -1; + ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); + if (ret) { + DRV_LOG(DEBUG, + "port %u ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS)" + " failed: %s", + dev->data->port_id, strerror(rte_errno)); + return ret; } dev_link.link_speed = ecmd->speed; sc = ecmd->link_mode_masks[0] | @@ -889,121 +689,13 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete) ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED); - if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { - /* Link status changed. */ - dev->data->dev_link = dev_link; - return 0; + if ((dev_link.link_speed && !dev_link.link_status) || + (!dev_link.link_speed && dev_link.link_status)) { + rte_errno = EAGAIN; + return -rte_errno; } - /* Link status is still the same. */ - return -1; -} - -/** - * Enable receiving and transmitting traffic. - * - * @param priv - * Pointer to private structure. - */ -static void -priv_link_start(struct priv *priv) -{ - struct rte_eth_dev *dev = priv->dev; - int err; - - priv_dev_select_tx_function(priv, dev); - priv_dev_select_rx_function(priv, dev); - err = priv_dev_traffic_enable(priv, dev); - if (err) - ERROR("%p: error occurred while configuring control flows: %s", - (void *)priv, strerror(err)); - err = priv_flow_start(priv, &priv->flows); - if (err) - ERROR("%p: error occurred while configuring flows: %s", - (void *)priv, strerror(err)); -} - -/** - * Disable receiving and transmitting traffic. - * - * @param priv - * Pointer to private structure. - */ -static void -priv_link_stop(struct priv *priv) -{ - struct rte_eth_dev *dev = priv->dev; - - priv_flow_stop(priv, &priv->flows); - priv_dev_traffic_disable(priv, dev); - dev->rx_pkt_burst = removed_rx_burst; - dev->tx_pkt_burst = removed_tx_burst; -} - -/** - * Retrieve physical link information and update rx/tx_pkt_burst callbacks - * accordingly. - * - * @param priv - * Pointer to private structure. - * @param wait_to_complete - * Wait for request completion (ignored). - */ -int -priv_link_update(struct priv *priv, int wait_to_complete) -{ - struct rte_eth_dev *dev = priv->dev; - struct utsname utsname; - int ver[3]; - int ret; - struct rte_eth_link dev_link = dev->data->dev_link; - - if (uname(&utsname) == -1 || - sscanf(utsname.release, "%d.%d.%d", - &ver[0], &ver[1], &ver[2]) != 3 || - KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0)) - ret = mlx5_link_update_unlocked_gset(dev, wait_to_complete); - else - ret = mlx5_link_update_unlocked_gs(dev, wait_to_complete); - /* If lsc interrupt is disabled, should always be ready for traffic. */ - if (!dev->data->dev_conf.intr_conf.lsc) { - priv_link_start(priv); - return ret; - } - /* Re-select burst callbacks only if link status has been changed. */ - if (!ret && dev_link.link_status != dev->data->dev_link.link_status) { - if (dev->data->dev_link.link_status == ETH_LINK_UP) - priv_link_start(priv); - else - priv_link_stop(priv); - } - return ret; -} - -/** - * Querying the link status till it changes to the desired state. - * Number of query attempts is bounded by MLX5_MAX_LINK_QUERY_ATTEMPTS. - * - * @param priv - * Pointer to private structure. - * @param status - * Link desired status. - * - * @return - * 0 on success, negative errno value on failure. - */ -int -priv_force_link_status_change(struct priv *priv, int status) -{ - int try = 0; - - while (try < MLX5_MAX_LINK_QUERY_ATTEMPTS) { - priv_link_update(priv, 0); - if (priv->dev->data->dev_link.link_status == status) - return 0; - try++; - sleep(1); - } - return -EAGAIN; + *link = dev_link; + return 0; } /** @@ -1012,17 +704,42 @@ priv_force_link_status_change(struct priv *priv, int status) * @param dev * Pointer to Ethernet device structure. * @param wait_to_complete - * Wait for request completion (ignored). + * Wait for request completion. + * + * @return + * 0 if link status was not updated, positive if it was, a negative errno + * value otherwise and rte_errno is set. */ int mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) { - struct priv *priv = dev->data->dev_private; int ret; + struct rte_eth_link dev_link; + time_t start_time = time(NULL); - priv_lock(priv); - ret = priv_link_update(priv, wait_to_complete); - priv_unlock(priv); + do { + ret = mlx5_link_update_unlocked_gs(dev, &dev_link); + if (ret) + ret = mlx5_link_update_unlocked_gset(dev, &dev_link); + if (ret == 0) + break; + /* Handle wait to complete situation. */ + if (wait_to_complete && ret == -EAGAIN) { + if (abs((int)difftime(time(NULL), start_time)) < + MLX5_LINK_STATUS_TIMEOUT) { + usleep(0); + continue; + } else { + rte_errno = EBUSY; + return -rte_errno; + } + } else if (ret < 0) { + return ret; + } + } while (wait_to_complete); + ret = !!memcmp(&dev->data->dev_link, &dev_link, + sizeof(struct rte_eth_link)); + dev->data->dev_link = dev_link; return ret; } @@ -1035,39 +752,33 @@ mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete) * New MTU. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) { struct priv *priv = dev->data->dev_private; - uint16_t kern_mtu; - int ret = 0; + uint16_t kern_mtu = 0; + int ret; - priv_lock(priv); - ret = priv_get_mtu(priv, &kern_mtu); + ret = mlx5_get_mtu(dev, &kern_mtu); if (ret) - goto out; + return ret; /* Set kernel interface MTU first. */ - ret = priv_set_mtu(priv, mtu); + ret = mlx5_set_mtu(dev, mtu); if (ret) - goto out; - ret = priv_get_mtu(priv, &kern_mtu); + return ret; + ret = mlx5_get_mtu(dev, &kern_mtu); if (ret) - goto out; + return ret; if (kern_mtu == mtu) { priv->mtu = mtu; - DEBUG("adapter port %u MTU set to %u", priv->port, mtu); + DRV_LOG(DEBUG, "port %u adapter MTU set to %u", + dev->data->port_id, mtu); + return 0; } - priv_unlock(priv); - return 0; -out: - ret = errno; - WARN("cannot set port %u MTU to %u: %s", priv->port, mtu, - strerror(ret)); - priv_unlock(priv); - assert(ret >= 0); - return -ret; + rte_errno = EAGAIN; + return -rte_errno; } /** @@ -1079,12 +790,11 @@ out: * Flow control output buffer. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) { - struct priv *priv = dev->data->dev_private; struct ifreq ifr; struct ethtool_pauseparam ethpause = { .cmd = ETHTOOL_GPAUSEPARAM @@ -1092,15 +802,14 @@ mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) int ret; ifr.ifr_data = (void *)ðpause; - priv_lock(priv); - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { - ret = errno; - WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)" - " failed: %s", - strerror(ret)); - goto out; + ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); + if (ret) { + DRV_LOG(WARNING, + "port %u ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed:" + " %s", + dev->data->port_id, strerror(rte_errno)); + return ret; } - fc_conf->autoneg = ethpause.autoneg; if (ethpause.rx_pause && ethpause.tx_pause) fc_conf->mode = RTE_FC_FULL; @@ -1110,12 +819,7 @@ mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) fc_conf->mode = RTE_FC_TX_PAUSE; else fc_conf->mode = RTE_FC_NONE; - ret = 0; - -out: - priv_unlock(priv); - assert(ret >= 0); - return -ret; + return 0; } /** @@ -1127,12 +831,11 @@ out: * Flow control parameters. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) { - struct priv *priv = dev->data->dev_private; struct ifreq ifr; struct ethtool_pauseparam ethpause = { .cmd = ETHTOOL_SPAUSEPARAM @@ -1152,21 +855,15 @@ mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) ethpause.tx_pause = 1; else ethpause.tx_pause = 0; - - priv_lock(priv); - if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { - ret = errno; - WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" - " failed: %s", - strerror(ret)); - goto out; + ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); + if (ret) { + DRV_LOG(WARNING, + "port %u ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)" + " failed: %s", + dev->data->port_id, strerror(rte_errno)); + return ret; } - ret = 0; - -out: - priv_unlock(priv); - assert(ret >= 0); - return -ret; + return 0; } /** @@ -1178,7 +875,7 @@ out: * PCI bus address output buffer. * * @return - * 0 on success, -1 on failure and errno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, @@ -1189,8 +886,10 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, MKSTR(path, "%s/device/uevent", device->ibdev_path); file = fopen(path, "rb"); - if (file == NULL) - return -1; + if (file == NULL) { + rte_errno = errno; + return -rte_errno; + } while (fgets(line, sizeof(line), file) == line) { size_t len = strlen(line); int ret; @@ -1219,47 +918,11 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device, return 0; } -/** - * Update the link status. - * - * @param priv - * Pointer to private structure. - * - * @return - * Zero if the callback process can be called immediately. - */ -static int -priv_link_status_update(struct priv *priv) -{ - struct rte_eth_link *link = &priv->dev->data->dev_link; - - priv_link_update(priv, 0); - if (((link->link_speed == 0) && link->link_status) || - ((link->link_speed != 0) && !link->link_status)) { - /* - * Inconsistent status. Event likely occurred before the - * kernel netdevice exposes the new status. - */ - if (!priv->pending_alarm) { - priv->pending_alarm = 1; - rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US, - mlx5_dev_link_status_handler, - priv->dev); - } - return 1; - } else if (unlikely(priv->pending_alarm)) { - /* Link interrupt occurred while alarm is already scheduled. */ - priv->pending_alarm = 0; - rte_eal_alarm_cancel(mlx5_dev_link_status_handler, priv->dev); - } - return 0; -} - /** * Device status handler. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param events * Pointer to event flags holder. * @@ -1267,60 +930,36 @@ priv_link_status_update(struct priv *priv) * Events bitmap of callback process which can be called immediately. */ static uint32_t -priv_dev_status_handler(struct priv *priv) +mlx5_dev_status_handler(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct ibv_async_event event; uint32_t ret = 0; + if (mlx5_link_update(dev, 0) == -EAGAIN) { + usleep(0); + return 0; + } /* Read all message and acknowledge them. */ for (;;) { if (ibv_get_async_event(priv->ctx, &event)) break; if ((event.event_type == IBV_EVENT_PORT_ACTIVE || event.event_type == IBV_EVENT_PORT_ERR) && - (priv->dev->data->dev_conf.intr_conf.lsc == 1)) + (dev->data->dev_conf.intr_conf.lsc == 1)) ret |= (1 << RTE_ETH_EVENT_INTR_LSC); else if (event.event_type == IBV_EVENT_DEVICE_FATAL && - priv->dev->data->dev_conf.intr_conf.rmv == 1) + dev->data->dev_conf.intr_conf.rmv == 1) ret |= (1 << RTE_ETH_EVENT_INTR_RMV); else - DEBUG("event type %d on port %d not handled", - event.event_type, event.element.port_num); + DRV_LOG(DEBUG, + "port %u event type %d on not handled", + dev->data->port_id, event.event_type); ibv_ack_async_event(&event); } - if (ret & (1 << RTE_ETH_EVENT_INTR_LSC)) - if (priv_link_status_update(priv)) - ret &= ~(1 << RTE_ETH_EVENT_INTR_LSC); return ret; } -/** - * Handle delayed link status event. - * - * @param arg - * Registered argument. - */ -void -mlx5_dev_link_status_handler(void *arg) -{ - struct rte_eth_dev *dev = arg; - struct priv *priv = dev->data->dev_private; - int ret; - - while (!priv_trylock(priv)) { - /* Alarm is being canceled. */ - if (priv->pending_alarm == 0) - return; - rte_pause(); - } - priv->pending_alarm = 0; - ret = priv_link_status_update(priv); - priv_unlock(priv); - if (!ret) - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, - NULL); -} - /** * Handle interrupts from the NIC. * @@ -1333,12 +972,9 @@ void mlx5_dev_interrupt_handler(void *cb_arg) { struct rte_eth_dev *dev = cb_arg; - struct priv *priv = dev->data->dev_private; uint32_t events; - priv_lock(priv); - events = priv_dev_status_handler(priv); - priv_unlock(priv); + events = mlx5_dev_status_handler(dev); if (events & (1 << RTE_ETH_EVENT_INTR_LSC)) _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL, NULL); @@ -1357,24 +993,21 @@ static void mlx5_dev_handler_socket(void *cb_arg) { struct rte_eth_dev *dev = cb_arg; - struct priv *priv = dev->data->dev_private; - priv_lock(priv); - priv_socket_handle(priv); - priv_unlock(priv); + mlx5_socket_handle(dev); } /** * Uninstall interrupt handler. * - * @param priv - * Pointer to private structure. * @param dev - * Pointer to the rte_eth_dev structure. + * Pointer to Ethernet device. */ void -priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) +mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; + if (dev->data->dev_conf.intr_conf.lsc || dev->data->dev_conf.intr_conf.rmv) rte_intr_callback_unregister(&priv->intr_handle, @@ -1382,10 +1015,6 @@ priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) if (priv->primary_socket) rte_intr_callback_unregister(&priv->intr_handle_socket, mlx5_dev_handler_socket, dev); - if (priv->pending_alarm) { - priv->pending_alarm = 0; - rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev); - } priv->intr_handle.fd = 0; priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN; priv->intr_handle_socket.fd = 0; @@ -1395,21 +1024,24 @@ priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev) /** * Install interrupt handler. * - * @param priv - * Pointer to private structure. * @param dev - * Pointer to the rte_eth_dev structure. + * Pointer to Ethernet device. */ void -priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) +mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev) { - int rc, flags; + struct priv *priv = dev->data->dev_private; + int ret; + int flags; assert(priv->ctx->async_fd > 0); flags = fcntl(priv->ctx->async_fd, F_GETFL); - rc = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); - if (rc < 0) { - INFO("failed to change file descriptor async event queue"); + ret = fcntl(priv->ctx->async_fd, F_SETFL, flags | O_NONBLOCK); + if (ret) { + DRV_LOG(INFO, + "port %u failed to change file descriptor async event" + " queue", + dev->data->port_id); dev->data->dev_conf.intr_conf.lsc = 0; dev->data->dev_conf.intr_conf.rmv = 0; } @@ -1420,9 +1052,11 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) rte_intr_callback_register(&priv->intr_handle, mlx5_dev_interrupt_handler, dev); } - - rc = priv_socket_init(priv); - if (!rc && priv->primary_socket) { + ret = mlx5_socket_init(dev); + if (ret) + DRV_LOG(ERR, "port %u cannot initialise socket: %s", + dev->data->port_id, strerror(rte_errno)); + else if (priv->primary_socket) { priv->intr_handle_socket.fd = priv->primary_socket; priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT; rte_intr_callback_register(&priv->intr_handle_socket, @@ -1430,23 +1064,6 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev) } } -/** - * Change the link state (UP / DOWN). - * - * @param priv - * Pointer to private data structure. - * @param up - * Nonzero for link up, otherwise link down. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_dev_set_link(struct priv *priv, int up) -{ - return priv_set_flags(priv, ~IFF_UP, up ? IFF_UP : ~IFF_UP); -} - /** * DPDK callback to bring the link DOWN. * @@ -1454,18 +1071,12 @@ priv_dev_set_link(struct priv *priv, int up) * Pointer to Ethernet device structure. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_set_link_down(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - int err; - - priv_lock(priv); - err = priv_dev_set_link(priv, 0); - priv_unlock(priv); - return err; + return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP); } /** @@ -1475,72 +1086,77 @@ mlx5_set_link_down(struct rte_eth_dev *dev) * Pointer to Ethernet device structure. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_set_link_up(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - int err; - - priv_lock(priv); - err = priv_dev_set_link(priv, 1); - priv_unlock(priv); - return err; + return mlx5_set_flags(dev, ~IFF_UP, IFF_UP); } /** * Configure the TX function to use. * - * @param priv - * Pointer to private data structure. * @param dev * Pointer to rte_eth_dev structure. + * + * @return + * Pointer to selected Tx burst function. */ -void -priv_dev_select_tx_function(struct priv *priv, struct rte_eth_dev *dev) +eth_tx_burst_t +mlx5_select_tx_function(struct rte_eth_dev *dev) { - assert(priv != NULL); - assert(dev != NULL); - dev->tx_pkt_burst = mlx5_tx_burst; + struct priv *priv = dev->data->dev_private; + eth_tx_burst_t tx_pkt_burst = mlx5_tx_burst; + /* Select appropriate TX function. */ if (priv->mps == MLX5_MPW_ENHANCED) { - if (priv_check_vec_tx_support(priv) > 0) { - if (priv_check_raw_vec_tx_support(priv) > 0) - dev->tx_pkt_burst = mlx5_tx_burst_raw_vec; + if (mlx5_check_vec_tx_support(dev) > 0) { + if (mlx5_check_raw_vec_tx_support(dev) > 0) + tx_pkt_burst = mlx5_tx_burst_raw_vec; else - dev->tx_pkt_burst = mlx5_tx_burst_vec; - DEBUG("selected Enhanced MPW TX vectorized function"); + tx_pkt_burst = mlx5_tx_burst_vec; + DRV_LOG(DEBUG, + "port %u selected enhanced MPW Tx vectorized" + " function", + dev->data->port_id); } else { - dev->tx_pkt_burst = mlx5_tx_burst_empw; - DEBUG("selected Enhanced MPW TX function"); + tx_pkt_burst = mlx5_tx_burst_empw; + DRV_LOG(DEBUG, + "port %u selected enhanced MPW Tx function", + dev->data->port_id); } } else if (priv->mps && priv->txq_inline) { - dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline; - DEBUG("selected MPW inline TX function"); + tx_pkt_burst = mlx5_tx_burst_mpw_inline; + DRV_LOG(DEBUG, "port %u selected MPW inline Tx function", + dev->data->port_id); } else if (priv->mps) { - dev->tx_pkt_burst = mlx5_tx_burst_mpw; - DEBUG("selected MPW TX function"); + tx_pkt_burst = mlx5_tx_burst_mpw; + DRV_LOG(DEBUG, "port %u selected MPW Tx function", + dev->data->port_id); } + return tx_pkt_burst; } /** * Configure the RX function to use. * - * @param priv - * Pointer to private data structure. * @param dev * Pointer to rte_eth_dev structure. + * + * @return + * Pointer to selected Rx burst function. */ -void -priv_dev_select_rx_function(struct priv *priv, struct rte_eth_dev *dev) +eth_rx_burst_t +mlx5_select_rx_function(struct rte_eth_dev *dev) { - assert(priv != NULL); + eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst; + assert(dev != NULL); - if (priv_check_vec_rx_support(priv) > 0) { - dev->rx_pkt_burst = mlx5_rx_burst_vec; - DEBUG("selected RX vectorized function"); - } else { - dev->rx_pkt_burst = mlx5_rx_burst; + if (mlx5_check_vec_rx_support(dev) > 0) { + rx_pkt_burst = mlx5_rx_burst_vec; + DRV_LOG(DEBUG, "port %u selected Rx vectorized function", + dev->data->port_id); } + return rx_pkt_burst; } diff --git a/dpdk/drivers/net/mlx5/mlx5_flow.c b/dpdk/drivers/net/mlx5/mlx5_flow.c index 092644ff..1822c2bd 100644 --- a/dpdk/drivers/net/mlx5/mlx5_flow.c +++ b/dpdk/drivers/net/mlx5/mlx5_flow.c @@ -44,10 +44,12 @@ #pragma GCC diagnostic error "-Wpedantic" #endif +#include #include #include #include #include +#include #include "mlx5.h" #include "mlx5_defs.h" @@ -61,12 +63,6 @@ #define MLX5_IPV6 6 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT -struct ibv_counter_set_init_attr { - int dummy; -}; -struct ibv_flow_spec_counter_action { - int dummy; -}; struct ibv_counter_set { int dummy; }; @@ -83,40 +79,46 @@ ibv_destroy_counter_set(struct ibv_counter_set *cs) extern const struct eth_dev_ops mlx5_dev_ops; extern const struct eth_dev_ops mlx5_dev_ops_isolate; +/** Structure give to the conversion functions. */ +struct mlx5_flow_data { + struct mlx5_flow_parse *parser; /** Parser context. */ + struct rte_flow_error *error; /** Error context. */ +}; + static int mlx5_flow_create_eth(const struct rte_flow_item *item, const void *default_mask, - void *data); + struct mlx5_flow_data *data); static int mlx5_flow_create_vlan(const struct rte_flow_item *item, const void *default_mask, - void *data); + struct mlx5_flow_data *data); static int mlx5_flow_create_ipv4(const struct rte_flow_item *item, const void *default_mask, - void *data); + struct mlx5_flow_data *data); static int mlx5_flow_create_ipv6(const struct rte_flow_item *item, const void *default_mask, - void *data); + struct mlx5_flow_data *data); static int mlx5_flow_create_udp(const struct rte_flow_item *item, const void *default_mask, - void *data); + struct mlx5_flow_data *data); static int mlx5_flow_create_tcp(const struct rte_flow_item *item, const void *default_mask, - void *data); + struct mlx5_flow_data *data); static int mlx5_flow_create_vxlan(const struct rte_flow_item *item, const void *default_mask, - void *data); + struct mlx5_flow_data *data); struct mlx5_flow_parse; @@ -128,7 +130,7 @@ static int mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id); static int -mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser); +mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser); /* Hash RX queue types. */ enum hash_rxq_type { @@ -157,7 +159,7 @@ const struct hash_rxq_init hash_rxq_init[] = { IBV_RX_HASH_SRC_PORT_TCP | IBV_RX_HASH_DST_PORT_TCP), .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP, - .flow_priority = 0, + .flow_priority = 1, .ip_version = MLX5_IPV4, }, [HASH_RXQ_UDPV4] = { @@ -166,7 +168,7 @@ const struct hash_rxq_init hash_rxq_init[] = { IBV_RX_HASH_SRC_PORT_UDP | IBV_RX_HASH_DST_PORT_UDP), .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP, - .flow_priority = 0, + .flow_priority = 1, .ip_version = MLX5_IPV4, }, [HASH_RXQ_IPV4] = { @@ -174,7 +176,7 @@ const struct hash_rxq_init hash_rxq_init[] = { IBV_RX_HASH_DST_IPV4), .dpdk_rss_hf = (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4), - .flow_priority = 1, + .flow_priority = 2, .ip_version = MLX5_IPV4, }, [HASH_RXQ_TCPV6] = { @@ -183,7 +185,7 @@ const struct hash_rxq_init hash_rxq_init[] = { IBV_RX_HASH_SRC_PORT_TCP | IBV_RX_HASH_DST_PORT_TCP), .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP, - .flow_priority = 0, + .flow_priority = 1, .ip_version = MLX5_IPV6, }, [HASH_RXQ_UDPV6] = { @@ -192,7 +194,7 @@ const struct hash_rxq_init hash_rxq_init[] = { IBV_RX_HASH_SRC_PORT_UDP | IBV_RX_HASH_DST_PORT_UDP), .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP, - .flow_priority = 0, + .flow_priority = 1, .ip_version = MLX5_IPV6, }, [HASH_RXQ_IPV6] = { @@ -200,13 +202,13 @@ const struct hash_rxq_init hash_rxq_init[] = { IBV_RX_HASH_DST_IPV6), .dpdk_rss_hf = (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6), - .flow_priority = 1, + .flow_priority = 2, .ip_version = MLX5_IPV6, }, [HASH_RXQ_ETH] = { .hash_fields = 0, .dpdk_rss_hf = 0, - .flow_priority = 2, + .flow_priority = 3, }, }; @@ -286,11 +288,12 @@ struct mlx5_flow_items { * Internal structure to store the conversion. * * @return - * 0 on success, negative value otherwise. + * 0 on success, a negative errno value otherwise and rte_errno is + * set. */ int (*convert)(const struct rte_flow_item *item, const void *default_mask, - void *data); + struct mlx5_flow_data *data); /** Size in bytes of the destination structure. */ const unsigned int dst_sz; /** List of possible following items. */ @@ -474,10 +477,18 @@ struct mlx5_fdir { struct rte_flow_item_ipv4 ipv4; struct rte_flow_item_ipv6 ipv6; } l3; + union { + struct rte_flow_item_ipv4 ipv4; + struct rte_flow_item_ipv6 ipv6; + } l3_mask; union { struct rte_flow_item_udp udp; struct rte_flow_item_tcp tcp; } l4; + union { + struct rte_flow_item_udp udp; + struct rte_flow_item_tcp tcp; + } l4_mask; struct rte_flow_action_queue queue; }; @@ -488,7 +499,7 @@ struct ibv_spec_header { }; /** - * Check support for a given item. + * Check item is fully supported by the NIC matching capability. * * @param item[in] * Item specification. @@ -499,87 +510,71 @@ struct ibv_spec_header { * Bit-Mask size in bytes. * * @return - * 0 on success. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_flow_item_validate(const struct rte_flow_item *item, const uint8_t *mask, unsigned int size) { - int ret = 0; + unsigned int i; + const uint8_t *spec = item->spec; + const uint8_t *last = item->last; + const uint8_t *m = item->mask ? item->mask : mask; - if (!item->spec && (item->mask || item->last)) - return -1; - if (item->spec && !item->mask) { - unsigned int i; - const uint8_t *spec = item->spec; - - for (i = 0; i < size; ++i) - if ((spec[i] | mask[i]) != mask[i]) - return -1; + if (!spec && (item->mask || last)) + goto error; + if (!spec) + return 0; + /* + * Single-pass check to make sure that: + * - item->mask is supported, no bits are set outside mask. + * - Both masked item->spec and item->last are equal (no range + * supported). + */ + for (i = 0; i < size; i++) { + if (!m[i]) + continue; + if ((m[i] | mask[i]) != mask[i]) + goto error; + if (last && ((spec[i] & m[i]) != (last[i] & m[i]))) + goto error; } - if (item->last && !item->mask) { - unsigned int i; - const uint8_t *spec = item->last; - - for (i = 0; i < size; ++i) - if ((spec[i] | mask[i]) != mask[i]) - return -1; - } - if (item->mask) { - unsigned int i; - const uint8_t *spec = item->spec; - - for (i = 0; i < size; ++i) - if ((spec[i] | mask[i]) != mask[i]) - return -1; - } - if (item->spec && item->last) { - uint8_t spec[size]; - uint8_t last[size]; - const uint8_t *apply = mask; - unsigned int i; - - if (item->mask) - apply = item->mask; - for (i = 0; i < size; ++i) { - spec[i] = ((const uint8_t *)item->spec)[i] & apply[i]; - last[i] = ((const uint8_t *)item->last)[i] & apply[i]; - } - ret = memcmp(spec, last, size); - } - return ret; + return 0; +error: + rte_errno = ENOTSUP; + return -rte_errno; } /** * Copy the RSS configuration from the user ones, of the rss_conf is null, * uses the driver one. * - * @param priv - * Pointer to private structure. * @param parser * Internal parser structure. * @param rss_conf * User RSS configuration to save. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_convert_rss_conf(struct priv *priv, - struct mlx5_flow_parse *parser, +mlx5_flow_convert_rss_conf(struct mlx5_flow_parse *parser, const struct rte_eth_rss_conf *rss_conf) { /* * This function is also called at the beginning of - * priv_flow_convert_actions() to initialize the parser with the + * mlx5_flow_convert_actions() to initialize the parser with the * device default RSS configuration. */ - (void)priv; if (rss_conf) { - if (rss_conf->rss_hf & MLX5_RSS_HF_MASK) - return EINVAL; - if (rss_conf->rss_key_len != 40) - return EINVAL; + if (rss_conf->rss_hf & MLX5_RSS_HF_MASK) { + rte_errno = EINVAL; + return -rte_errno; + } + if (rss_conf->rss_key_len != 40) { + rte_errno = EINVAL; + return -rte_errno; + } if (rss_conf->rss_key_len && rss_conf->rss_key) { parser->rss_conf.rss_key_len = rss_conf->rss_key_len; memcpy(parser->rss_key, rss_conf->rss_key, @@ -594,26 +589,18 @@ priv_flow_convert_rss_conf(struct priv *priv, /** * Extract attribute to the parser. * - * @param priv - * Pointer to private structure. * @param[in] attr * Flow rule attributes. * @param[out] error * Perform verbose error reporting if not NULL. - * @param[in, out] parser - * Internal parser structure. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_convert_attributes(struct priv *priv, - const struct rte_flow_attr *attr, - struct rte_flow_error *error, - struct mlx5_flow_parse *parser) +mlx5_flow_convert_attributes(const struct rte_flow_attr *attr, + struct rte_flow_error *error) { - (void)priv; - (void)parser; if (attr->group) { rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ATTR_GROUP, @@ -648,8 +635,8 @@ priv_flow_convert_attributes(struct priv *priv, /** * Extract actions request to the parser. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param[in] actions * Associated actions (list terminated by the END action). * @param[out] error @@ -661,18 +648,23 @@ priv_flow_convert_attributes(struct priv *priv, * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_convert_actions(struct priv *priv, +mlx5_flow_convert_actions(struct rte_eth_dev *dev, const struct rte_flow_action actions[], struct rte_flow_error *error, struct mlx5_flow_parse *parser) { + struct priv *priv = dev->data->dev_private; + int ret; + /* * Add default RSS configuration necessary for Verbs to create QP even * if no RSS is necessary. */ - priv_flow_convert_rss_conf(priv, parser, - (const struct rte_eth_rss_conf *) - &priv->rss_conf); + ret = mlx5_flow_convert_rss_conf(parser, + (const struct rte_eth_rss_conf *) + &priv->rss_conf); + if (ret) + return ret; for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) { if (actions->type == RTE_FLOW_ACTION_TYPE_VOID) { continue; @@ -737,6 +729,14 @@ priv_flow_convert_actions(struct priv *priv, return -rte_errno; } } + if (rss->num > RTE_DIM(parser->queues)) { + rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_ACTION, + actions, + "too many queues for RSS" + " context"); + return -rte_errno; + } for (n = 0; n < rss->num; ++n) { if (rss->queue[n] >= priv->rxqs_n) { rte_flow_error_set(error, EINVAL, @@ -750,8 +750,7 @@ priv_flow_convert_actions(struct priv *priv, for (n = 0; n < rss->num; ++n) parser->queues[n] = rss->queue[n]; parser->queues_n = rss->num; - if (priv_flow_convert_rss_conf(priv, parser, - rss->rss_conf)) { + if (mlx5_flow_convert_rss_conf(parser, rss->rss_conf)) { rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION, actions, @@ -805,8 +804,6 @@ exit_action_not_supported: /** * Validate items. * - * @param priv - * Pointer to private structure. * @param[in] items * Pattern specification (list terminated by the END pattern item). * @param[out] error @@ -818,22 +815,20 @@ exit_action_not_supported: * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_convert_items_validate(struct priv *priv, - const struct rte_flow_item items[], +mlx5_flow_convert_items_validate(const struct rte_flow_item items[], struct rte_flow_error *error, struct mlx5_flow_parse *parser) { const struct mlx5_flow_items *cur_item = mlx5_flow_items; unsigned int i; + int ret = 0; - (void)priv; /* Initialise the offsets to start after verbs attribute. */ for (i = 0; i != hash_rxq_init_n; ++i) parser->queue[i].offset = sizeof(struct ibv_flow_attr); for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { const struct mlx5_flow_items *token = NULL; unsigned int n; - int err; if (items->type == RTE_FLOW_ITEM_TYPE_VOID) continue; @@ -846,13 +841,15 @@ priv_flow_convert_items_validate(struct priv *priv, break; } } - if (!token) + if (!token) { + ret = -ENOTSUP; goto exit_item_not_supported; + } cur_item = token; - err = mlx5_flow_item_validate(items, + ret = mlx5_flow_item_validate(items, (const uint8_t *)cur_item->mask, cur_item->mask_sz); - if (err) + if (ret) goto exit_item_not_supported; if (items->type == RTE_FLOW_ITEM_TYPE_VXLAN) { if (parser->inner) { @@ -882,64 +879,90 @@ priv_flow_convert_items_validate(struct priv *priv, sizeof(struct ibv_flow_spec_action_tag); } if (parser->count) { +#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT unsigned int size = sizeof(struct ibv_flow_spec_counter_action); for (i = 0; i != hash_rxq_init_n; ++i) parser->queue[i].offset += size; +#else + rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION, + items, + "Count action supported only on " + "MLNX_OFED_4.2 and above"); +#endif } return 0; exit_item_not_supported: - rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, - items, "item not supported"); - return -rte_errno; + return rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_ITEM, + items, "item not supported"); } /** * Allocate memory space to store verbs flow attributes. * - * @param priv - * Pointer to private structure. - * @param[in] priority - * Flow priority. * @param[in] size * Amount of byte to allocate. * @param[out] error * Perform verbose error reporting if not NULL. * * @return - * A verbs flow attribute on success, NULL otherwise. + * A verbs flow attribute on success, NULL otherwise and rte_errno is set. */ -static struct ibv_flow_attr* -priv_flow_convert_allocate(struct priv *priv, - unsigned int priority, - unsigned int size, - struct rte_flow_error *error) +static struct ibv_flow_attr * +mlx5_flow_convert_allocate(unsigned int size, struct rte_flow_error *error) { struct ibv_flow_attr *ibv_attr; - (void)priv; ibv_attr = rte_calloc(__func__, 1, size, 0); if (!ibv_attr) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, - "cannot allocate verbs spec attributes."); + "cannot allocate verbs spec attributes"); return NULL; } - ibv_attr->priority = priority; return ibv_attr; } +/** + * Make inner packet matching with an higher priority from the non Inner + * matching. + * + * @param[in, out] parser + * Internal parser structure. + * @param attr + * User flow attribute. + */ +static void +mlx5_flow_update_priority(struct mlx5_flow_parse *parser, + const struct rte_flow_attr *attr) +{ + unsigned int i; + + if (parser->drop) { + parser->queue[HASH_RXQ_ETH].ibv_attr->priority = + attr->priority + + hash_rxq_init[HASH_RXQ_ETH].flow_priority; + return; + } + for (i = 0; i != hash_rxq_init_n; ++i) { + if (parser->queue[i].ibv_attr) { + parser->queue[i].ibv_attr->priority = + attr->priority + + hash_rxq_init[i].flow_priority - + (parser->inner ? 1 : 0); + } + } +} + /** * Finalise verbs flow attributes. * - * @param priv - * Pointer to private structure. * @param[in, out] parser * Internal parser structure. */ static void -priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser) +mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser) { const unsigned int ipv4 = hash_rxq_init[parser->layer].ip_version == MLX5_IPV4; @@ -950,7 +973,16 @@ priv_flow_convert_finalise(struct priv *priv, struct mlx5_flow_parse *parser) const enum hash_rxq_type ip = ipv4 ? HASH_RXQ_IPV4 : HASH_RXQ_IPV6; unsigned int i; - (void)priv; + /* Remove any other flow not matching the pattern. */ + if (parser->queues_n == 1 && !parser->rss_conf.rss_hf) { + for (i = 0; i != hash_rxq_init_n; ++i) { + if (i == HASH_RXQ_ETH) + continue; + rte_free(parser->queue[i].ibv_attr); + parser->queue[i].ibv_attr = NULL; + } + return; + } if (parser->layer == HASH_RXQ_ETH) { goto fill; } else { @@ -1049,8 +1081,8 @@ fill: /** * Validate and convert a flow supported by the NIC. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param[in] attr * Flow rule attributes. * @param[in] pattern @@ -1066,7 +1098,7 @@ fill: * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_convert(struct priv *priv, +mlx5_flow_convert(struct rte_eth_dev *dev, const struct rte_flow_attr *attr, const struct rte_flow_item items[], const struct rte_flow_action actions[], @@ -1083,35 +1115,31 @@ priv_flow_convert(struct priv *priv, .layer = HASH_RXQ_ETH, .mark_id = MLX5_FLOW_MARK_DEFAULT, }; - ret = priv_flow_convert_attributes(priv, attr, error, parser); + ret = mlx5_flow_convert_attributes(attr, error); if (ret) return ret; - ret = priv_flow_convert_actions(priv, actions, error, parser); + ret = mlx5_flow_convert_actions(dev, actions, error, parser); if (ret) return ret; - ret = priv_flow_convert_items_validate(priv, items, error, parser); + ret = mlx5_flow_convert_items_validate(items, error, parser); if (ret) return ret; - priv_flow_convert_finalise(priv, parser); + mlx5_flow_convert_finalise(parser); /* * Second step. * Allocate the memory space to store verbs specifications. */ if (parser->drop) { + unsigned int offset = parser->queue[HASH_RXQ_ETH].offset; + parser->queue[HASH_RXQ_ETH].ibv_attr = - priv_flow_convert_allocate - (priv, attr->priority, - parser->queue[HASH_RXQ_ETH].offset, - error); + mlx5_flow_convert_allocate(offset, error); if (!parser->queue[HASH_RXQ_ETH].ibv_attr) - return ENOMEM; + goto exit_enomem; parser->queue[HASH_RXQ_ETH].offset = sizeof(struct ibv_flow_attr); } else { for (i = 0; i != hash_rxq_init_n; ++i) { - unsigned int priority = - attr->priority + - hash_rxq_init[i].flow_priority; unsigned int offset; if (!(parser->rss_conf.rss_hf & @@ -1120,8 +1148,7 @@ priv_flow_convert(struct priv *priv, continue; offset = parser->queue[i].offset; parser->queue[i].ibv_attr = - priv_flow_convert_allocate(priv, priority, - offset, error); + mlx5_flow_convert_allocate(offset, error); if (!parser->queue[i].ibv_attr) goto exit_enomem; parser->queue[i].offset = sizeof(struct ibv_flow_attr); @@ -1130,6 +1157,11 @@ priv_flow_convert(struct priv *priv, /* Third step. Conversion parse, fill the specifications. */ parser->inner = 0; for (; items->type != RTE_FLOW_ITEM_TYPE_END; ++items) { + struct mlx5_flow_data data = { + .parser = parser, + .error = error, + }; + if (items->type == RTE_FLOW_ITEM_TYPE_VOID) continue; cur_item = &mlx5_flow_items[items->type]; @@ -1137,18 +1169,14 @@ priv_flow_convert(struct priv *priv, (cur_item->default_mask ? cur_item->default_mask : cur_item->mask), - parser); - if (ret) { - rte_flow_error_set(error, ret, - RTE_FLOW_ERROR_TYPE_ITEM, - items, "item not supported"); + &data); + if (ret) goto exit_free; - } } if (parser->mark) mlx5_flow_create_flag_mark(parser, parser->mark_id); if (parser->count && parser->create) { - mlx5_flow_create_count(priv, parser); + mlx5_flow_create_count(dev, parser); if (!parser->cs) goto exit_count_error; } @@ -1156,13 +1184,9 @@ priv_flow_convert(struct priv *priv, * Last step. Complete missing specification to reach the RSS * configuration. */ - if (!parser->drop) { - priv_flow_convert_finalise(priv, parser); - } else { - parser->queue[HASH_RXQ_ETH].ibv_attr->priority = - attr->priority + - hash_rxq_init[parser->layer].flow_priority; - } + if (!parser->drop) + mlx5_flow_convert_finalise(parser); + mlx5_flow_update_priority(parser, attr); exit_free: /* Only verification is expected, all resources should be released. */ if (!parser->create) { @@ -1181,13 +1205,13 @@ exit_enomem: parser->queue[i].ibv_attr = NULL; } } - rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot allocate verbs spec attributes."); - return ret; + rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, "cannot allocate verbs spec attributes"); + return -rte_errno; exit_count_error: rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, "cannot create counter."); - return rte_errno; + NULL, "cannot create counter"); + return -rte_errno; } /** @@ -1233,15 +1257,18 @@ mlx5_flow_create_copy(struct mlx5_flow_parse *parser, void *src, * Default bit-masks to use when item->mask is not provided. * @param data[in, out] * User structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_flow_create_eth(const struct rte_flow_item *item, const void *default_mask, - void *data) + struct mlx5_flow_data *data) { const struct rte_flow_item_eth *spec = item->spec; const struct rte_flow_item_eth *mask = item->mask; - struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + struct mlx5_flow_parse *parser = data->parser; const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); struct ibv_flow_spec_eth eth = { .type = parser->inner | IBV_FLOW_SPEC_ETH, @@ -1282,15 +1309,18 @@ mlx5_flow_create_eth(const struct rte_flow_item *item, * Default bit-masks to use when item->mask is not provided. * @param data[in, out] * User structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_flow_create_vlan(const struct rte_flow_item *item, const void *default_mask, - void *data) + struct mlx5_flow_data *data) { const struct rte_flow_item_vlan *spec = item->spec; const struct rte_flow_item_vlan *mask = item->mask; - struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + struct mlx5_flow_parse *parser = data->parser; struct ibv_flow_spec_eth *eth; const unsigned int eth_size = sizeof(struct ibv_flow_spec_eth); @@ -1308,9 +1338,18 @@ mlx5_flow_create_vlan(const struct rte_flow_item *item, eth->val.vlan_tag = spec->tci; eth->mask.vlan_tag = mask->tci; eth->val.vlan_tag &= eth->mask.vlan_tag; + /* + * From verbs perspective an empty VLAN is equivalent + * to a packet without VLAN layer. + */ + if (!eth->mask.vlan_tag) + goto error; } + return 0; } - return 0; +error: + return rte_flow_error_set(data->error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, + item, "VLAN cannot be empty"); } /** @@ -1322,15 +1361,18 @@ mlx5_flow_create_vlan(const struct rte_flow_item *item, * Default bit-masks to use when item->mask is not provided. * @param data[in, out] * User structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_flow_create_ipv4(const struct rte_flow_item *item, const void *default_mask, - void *data) + struct mlx5_flow_data *data) { const struct rte_flow_item_ipv4 *spec = item->spec; const struct rte_flow_item_ipv4 *mask = item->mask; - struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + struct mlx5_flow_parse *parser = data->parser; unsigned int ipv4_size = sizeof(struct ibv_flow_spec_ipv4_ext); struct ibv_flow_spec_ipv4_ext ipv4 = { .type = parser->inner | IBV_FLOW_SPEC_IPV4_EXT, @@ -1374,15 +1416,18 @@ mlx5_flow_create_ipv4(const struct rte_flow_item *item, * Default bit-masks to use when item->mask is not provided. * @param data[in, out] * User structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_flow_create_ipv6(const struct rte_flow_item *item, const void *default_mask, - void *data) + struct mlx5_flow_data *data) { const struct rte_flow_item_ipv6 *spec = item->spec; const struct rte_flow_item_ipv6 *mask = item->mask; - struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + struct mlx5_flow_parse *parser = data->parser; unsigned int ipv6_size = sizeof(struct ibv_flow_spec_ipv6); struct ibv_flow_spec_ipv6 ipv6 = { .type = parser->inner | IBV_FLOW_SPEC_IPV6, @@ -1394,6 +1439,8 @@ mlx5_flow_create_ipv6(const struct rte_flow_item *item, parser->layer = HASH_RXQ_IPV6; if (spec) { unsigned int i; + uint32_t vtc_flow_val; + uint32_t vtc_flow_mask; if (!mask) mask = default_mask; @@ -1405,7 +1452,20 @@ mlx5_flow_create_ipv6(const struct rte_flow_item *item, RTE_DIM(ipv6.mask.src_ip)); memcpy(&ipv6.mask.dst_ip, mask->hdr.dst_addr, RTE_DIM(ipv6.mask.dst_ip)); - ipv6.mask.flow_label = mask->hdr.vtc_flow; + vtc_flow_val = rte_be_to_cpu_32(spec->hdr.vtc_flow); + vtc_flow_mask = rte_be_to_cpu_32(mask->hdr.vtc_flow); + ipv6.val.flow_label = + rte_cpu_to_be_32((vtc_flow_val & IPV6_HDR_FL_MASK) >> + IPV6_HDR_FL_SHIFT); + ipv6.val.traffic_class = (vtc_flow_val & IPV6_HDR_TC_MASK) >> + IPV6_HDR_TC_SHIFT; + ipv6.val.next_hdr = spec->hdr.proto; + ipv6.val.hop_limit = spec->hdr.hop_limits; + ipv6.mask.flow_label = + rte_cpu_to_be_32((vtc_flow_mask & IPV6_HDR_FL_MASK) >> + IPV6_HDR_FL_SHIFT); + ipv6.mask.traffic_class = (vtc_flow_mask & IPV6_HDR_TC_MASK) >> + IPV6_HDR_TC_SHIFT; ipv6.mask.next_hdr = mask->hdr.proto; ipv6.mask.hop_limit = mask->hdr.hop_limits; /* Remove unwanted bits from values. */ @@ -1414,6 +1474,7 @@ mlx5_flow_create_ipv6(const struct rte_flow_item *item, ipv6.val.dst_ip[i] &= ipv6.mask.dst_ip[i]; } ipv6.val.flow_label &= ipv6.mask.flow_label; + ipv6.val.traffic_class &= ipv6.mask.traffic_class; ipv6.val.next_hdr &= ipv6.mask.next_hdr; ipv6.val.hop_limit &= ipv6.mask.hop_limit; } @@ -1430,15 +1491,18 @@ mlx5_flow_create_ipv6(const struct rte_flow_item *item, * Default bit-masks to use when item->mask is not provided. * @param data[in, out] * User structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_flow_create_udp(const struct rte_flow_item *item, const void *default_mask, - void *data) + struct mlx5_flow_data *data) { const struct rte_flow_item_udp *spec = item->spec; const struct rte_flow_item_udp *mask = item->mask; - struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + struct mlx5_flow_parse *parser = data->parser; unsigned int udp_size = sizeof(struct ibv_flow_spec_tcp_udp); struct ibv_flow_spec_tcp_udp udp = { .type = parser->inner | IBV_FLOW_SPEC_UDP, @@ -1476,15 +1540,18 @@ mlx5_flow_create_udp(const struct rte_flow_item *item, * Default bit-masks to use when item->mask is not provided. * @param data[in, out] * User structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_flow_create_tcp(const struct rte_flow_item *item, const void *default_mask, - void *data) + struct mlx5_flow_data *data) { const struct rte_flow_item_tcp *spec = item->spec; const struct rte_flow_item_tcp *mask = item->mask; - struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + struct mlx5_flow_parse *parser = data->parser; unsigned int tcp_size = sizeof(struct ibv_flow_spec_tcp_udp); struct ibv_flow_spec_tcp_udp tcp = { .type = parser->inner | IBV_FLOW_SPEC_TCP, @@ -1522,15 +1589,18 @@ mlx5_flow_create_tcp(const struct rte_flow_item *item, * Default bit-masks to use when item->mask is not provided. * @param data[in, out] * User structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_flow_create_vxlan(const struct rte_flow_item *item, const void *default_mask, - void *data) + struct mlx5_flow_data *data) { const struct rte_flow_item_vxlan *spec = item->spec; const struct rte_flow_item_vxlan *mask = item->mask; - struct mlx5_flow_parse *parser = (struct mlx5_flow_parse *)data; + struct mlx5_flow_parse *parser = data->parser; unsigned int size = sizeof(struct ibv_flow_spec_tunnel); struct ibv_flow_spec_tunnel vxlan = { .type = parser->inner | IBV_FLOW_SPEC_VXLAN_TUNNEL, @@ -1562,7 +1632,10 @@ mlx5_flow_create_vxlan(const struct rte_flow_item *item, * To avoid such situation, VNI 0 is currently refused. */ if (!vxlan.val.tunnel_id) - return EINVAL; + return rte_flow_error_set(data->error, EINVAL, + RTE_FLOW_ERROR_TYPE_ITEM, + item, + "VxLAN vni cannot be 0"); mlx5_flow_create_copy(parser, &vxlan, size); return 0; } @@ -1574,6 +1647,9 @@ mlx5_flow_create_vxlan(const struct rte_flow_item *item, * Internal parser structure. * @param mark_id * Mark identifier. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) @@ -1593,19 +1669,20 @@ mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id) /** * Convert count action to Verbs specification. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param parser * Pointer to MLX5 flow parser structure. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -mlx5_flow_create_count(struct priv *priv __rte_unused, +mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused, struct mlx5_flow_parse *parser __rte_unused) { #ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT + struct priv *priv = dev->data->dev_private; unsigned int size = sizeof(struct ibv_flow_spec_counter_action); struct ibv_counter_set_init_attr init_attr = {0}; struct ibv_flow_spec_counter_action counter = { @@ -1616,8 +1693,10 @@ mlx5_flow_create_count(struct priv *priv __rte_unused, init_attr.counter_set_id = 0; parser->cs = ibv_create_counter_set(priv->ctx, &init_attr); - if (!parser->cs) - return EINVAL; + if (!parser->cs) { + rte_errno = EINVAL; + return -rte_errno; + } counter.counter_set_handle = parser->cs->handle; mlx5_flow_create_copy(parser, &counter, size); #endif @@ -1627,8 +1706,8 @@ mlx5_flow_create_count(struct priv *priv __rte_unused, /** * Complete flow rule creation with a drop queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param parser * Internal parser structure. * @param flow @@ -1637,17 +1716,17 @@ mlx5_flow_create_count(struct priv *priv __rte_unused, * Perform verbose error reporting if not NULL. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_create_action_queue_drop(struct priv *priv, +mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser, struct rte_flow *flow, struct rte_flow_error *error) { + struct priv *priv = dev->data->dev_private; struct ibv_flow_spec_action_drop *drop; unsigned int size = sizeof(struct ibv_flow_spec_action_drop); - int err = 0; assert(priv->pd); assert(priv->ctx); @@ -1664,7 +1743,7 @@ priv_flow_create_action_queue_drop(struct priv *priv, parser->queue[HASH_RXQ_ETH].ibv_attr; if (parser->count) flow->cs = parser->cs; - if (!priv->dev->data->dev_started) + if (!dev->data->dev_started) return 0; parser->queue[HASH_RXQ_ETH].ibv_attr = NULL; flow->frxq[HASH_RXQ_ETH].ibv_flow = @@ -1673,7 +1752,6 @@ priv_flow_create_action_queue_drop(struct priv *priv, if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "flow rule creation failure"); - err = ENOMEM; goto error; } return 0; @@ -1692,14 +1770,14 @@ error: flow->cs = NULL; parser->cs = NULL; } - return err; + return -rte_errno; } /** * Create hash Rx queues when RSS is enabled. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param parser * Internal parser structure. * @param flow @@ -1708,10 +1786,10 @@ error: * Perform verbose error reporting if not NULL. * * @return - * 0 on success, a errno value otherwise and rte_errno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_create_action_queue_rss(struct priv *priv, +mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser, struct rte_flow *flow, struct rte_flow_error *error) @@ -1726,29 +1804,29 @@ priv_flow_create_action_queue_rss(struct priv *priv, flow->frxq[i].ibv_attr = parser->queue[i].ibv_attr; parser->queue[i].ibv_attr = NULL; hash_fields = hash_rxq_init[i].hash_fields; - if (!priv->dev->data->dev_started) + if (!dev->data->dev_started) continue; flow->frxq[i].hrxq = - mlx5_priv_hrxq_get(priv, - parser->rss_conf.rss_key, - parser->rss_conf.rss_key_len, - hash_fields, - parser->queues, - parser->queues_n); + mlx5_hrxq_get(dev, + parser->rss_conf.rss_key, + parser->rss_conf.rss_key_len, + hash_fields, + parser->queues, + parser->queues_n); if (flow->frxq[i].hrxq) continue; flow->frxq[i].hrxq = - mlx5_priv_hrxq_new(priv, - parser->rss_conf.rss_key, - parser->rss_conf.rss_key_len, - hash_fields, - parser->queues, - parser->queues_n); + mlx5_hrxq_new(dev, + parser->rss_conf.rss_key, + parser->rss_conf.rss_key_len, + hash_fields, + parser->queues, + parser->queues_n); if (!flow->frxq[i].hrxq) { - rte_flow_error_set(error, ENOMEM, - RTE_FLOW_ERROR_TYPE_HANDLE, - NULL, "cannot create hash rxq"); - return ENOMEM; + return rte_flow_error_set(error, ENOMEM, + RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, + "cannot create hash rxq"); } } return 0; @@ -1757,8 +1835,8 @@ priv_flow_create_action_queue_rss(struct priv *priv, /** * Complete flow rule creation. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param parser * Internal parser structure. * @param flow @@ -1767,26 +1845,28 @@ priv_flow_create_action_queue_rss(struct priv *priv, * Perform verbose error reporting if not NULL. * * @return - * 0 on success, a errno value otherwise and rte_errno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_create_action_queue(struct priv *priv, +mlx5_flow_create_action_queue(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser, struct rte_flow *flow, struct rte_flow_error *error) { - int err = 0; + struct priv *priv __rte_unused = dev->data->dev_private; + int ret; unsigned int i; + unsigned int flows_n = 0; assert(priv->pd); assert(priv->ctx); assert(!parser->drop); - err = priv_flow_create_action_queue_rss(priv, parser, flow, error); - if (err) + ret = mlx5_flow_create_action_queue_rss(dev, parser, flow, error); + if (ret) goto error; if (parser->count) flow->cs = parser->cs; - if (!priv->dev->data->dev_started) + if (!dev->data->dev_started) return 0; for (i = 0; i != hash_rxq_init_n; ++i) { if (!flow->frxq[i].hrxq) @@ -1798,13 +1878,19 @@ priv_flow_create_action_queue(struct priv *priv, rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_HANDLE, NULL, "flow rule creation failure"); - err = ENOMEM; goto error; } - DEBUG("%p type %d QP %p ibv_flow %p", - (void *)flow, i, - (void *)flow->frxq[i].hrxq, - (void *)flow->frxq[i].ibv_flow); + ++flows_n; + DRV_LOG(DEBUG, "port %u %p type %d QP %p ibv_flow %p", + dev->data->port_id, + (void *)flow, i, + (void *)flow->frxq[i].hrxq, + (void *)flow->frxq[i].ibv_flow); + } + if (!flows_n) { + rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_HANDLE, + NULL, "internal error in flow creation"); + goto error; } for (i = 0; i != parser->queues_n; ++i) { struct mlx5_rxq_data *q = @@ -1814,6 +1900,7 @@ priv_flow_create_action_queue(struct priv *priv, } return 0; error: + ret = rte_errno; /* Save rte_errno before cleanup. */ assert(flow); for (i = 0; i != hash_rxq_init_n; ++i) { if (flow->frxq[i].ibv_flow) { @@ -1822,7 +1909,7 @@ error: claim_zero(ibv_destroy_flow(ibv_flow)); } if (flow->frxq[i].hrxq) - mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); + mlx5_hrxq_release(dev, flow->frxq[i].hrxq); if (flow->frxq[i].ibv_attr) rte_free(flow->frxq[i].ibv_attr); } @@ -1831,14 +1918,15 @@ error: flow->cs = NULL; parser->cs = NULL; } - return err; + rte_errno = ret; /* Restore rte_errno. */ + return -rte_errno; } /** * Convert a flow. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param list * Pointer to a TAILQ flow list. * @param[in] attr @@ -1851,23 +1939,23 @@ error: * Perform verbose error reporting if not NULL. * * @return - * A flow on success, NULL otherwise. + * A flow on success, NULL otherwise and rte_errno is set. */ static struct rte_flow * -priv_flow_create(struct priv *priv, - struct mlx5_flows *list, - const struct rte_flow_attr *attr, - const struct rte_flow_item items[], - const struct rte_flow_action actions[], - struct rte_flow_error *error) +mlx5_flow_list_create(struct rte_eth_dev *dev, + struct mlx5_flows *list, + const struct rte_flow_attr *attr, + const struct rte_flow_item items[], + const struct rte_flow_action actions[], + struct rte_flow_error *error) { struct mlx5_flow_parse parser = { .create = 1, }; struct rte_flow *flow = NULL; unsigned int i; - int err; + int ret; - err = priv_flow_convert(priv, attr, items, actions, error, &parser); - if (err) + ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser); + if (ret) goto exit; flow = rte_calloc(__func__, 1, sizeof(*flow) + parser.queues_n * sizeof(uint16_t), @@ -1890,14 +1978,15 @@ priv_flow_create(struct priv *priv, memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len); /* finalise the flow. */ if (parser.drop) - err = priv_flow_create_action_queue_drop(priv, &parser, flow, + ret = mlx5_flow_create_action_queue_drop(dev, &parser, flow, error); else - err = priv_flow_create_action_queue(priv, &parser, flow, error); - if (err) + ret = mlx5_flow_create_action_queue(dev, &parser, flow, error); + if (ret) goto exit; TAILQ_INSERT_TAIL(list, flow, next); - DEBUG("Flow created %p", (void *)flow); + DRV_LOG(DEBUG, "port %u flow created %p", dev->data->port_id, + (void *)flow); return flow; exit: for (i = 0; i != hash_rxq_init_n; ++i) { @@ -1921,14 +2010,9 @@ mlx5_flow_validate(struct rte_eth_dev *dev, const struct rte_flow_action actions[], struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; - int ret; struct mlx5_flow_parse parser = { .create = 0, }; - priv_lock(priv); - ret = priv_flow_convert(priv, attr, items, actions, error, &parser); - priv_unlock(priv); - return ret; + return mlx5_flow_convert(dev, attr, items, actions, error, &parser); } /** @@ -1945,30 +2029,26 @@ mlx5_flow_create(struct rte_eth_dev *dev, struct rte_flow_error *error) { struct priv *priv = dev->data->dev_private; - struct rte_flow *flow; - priv_lock(priv); - flow = priv_flow_create(priv, &priv->flows, attr, items, actions, - error); - priv_unlock(priv); - return flow; + return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions, + error); } /** - * Destroy a flow. + * Destroy a flow in a list. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param list * Pointer to a TAILQ flow list. * @param[in] flow * Flow to destroy. */ static void -priv_flow_destroy(struct priv *priv, - struct mlx5_flows *list, - struct rte_flow *flow) +mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list, + struct rte_flow *flow) { + struct priv *priv = dev->data->dev_private; unsigned int i; if (flow->drop || !flow->mark) @@ -2015,7 +2095,7 @@ free: if (frxq->ibv_flow) claim_zero(ibv_destroy_flow(frxq->ibv_flow)); if (frxq->hrxq) - mlx5_priv_hrxq_release(priv, frxq->hrxq); + mlx5_hrxq_release(dev, frxq->hrxq); if (frxq->ibv_attr) rte_free(frxq->ibv_attr); } @@ -2025,53 +2105,60 @@ free: flow->cs = NULL; } TAILQ_REMOVE(list, flow, next); - DEBUG("Flow destroyed %p", (void *)flow); + DRV_LOG(DEBUG, "port %u flow destroyed %p", dev->data->port_id, + (void *)flow); rte_free(flow); } /** * Destroy all flows. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param list * Pointer to a TAILQ flow list. */ void -priv_flow_flush(struct priv *priv, struct mlx5_flows *list) +mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list) { while (!TAILQ_EMPTY(list)) { struct rte_flow *flow; flow = TAILQ_FIRST(list); - priv_flow_destroy(priv, list, flow); + mlx5_flow_list_destroy(dev, list, flow); } } /** * Create drop queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return - * 0 on success. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_flow_create_drop_queue(struct priv *priv) +mlx5_flow_create_drop_queue(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_hrxq_drop *fdq = NULL; assert(priv->pd); assert(priv->ctx); fdq = rte_calloc(__func__, 1, sizeof(*fdq), 0); if (!fdq) { - WARN("cannot allocate memory for drop queue"); - goto error; + DRV_LOG(WARNING, + "port %u cannot allocate memory for drop queue", + dev->data->port_id); + rte_errno = ENOMEM; + return -rte_errno; } fdq->cq = ibv_create_cq(priv->ctx, 1, NULL, NULL, 0); if (!fdq->cq) { - WARN("cannot allocate CQ for drop queue"); + DRV_LOG(WARNING, "port %u cannot allocate CQ for drop queue", + dev->data->port_id); + rte_errno = errno; goto error; } fdq->wq = ibv_create_wq(priv->ctx, @@ -2083,7 +2170,9 @@ priv_flow_create_drop_queue(struct priv *priv) .cq = fdq->cq, }); if (!fdq->wq) { - WARN("cannot allocate WQ for drop queue"); + DRV_LOG(WARNING, "port %u cannot allocate WQ for drop queue", + dev->data->port_id); + rte_errno = errno; goto error; } fdq->ind_table = ibv_create_rwq_ind_table(priv->ctx, @@ -2093,7 +2182,11 @@ priv_flow_create_drop_queue(struct priv *priv) .comp_mask = 0, }); if (!fdq->ind_table) { - WARN("cannot allocate indirection table for drop queue"); + DRV_LOG(WARNING, + "port %u cannot allocate indirection table for drop" + " queue", + dev->data->port_id); + rte_errno = errno; goto error; } fdq->qp = ibv_create_qp_ex(priv->ctx, @@ -2114,7 +2207,9 @@ priv_flow_create_drop_queue(struct priv *priv) .pd = priv->pd }); if (!fdq->qp) { - WARN("cannot allocate QP for drop queue"); + DRV_LOG(WARNING, "port %u cannot allocate QP for drop queue", + dev->data->port_id); + rte_errno = errno; goto error; } priv->flow_drop_queue = fdq; @@ -2131,18 +2226,19 @@ error: if (fdq) rte_free(fdq); priv->flow_drop_queue = NULL; - return -1; + return -rte_errno; } /** * Delete drop queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. */ void -priv_flow_delete_drop_queue(struct priv *priv) +mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue; if (!fdq) @@ -2162,14 +2258,15 @@ priv_flow_delete_drop_queue(struct priv *priv) /** * Remove all flows. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param list * Pointer to a TAILQ flow list. */ void -priv_flow_stop(struct priv *priv, struct mlx5_flows *list) +mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list) { + struct priv *priv = dev->data->dev_private; struct rte_flow *flow; TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) { @@ -2182,7 +2279,8 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list) claim_zero(ibv_destroy_flow (flow->frxq[HASH_RXQ_ETH].ibv_flow)); flow->frxq[HASH_RXQ_ETH].ibv_flow = NULL; - DEBUG("Flow %p removed", (void *)flow); + DRV_LOG(DEBUG, "port %u flow %p removed", + dev->data->port_id, (void *)flow); /* Next flow. */ continue; } @@ -2211,27 +2309,29 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list) continue; claim_zero(ibv_destroy_flow(flow->frxq[i].ibv_flow)); flow->frxq[i].ibv_flow = NULL; - mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq); + mlx5_hrxq_release(dev, flow->frxq[i].hrxq); flow->frxq[i].hrxq = NULL; } - DEBUG("Flow %p removed", (void *)flow); + DRV_LOG(DEBUG, "port %u flow %p removed", dev->data->port_id, + (void *)flow); } } /** * Add all flows. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param list * Pointer to a TAILQ flow list. * * @return - * 0 on success, a errno value otherwise and rte_errno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_flow_start(struct priv *priv, struct mlx5_flows *list) +mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list) { + struct priv *priv = dev->data->dev_private; struct rte_flow *flow; TAILQ_FOREACH(flow, list, next) { @@ -2243,12 +2343,14 @@ priv_flow_start(struct priv *priv, struct mlx5_flows *list) (priv->flow_drop_queue->qp, flow->frxq[HASH_RXQ_ETH].ibv_attr); if (!flow->frxq[HASH_RXQ_ETH].ibv_flow) { - DEBUG("Flow %p cannot be applied", - (void *)flow); + DRV_LOG(DEBUG, + "port %u flow %p cannot be applied", + dev->data->port_id, (void *)flow); rte_errno = EINVAL; - return rte_errno; + return -rte_errno; } - DEBUG("Flow %p applied", (void *)flow); + DRV_LOG(DEBUG, "port %u flow %p applied", + dev->data->port_id, (void *)flow); /* Next flow. */ continue; } @@ -2256,36 +2358,39 @@ priv_flow_start(struct priv *priv, struct mlx5_flows *list) if (!flow->frxq[i].ibv_attr) continue; flow->frxq[i].hrxq = - mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key, - flow->rss_conf.rss_key_len, - hash_rxq_init[i].hash_fields, - (*flow->queues), - flow->queues_n); + mlx5_hrxq_get(dev, flow->rss_conf.rss_key, + flow->rss_conf.rss_key_len, + hash_rxq_init[i].hash_fields, + (*flow->queues), + flow->queues_n); if (flow->frxq[i].hrxq) goto flow_create; flow->frxq[i].hrxq = - mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key, - flow->rss_conf.rss_key_len, - hash_rxq_init[i].hash_fields, - (*flow->queues), - flow->queues_n); + mlx5_hrxq_new(dev, flow->rss_conf.rss_key, + flow->rss_conf.rss_key_len, + hash_rxq_init[i].hash_fields, + (*flow->queues), + flow->queues_n); if (!flow->frxq[i].hrxq) { - DEBUG("Flow %p cannot be applied", - (void *)flow); + DRV_LOG(DEBUG, + "port %u flow %p cannot be applied", + dev->data->port_id, (void *)flow); rte_errno = EINVAL; - return rte_errno; + return -rte_errno; } flow_create: flow->frxq[i].ibv_flow = ibv_create_flow(flow->frxq[i].hrxq->qp, flow->frxq[i].ibv_attr); if (!flow->frxq[i].ibv_flow) { - DEBUG("Flow %p cannot be applied", - (void *)flow); + DRV_LOG(DEBUG, + "port %u flow %p cannot be applied", + dev->data->port_id, (void *)flow); rte_errno = EINVAL; - return rte_errno; + return -rte_errno; } - DEBUG("Flow %p applied", (void *)flow); + DRV_LOG(DEBUG, "port %u flow %p applied", + dev->data->port_id, (void *)flow); } if (!flow->mark) continue; @@ -2298,20 +2403,21 @@ flow_create: /** * Verify the flow list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return the number of flows not released. */ int -priv_flow_verify(struct priv *priv) +mlx5_flow_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct rte_flow *flow; int ret = 0; TAILQ_FOREACH(flow, &priv->flows, next) { - DEBUG("%p: flow %p still referenced", (void *)priv, - (void *)flow); + DRV_LOG(DEBUG, "port %u flow %p still referenced", + dev->data->port_id, (void *)flow); ++ret; } return ret; @@ -2332,7 +2438,7 @@ priv_flow_verify(struct priv *priv) * A VLAN flow mask to apply. * * @return - * 0 on success. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, @@ -2384,17 +2490,19 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, } local; } action_rss; - if (!priv->reta_idx_n) - return EINVAL; + if (!priv->reta_idx_n) { + rte_errno = EINVAL; + return -rte_errno; + } for (i = 0; i != priv->reta_idx_n; ++i) action_rss.local.queue[i] = (*priv->reta_idx)[i]; action_rss.local.rss_conf = &priv->rss_conf; action_rss.local.num = priv->reta_idx_n; actions[0].conf = (const void *)&action_rss.rss; - flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions, - &error); + flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items, + actions, &error); if (!flow) - return rte_errno; + return -rte_errno; return 0; } @@ -2409,7 +2517,7 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev, * An Ethernet flow mask to apply. * * @return - * 0 on success. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_ctrl_flow(struct rte_eth_dev *dev, @@ -2428,14 +2536,11 @@ mlx5_ctrl_flow(struct rte_eth_dev *dev, int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow, - struct rte_flow_error *error) + struct rte_flow_error *error __rte_unused) { struct priv *priv = dev->data->dev_private; - (void)error; - priv_lock(priv); - priv_flow_destroy(priv, &priv->flows, flow); - priv_unlock(priv); + mlx5_flow_list_destroy(dev, &priv->flows, flow); return 0; } @@ -2447,14 +2552,11 @@ mlx5_flow_destroy(struct rte_eth_dev *dev, */ int mlx5_flow_flush(struct rte_eth_dev *dev, - struct rte_flow_error *error) + struct rte_flow_error *error __rte_unused) { struct priv *priv = dev->data->dev_private; - (void)error; - priv_lock(priv); - priv_flow_flush(priv, &priv->flows); - priv_unlock(priv); + mlx5_flow_list_flush(dev, &priv->flows); return 0; } @@ -2468,10 +2570,10 @@ mlx5_flow_flush(struct rte_eth_dev *dev, * returned data from the counter. * * @return - * 0 on success, a errno value otherwise and rte_errno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_flow_query_count(struct ibv_counter_set *cs, +mlx5_flow_query_count(struct ibv_counter_set *cs, struct mlx5_flow_counter_stats *counter_stats, struct rte_flow_query_count *query_count, struct rte_flow_error *error) @@ -2485,15 +2587,13 @@ priv_flow_query_count(struct ibv_counter_set *cs, .out = counters, .outlen = 2 * sizeof(uint64_t), }; - int res = ibv_query_counter_set(&query_cs_attr, &query_out); + int err = ibv_query_counter_set(&query_cs_attr, &query_out); - if (res) { - rte_flow_error_set(error, -res, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "cannot read counter"); - return -res; - } + if (err) + return rte_flow_error_set(error, err, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "cannot read counter"); query_count->hits_set = 1; query_count->bytes_set = 1; query_count->hits = counters[0] - counter_stats->hits; @@ -2512,29 +2612,28 @@ priv_flow_query_count(struct ibv_counter_set *cs, * @see rte_flow_ops */ int -mlx5_flow_query(struct rte_eth_dev *dev, +mlx5_flow_query(struct rte_eth_dev *dev __rte_unused, struct rte_flow *flow, enum rte_flow_action_type action __rte_unused, void *data, struct rte_flow_error *error) { - struct priv *priv = dev->data->dev_private; - int res = EINVAL; - - priv_lock(priv); if (flow->cs) { - res = priv_flow_query_count(flow->cs, - &flow->counter_stats, - (struct rte_flow_query_count *)data, - error); + int ret; + + ret = mlx5_flow_query_count(flow->cs, + &flow->counter_stats, + (struct rte_flow_query_count *)data, + error); + if (ret) + return ret; } else { - rte_flow_error_set(error, res, - RTE_FLOW_ERROR_TYPE_UNSPECIFIED, - NULL, - "no counter found for flow"); + return rte_flow_error_set(error, EINVAL, + RTE_FLOW_ERROR_TYPE_UNSPECIFIED, + NULL, + "no counter found for flow"); } - priv_unlock(priv); - return -res; + return 0; } #endif @@ -2551,48 +2650,50 @@ mlx5_flow_isolate(struct rte_eth_dev *dev, { struct priv *priv = dev->data->dev_private; - priv_lock(priv); if (dev->data->dev_started) { rte_flow_error_set(error, EBUSY, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "port must be stopped first"); - priv_unlock(priv); return -rte_errno; } priv->isolated = !!enable; if (enable) - priv->dev->dev_ops = &mlx5_dev_ops_isolate; + dev->dev_ops = &mlx5_dev_ops_isolate; else - priv->dev->dev_ops = &mlx5_dev_ops; - priv_unlock(priv); + dev->dev_ops = &mlx5_dev_ops; return 0; } /** * Convert a flow director filter to a generic flow. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. * @param fdir_filter * Flow director filter to add. * @param attributes * Generic flow parameters structure. * * @return - * 0 on success, errno value on error. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_fdir_filter_convert(struct priv *priv, +mlx5_fdir_filter_convert(struct rte_eth_dev *dev, const struct rte_eth_fdir_filter *fdir_filter, struct mlx5_fdir *attributes) { + struct priv *priv = dev->data->dev_private; const struct rte_eth_fdir_input *input = &fdir_filter->input; + const struct rte_eth_fdir_masks *mask = + &dev->data->dev_conf.fdir_conf.mask; /* Validate queue number. */ if (fdir_filter->action.rx_queue >= priv->rxqs_n) { - ERROR("invalid queue number %d", fdir_filter->action.rx_queue); - return EINVAL; + DRV_LOG(ERR, "port %u invalid queue number %d", + dev->data->port_id, fdir_filter->action.rx_queue); + rte_errno = EINVAL; + return -rte_errno; } attributes->attr.ingress = 1; attributes->items[0] = (struct rte_flow_item) { @@ -2613,53 +2714,17 @@ priv_fdir_filter_convert(struct priv *priv, }; break; default: - ERROR("invalid behavior %d", fdir_filter->action.behavior); - return ENOTSUP; + DRV_LOG(ERR, "port %u invalid behavior %d", + dev->data->port_id, + fdir_filter->action.behavior); + rte_errno = ENOTSUP; + return -rte_errno; } attributes->queue.index = fdir_filter->action.rx_queue; + /* Handle L3. */ switch (fdir_filter->input.flow_type) { case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: - attributes->l3.ipv4.hdr = (struct ipv4_hdr){ - .src_addr = input->flow.udp4_flow.ip.src_ip, - .dst_addr = input->flow.udp4_flow.ip.dst_ip, - .time_to_live = input->flow.udp4_flow.ip.ttl, - .type_of_service = input->flow.udp4_flow.ip.tos, - .next_proto_id = input->flow.udp4_flow.ip.proto, - }; - attributes->l4.udp.hdr = (struct udp_hdr){ - .src_port = input->flow.udp4_flow.src_port, - .dst_port = input->flow.udp4_flow.dst_port, - }; - attributes->items[1] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_IPV4, - .spec = &attributes->l3, - }; - attributes->items[2] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_UDP, - .spec = &attributes->l4, - }; - break; case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: - attributes->l3.ipv4.hdr = (struct ipv4_hdr){ - .src_addr = input->flow.tcp4_flow.ip.src_ip, - .dst_addr = input->flow.tcp4_flow.ip.dst_ip, - .time_to_live = input->flow.tcp4_flow.ip.ttl, - .type_of_service = input->flow.tcp4_flow.ip.tos, - .next_proto_id = input->flow.tcp4_flow.ip.proto, - }; - attributes->l4.tcp.hdr = (struct tcp_hdr){ - .src_port = input->flow.tcp4_flow.src_port, - .dst_port = input->flow.tcp4_flow.dst_port, - }; - attributes->items[1] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_IPV4, - .spec = &attributes->l3, - }; - attributes->items[2] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_TCP, - .spec = &attributes->l4, - }; - break; case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: attributes->l3.ipv4.hdr = (struct ipv4_hdr){ .src_addr = input->flow.ip4_flow.src_ip, @@ -2668,79 +2733,121 @@ priv_fdir_filter_convert(struct priv *priv, .type_of_service = input->flow.ip4_flow.tos, .next_proto_id = input->flow.ip4_flow.proto, }; + attributes->l3_mask.ipv4.hdr = (struct ipv4_hdr){ + .src_addr = mask->ipv4_mask.src_ip, + .dst_addr = mask->ipv4_mask.dst_ip, + .time_to_live = mask->ipv4_mask.ttl, + .type_of_service = mask->ipv4_mask.tos, + .next_proto_id = mask->ipv4_mask.proto, + }; attributes->items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_IPV4, .spec = &attributes->l3, + .mask = &attributes->l3_mask, }; break; case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: - attributes->l3.ipv6.hdr = (struct ipv6_hdr){ - .hop_limits = input->flow.udp6_flow.ip.hop_limits, - .proto = input->flow.udp6_flow.ip.proto, - }; - memcpy(attributes->l3.ipv6.hdr.src_addr, - input->flow.udp6_flow.ip.src_ip, - RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); - memcpy(attributes->l3.ipv6.hdr.dst_addr, - input->flow.udp6_flow.ip.dst_ip, - RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); - attributes->l4.udp.hdr = (struct udp_hdr){ - .src_port = input->flow.udp6_flow.src_port, - .dst_port = input->flow.udp6_flow.dst_port, - }; - attributes->items[1] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_IPV6, - .spec = &attributes->l3, - }; - attributes->items[2] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_UDP, - .spec = &attributes->l4, - }; - break; case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: - attributes->l3.ipv6.hdr = (struct ipv6_hdr){ - .hop_limits = input->flow.tcp6_flow.ip.hop_limits, - .proto = input->flow.tcp6_flow.ip.proto, - }; - memcpy(attributes->l3.ipv6.hdr.src_addr, - input->flow.tcp6_flow.ip.src_ip, - RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); - memcpy(attributes->l3.ipv6.hdr.dst_addr, - input->flow.tcp6_flow.ip.dst_ip, - RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); - attributes->l4.tcp.hdr = (struct tcp_hdr){ - .src_port = input->flow.tcp6_flow.src_port, - .dst_port = input->flow.tcp6_flow.dst_port, - }; - attributes->items[1] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_IPV6, - .spec = &attributes->l3, - }; - attributes->items[2] = (struct rte_flow_item){ - .type = RTE_FLOW_ITEM_TYPE_TCP, - .spec = &attributes->l4, - }; - break; case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: attributes->l3.ipv6.hdr = (struct ipv6_hdr){ .hop_limits = input->flow.ipv6_flow.hop_limits, .proto = input->flow.ipv6_flow.proto, }; + memcpy(attributes->l3.ipv6.hdr.src_addr, input->flow.ipv6_flow.src_ip, RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); memcpy(attributes->l3.ipv6.hdr.dst_addr, input->flow.ipv6_flow.dst_ip, RTE_DIM(attributes->l3.ipv6.hdr.src_addr)); + memcpy(attributes->l3_mask.ipv6.hdr.src_addr, + mask->ipv6_mask.src_ip, + RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); + memcpy(attributes->l3_mask.ipv6.hdr.dst_addr, + mask->ipv6_mask.dst_ip, + RTE_DIM(attributes->l3_mask.ipv6.hdr.src_addr)); attributes->items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_IPV6, .spec = &attributes->l3, + .mask = &attributes->l3_mask, }; break; default: - ERROR("invalid flow type%d", - fdir_filter->input.flow_type); - return ENOTSUP; + DRV_LOG(ERR, "port %u invalid flow type%d", + dev->data->port_id, fdir_filter->input.flow_type); + rte_errno = ENOTSUP; + return -rte_errno; + } + /* Handle L4. */ + switch (fdir_filter->input.flow_type) { + case RTE_ETH_FLOW_NONFRAG_IPV4_UDP: + attributes->l4.udp.hdr = (struct udp_hdr){ + .src_port = input->flow.udp4_flow.src_port, + .dst_port = input->flow.udp4_flow.dst_port, + }; + attributes->l4_mask.udp.hdr = (struct udp_hdr){ + .src_port = mask->src_port_mask, + .dst_port = mask->dst_port_mask, + }; + attributes->items[2] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_UDP, + .spec = &attributes->l4, + .mask = &attributes->l4_mask, + }; + break; + case RTE_ETH_FLOW_NONFRAG_IPV4_TCP: + attributes->l4.tcp.hdr = (struct tcp_hdr){ + .src_port = input->flow.tcp4_flow.src_port, + .dst_port = input->flow.tcp4_flow.dst_port, + }; + attributes->l4_mask.tcp.hdr = (struct tcp_hdr){ + .src_port = mask->src_port_mask, + .dst_port = mask->dst_port_mask, + }; + attributes->items[2] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_TCP, + .spec = &attributes->l4, + .mask = &attributes->l4_mask, + }; + break; + case RTE_ETH_FLOW_NONFRAG_IPV6_UDP: + attributes->l4.udp.hdr = (struct udp_hdr){ + .src_port = input->flow.udp6_flow.src_port, + .dst_port = input->flow.udp6_flow.dst_port, + }; + attributes->l4_mask.udp.hdr = (struct udp_hdr){ + .src_port = mask->src_port_mask, + .dst_port = mask->dst_port_mask, + }; + attributes->items[2] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_UDP, + .spec = &attributes->l4, + .mask = &attributes->l4_mask, + }; + break; + case RTE_ETH_FLOW_NONFRAG_IPV6_TCP: + attributes->l4.tcp.hdr = (struct tcp_hdr){ + .src_port = input->flow.tcp6_flow.src_port, + .dst_port = input->flow.tcp6_flow.dst_port, + }; + attributes->l4_mask.tcp.hdr = (struct tcp_hdr){ + .src_port = mask->src_port_mask, + .dst_port = mask->dst_port_mask, + }; + attributes->items[2] = (struct rte_flow_item){ + .type = RTE_FLOW_ITEM_TYPE_TCP, + .spec = &attributes->l4, + .mask = &attributes->l4_mask, + }; + break; + case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER: + case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER: + break; + default: + DRV_LOG(ERR, "port %u invalid flow type%d", + dev->data->port_id, fdir_filter->input.flow_type); + rte_errno = ENOTSUP; + return -rte_errno; } return 0; } @@ -2748,18 +2855,19 @@ priv_fdir_filter_convert(struct priv *priv, /** * Add new flow director filter and store it in list. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. * @param fdir_filter * Flow director filter to add. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_fdir_filter_add(struct priv *priv, +mlx5_fdir_filter_add(struct rte_eth_dev *dev, const struct rte_eth_fdir_filter *fdir_filter) { + struct priv *priv = dev->data->dev_private; struct mlx5_fdir attributes = { .attr.group = 0, .l2_mask = { @@ -2775,41 +2883,40 @@ priv_fdir_filter_add(struct priv *priv, struct rte_flow *flow; int ret; - ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); + ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes); if (ret) - return -ret; - ret = priv_flow_convert(priv, &attributes.attr, attributes.items, + return ret; + ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items, attributes.actions, &error, &parser); if (ret) - return -ret; - flow = priv_flow_create(priv, - &priv->flows, - &attributes.attr, - attributes.items, - attributes.actions, - &error); + return ret; + flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr, + attributes.items, attributes.actions, + &error); if (flow) { - DEBUG("FDIR created %p", (void *)flow); + DRV_LOG(DEBUG, "port %u FDIR created %p", dev->data->port_id, + (void *)flow); return 0; } - return ENOTSUP; + return -rte_errno; } /** * Delete specific filter. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. * @param fdir_filter * Filter to be deleted. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_fdir_filter_delete(struct priv *priv, +mlx5_fdir_filter_delete(struct rte_eth_dev *dev, const struct rte_eth_fdir_filter *fdir_filter) { + struct priv *priv = dev->data->dev_private; struct mlx5_fdir attributes = { .attr.group = 0, }; @@ -2822,10 +2929,10 @@ priv_fdir_filter_delete(struct priv *priv, unsigned int i; int ret; - ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes); + ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes); if (ret) - return -ret; - ret = priv_flow_convert(priv, &attributes.attr, attributes.items, + return ret; + ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items, attributes.actions, &error, &parser); if (ret) goto exit; @@ -2853,11 +2960,25 @@ priv_fdir_filter_delete(struct priv *priv, struct ibv_spec_header *flow_h; void *flow_spec; unsigned int specs_n; + unsigned int queue_id; - attr = parser.queue[HASH_RXQ_ETH].ibv_attr; - flow_attr = flow->frxq[HASH_RXQ_ETH].ibv_attr; + /* + * Search for a non-empty ibv_attr. There should be only one + * because no RSS action is allowed for FDIR. This should have + * been referenced directly by parser.layer but due to a bug in + * mlx5_flow_convert() as of v17.11.4, parser.layer isn't + * correct. This bug will have to be addressed later. + */ + for (queue_id = 0; queue_id != hash_rxq_init_n; ++queue_id) { + attr = parser.queue[queue_id].ibv_attr; + if (attr) + break; + } + assert(!parser.drop || queue_id == HASH_RXQ_ETH); + flow_attr = flow->frxq[queue_id].ibv_attr; /* Compare first the attributes. */ - if (memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr))) + if (!flow_attr || + memcmp(attr, flow_attr, sizeof(struct ibv_flow_attr))) continue; if (attr->num_of_specs == 0) continue; @@ -2883,66 +3004,73 @@ wrong_flow: continue; } if (flow) - priv_flow_destroy(priv, &priv->flows, flow); + mlx5_flow_list_destroy(dev, &priv->flows, flow); exit: + if (ret) + ret = rte_errno; /* Save rte_errno before cleanup. */ for (i = 0; i != hash_rxq_init_n; ++i) { if (parser.queue[i].ibv_attr) rte_free(parser.queue[i].ibv_attr); } - return -ret; + if (ret) { + rte_errno = ret; /* Restore rte_errno. */ + return -rte_errno; + } + return 0; } /** * Update queue for specific filter. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. * @param fdir_filter * Filter to be updated. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_fdir_filter_update(struct priv *priv, +mlx5_fdir_filter_update(struct rte_eth_dev *dev, const struct rte_eth_fdir_filter *fdir_filter) { int ret; - ret = priv_fdir_filter_delete(priv, fdir_filter); + ret = mlx5_fdir_filter_delete(dev, fdir_filter); if (ret) return ret; - ret = priv_fdir_filter_add(priv, fdir_filter); - return ret; + return mlx5_fdir_filter_add(dev, fdir_filter); } /** * Flush all filters. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. */ static void -priv_fdir_filter_flush(struct priv *priv) +mlx5_fdir_filter_flush(struct rte_eth_dev *dev) { - priv_flow_flush(priv, &priv->flows); + struct priv *priv = dev->data->dev_private; + + mlx5_flow_list_flush(dev, &priv->flows); } /** * Get flow director information. * - * @param priv - * Private structure. + * @param dev + * Pointer to Ethernet device. * @param[out] fdir_info * Resulting flow director information. */ static void -priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) +mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info) { struct rte_eth_fdir_masks *mask = - &priv->dev->data->dev_conf.fdir_conf.mask; + &dev->data->dev_conf.fdir_conf.mask; - fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode; + fdir_info->mode = dev->data->dev_conf.fdir_conf.mode; fdir_info->guarant_spc = 0; rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask)); fdir_info->max_flexpayload = 0; @@ -2956,54 +3084,52 @@ priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info) /** * Deal with flow director operations. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param filter_op * Operation to perform. * @param arg * Pointer to operation-specific structure. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ static int -priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg) +mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op, + void *arg) { enum rte_fdir_mode fdir_mode = - priv->dev->data->dev_conf.fdir_conf.mode; - int ret = 0; + dev->data->dev_conf.fdir_conf.mode; if (filter_op == RTE_ETH_FILTER_NOP) return 0; if (fdir_mode != RTE_FDIR_MODE_PERFECT && fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) { - ERROR("%p: flow director mode %d not supported", - (void *)priv, fdir_mode); - return EINVAL; + DRV_LOG(ERR, "port %u flow director mode %d not supported", + dev->data->port_id, fdir_mode); + rte_errno = EINVAL; + return -rte_errno; } switch (filter_op) { case RTE_ETH_FILTER_ADD: - ret = priv_fdir_filter_add(priv, arg); - break; + return mlx5_fdir_filter_add(dev, arg); case RTE_ETH_FILTER_UPDATE: - ret = priv_fdir_filter_update(priv, arg); - break; + return mlx5_fdir_filter_update(dev, arg); case RTE_ETH_FILTER_DELETE: - ret = priv_fdir_filter_delete(priv, arg); - break; + return mlx5_fdir_filter_delete(dev, arg); case RTE_ETH_FILTER_FLUSH: - priv_fdir_filter_flush(priv); + mlx5_fdir_filter_flush(dev); break; case RTE_ETH_FILTER_INFO: - priv_fdir_info_get(priv, arg); + mlx5_fdir_info_get(dev, arg); break; default: - DEBUG("%p: unknown operation %u", (void *)priv, - filter_op); - ret = EINVAL; - break; + DRV_LOG(DEBUG, "port %u unknown operation %u", + dev->data->port_id, filter_op); + rte_errno = EINVAL; + return -rte_errno; } - return ret; + return 0; } /** @@ -3019,7 +3145,7 @@ priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg) * Pointer to operation-specific structure. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, @@ -3027,24 +3153,21 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_op filter_op, void *arg) { - int ret = EINVAL; - struct priv *priv = dev->data->dev_private; - switch (filter_type) { case RTE_ETH_FILTER_GENERIC: - if (filter_op != RTE_ETH_FILTER_GET) - return -EINVAL; + if (filter_op != RTE_ETH_FILTER_GET) { + rte_errno = EINVAL; + return -rte_errno; + } *(const void **)arg = &mlx5_flow_ops; return 0; case RTE_ETH_FILTER_FDIR: - priv_lock(priv); - ret = priv_fdir_ctrl_func(priv, filter_op, arg); - priv_unlock(priv); - break; + return mlx5_fdir_ctrl_func(dev, filter_op, arg); default: - ERROR("%p: filter type (%d) not supported", - (void *)dev, filter_type); - break; + DRV_LOG(ERR, "port %u filter type (%d) not supported", + dev->data->port_id, filter_type); + rte_errno = ENOTSUP; + return -rte_errno; } - return -ret; + return 0; } diff --git a/dpdk/drivers/net/mlx5/mlx5_mac.c b/dpdk/drivers/net/mlx5/mlx5_mac.c index 9fb5ba5e..9de35142 100644 --- a/dpdk/drivers/net/mlx5/mlx5_mac.c +++ b/dpdk/drivers/net/mlx5/mlx5_mac.c @@ -63,21 +63,23 @@ /** * Get MAC address by querying netdevice. * - * @param[in] priv - * struct priv for the requested device. + * @param[in] dev + * Pointer to Ethernet device. * @param[out] mac * MAC address output buffer. * * @return - * 0 on success, -1 on failure and errno is set. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN]) +mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN]) { struct ifreq request; + int ret; - if (priv_ifreq(priv, SIOCGIFHWADDR, &request)) - return -1; + ret = mlx5_ifreq(dev, SIOCGIFHWADDR, &request); + if (ret) + return ret; memcpy(mac, request.ifr_hwaddr.sa_data, ETHER_ADDR_LEN); return 0; } @@ -95,8 +97,13 @@ mlx5_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) { assert(index < MLX5_MAX_MAC_ADDRESSES); memset(&dev->data->mac_addrs[index], 0, sizeof(struct ether_addr)); - if (!dev->data->promiscuous && !dev->data->all_multicast) - mlx5_traffic_restart(dev); + if (!dev->data->promiscuous) { + int ret = mlx5_traffic_restart(dev); + + if (ret) + DRV_LOG(ERR, "port %u cannot remove mac address: %s", + dev->data->port_id, strerror(rte_errno)); + } } /** @@ -112,16 +119,14 @@ mlx5_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) * VMDq pool index to associate address with (ignored). * * @return - * 0 on success. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac, - uint32_t index, uint32_t vmdq) + uint32_t index, uint32_t vmdq __rte_unused) { unsigned int i; - int ret = 0; - (void)vmdq; assert(index < MLX5_MAX_MAC_ADDRESSES); /* First, make sure this address isn't already configured. */ for (i = 0; (i != MLX5_MAX_MAC_ADDRESSES); ++i) { @@ -131,12 +136,13 @@ mlx5_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac, if (memcmp(&dev->data->mac_addrs[i], mac, sizeof(*mac))) continue; /* Address already configured elsewhere, return with error. */ - return EADDRINUSE; + rte_errno = EADDRINUSE; + return -rte_errno; } dev->data->mac_addrs[index] = *mac; - if (!dev->data->promiscuous && !dev->data->all_multicast) - mlx5_traffic_restart(dev); - return ret; + if (!dev->data->promiscuous) + return mlx5_traffic_restart(dev); + return 0; } /** @@ -150,6 +156,13 @@ mlx5_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac, void mlx5_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) { - DEBUG("%p: setting primary MAC address", (void *)dev); - mlx5_mac_addr_add(dev, mac_addr, 0, 0); + int ret; + + DRV_LOG(DEBUG, "port %u setting primary MAC address", + dev->data->port_id); + + ret = mlx5_mac_addr_add(dev, mac_addr, 0, 0); + if (ret) + DRV_LOG(ERR, "port %u cannot set mac address: %s", + dev->data->port_id, strerror(rte_errno)); } diff --git a/dpdk/drivers/net/mlx5/mlx5_mr.c b/dpdk/drivers/net/mlx5/mlx5_mr.c index 2776dc70..a50c5208 100644 --- a/dpdk/drivers/net/mlx5/mlx5_mr.c +++ b/dpdk/drivers/net/mlx5/mlx5_mr.c @@ -55,15 +55,12 @@ struct mlx5_check_mempool_data { /* Called by mlx5_check_mempool() when iterating the memory chunks. */ static void -mlx5_check_mempool_cb(struct rte_mempool *mp, +mlx5_check_mempool_cb(struct rte_mempool *mp __rte_unused, void *opaque, struct rte_mempool_memhdr *memhdr, - unsigned int mem_idx) + unsigned int mem_idx __rte_unused) { struct mlx5_check_mempool_data *data = opaque; - (void)mp; - (void)mem_idx; - /* It already failed, skip the next chunks. */ if (data->ret != 0) return; @@ -98,8 +95,9 @@ mlx5_check_mempool_cb(struct rte_mempool *mp, * @return * 0 on success (mempool is virtually contiguous), -1 on error. */ -static int mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start, - uintptr_t *end) +static int +mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start, + uintptr_t *end) { struct mlx5_check_mempool_data data; @@ -107,7 +105,6 @@ static int mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start, rte_mempool_mem_iter(mp, mlx5_check_mempool_cb, &data); *start = (uintptr_t)data.start; *end = (uintptr_t)data.end; - return data.ret; } @@ -115,10 +112,6 @@ static int mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start, * Register a Memory Region (MR) <-> Memory Pool (MP) association in * txq->mp2mr[]. If mp2mr[] is full, remove an entry first. * - * This function should only be called by txq_mp2mr(). - * - * @param priv - * Pointer to private structure. * @param txq * Pointer to TX queue structure. * @param[in] mp @@ -127,71 +120,63 @@ static int mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start, * Index of the next available entry. * * @return - * mr on success, NULL on failure. + * mr on success, NULL on failure and rte_errno is set. */ -struct mlx5_mr* -priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq, - struct rte_mempool *mp, unsigned int idx) -{ - struct mlx5_txq_ctrl *txq_ctrl = - container_of(txq, struct mlx5_txq_ctrl, txq); - struct mlx5_mr *mr; - - /* Add a new entry, register MR first. */ - DEBUG("%p: discovered new memory pool \"%s\" (%p)", - (void *)txq_ctrl, mp->name, (void *)mp); - mr = priv_mr_get(priv, mp); - if (mr == NULL) - mr = priv_mr_new(priv, mp); - if (unlikely(mr == NULL)) { - DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.", - (void *)txq_ctrl); - return NULL; - } - if (unlikely(idx == RTE_DIM(txq->mp2mr))) { - /* Table is full, remove oldest entry. */ - DEBUG("%p: MR <-> MP table full, dropping oldest entry.", - (void *)txq_ctrl); - --idx; - priv_mr_release(priv, txq->mp2mr[0]); - memmove(&txq->mp2mr[0], &txq->mp2mr[1], - (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0]))); - } - /* Store the new entry. */ - txq_ctrl->txq.mp2mr[idx] = mr; - DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32, - (void *)txq_ctrl, mp->name, (void *)mp, - txq_ctrl->txq.mp2mr[idx]->lkey); - return mr; -} - -/** - * Register a Memory Region (MR) <-> Memory Pool (MP) association in - * txq->mp2mr[]. If mp2mr[] is full, remove an entry first. - * - * This function should only be called by txq_mp2mr(). - * - * @param txq - * Pointer to TX queue structure. - * @param[in] mp - * Memory Pool for which a Memory Region lkey must be returned. - * @param idx - * Index of the next available entry. - * - * @return - * mr on success, NULL on failure. - */ -struct mlx5_mr* +struct mlx5_mr * mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp, unsigned int idx) { struct mlx5_txq_ctrl *txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); + struct rte_eth_dev *dev; struct mlx5_mr *mr; - priv_lock(txq_ctrl->priv); - mr = priv_txq_mp2mr_reg(txq_ctrl->priv, txq, mp, idx); - priv_unlock(txq_ctrl->priv); + rte_spinlock_lock(&txq_ctrl->priv->mr_lock); + /* Add a new entry, register MR first. */ + DRV_LOG(DEBUG, "port %u discovered new memory pool \"%s\" (%p)", + PORT_ID(txq_ctrl->priv), mp->name, (void *)mp); + dev = ETH_DEV(txq_ctrl->priv); + mr = mlx5_mr_get(dev, mp); + if (mr == NULL) { + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + DRV_LOG(DEBUG, + "port %u using unregistered mempool 0x%p(%s)" + " in secondary process, please create mempool" + " before rte_eth_dev_start()", + PORT_ID(txq_ctrl->priv), (void *)mp, mp->name); + rte_spinlock_unlock(&txq_ctrl->priv->mr_lock); + rte_errno = ENOTSUP; + return NULL; + } + mr = mlx5_mr_new(dev, mp); + } + if (unlikely(mr == NULL)) { + DRV_LOG(DEBUG, + "port %u unable to configure memory region," + " ibv_reg_mr() failed.", + PORT_ID(txq_ctrl->priv)); + rte_spinlock_unlock(&txq_ctrl->priv->mr_lock); + return NULL; + } + if (unlikely(idx == RTE_DIM(txq->mp2mr))) { + /* Table is full, remove oldest entry. */ + DRV_LOG(DEBUG, + "port %u memory region <-> memory pool table full, " + " dropping oldest entry", + PORT_ID(txq_ctrl->priv)); + --idx; + mlx5_mr_release(txq->mp2mr[0]); + memmove(&txq->mp2mr[0], &txq->mp2mr[1], + (sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0]))); + } + /* Store the new entry. */ + txq_ctrl->txq.mp2mr[idx] = mr; + DRV_LOG(DEBUG, + "port %u new memory region lkey for MP \"%s\" (%p): 0x%08" + PRIu32, + PORT_ID(txq_ctrl->priv), mp->name, (void *)mp, + txq_ctrl->txq.mp2mr[idx]->lkey); + rte_spinlock_unlock(&txq_ctrl->priv->mr_lock); return mr; } @@ -250,28 +235,33 @@ mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg) if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 || data.ret == -1) return; - mr = priv_mr_get(priv, mp); + mr = mlx5_mr_get(ETH_DEV(priv), mp); if (mr) { - priv_mr_release(priv, mr); + mlx5_mr_release(mr); return; } - priv_mr_new(priv, mp); + mr = mlx5_mr_new(ETH_DEV(priv), mp); + if (!mr) + DRV_LOG(ERR, "port %u cannot create memory region: %s", + PORT_ID(priv), strerror(rte_errno)); } /** * Register a new memory region from the mempool and store it in the memory * region list. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param mp * Pointer to the memory pool to register. + * * @return - * The memory region on success. + * The memory region on success, NULL on failure and rte_errno is set. */ -struct mlx5_mr* -priv_mr_new(struct priv *priv, struct rte_mempool *mp) +struct mlx5_mr * +mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp) { + struct priv *priv = dev->data->dev_private; const struct rte_memseg *ms = rte_eal_get_physmem_layout(); uintptr_t start; uintptr_t end; @@ -280,17 +270,22 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp) mr = rte_zmalloc_socket(__func__, sizeof(*mr), 0, mp->socket_id); if (!mr) { - DEBUG("unable to configure MR, ibv_reg_mr() failed."); + DRV_LOG(DEBUG, + "port %u unable to configure memory region," + " ibv_reg_mr() failed.", + dev->data->port_id); + rte_errno = ENOMEM; return NULL; } if (mlx5_check_mempool(mp, &start, &end) != 0) { - ERROR("mempool %p: not virtually contiguous", - (void *)mp); + DRV_LOG(ERR, "port %u mempool %p: not virtually contiguous", + dev->data->port_id, (void *)mp); + rte_errno = ENOMEM; return NULL; } - DEBUG("mempool %p area start=%p end=%p size=%zu", - (void *)mp, (void *)start, (void *)end, - (size_t)(end - start)); + DRV_LOG(DEBUG, "port %u mempool %p area start=%p end=%p size=%zu", + dev->data->port_id, (void *)mp, (void *)start, (void *)end, + (size_t)(end - start)); /* Save original addresses for exact MR lookup. */ mr->start = start; mr->end = end; @@ -305,16 +300,22 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp) if ((end > addr) && (end < addr + len)) end = RTE_ALIGN_CEIL(end, align); } - DEBUG("mempool %p using start=%p end=%p size=%zu for MR", - (void *)mp, (void *)start, (void *)end, - (size_t)(end - start)); + DRV_LOG(DEBUG, + "port %u mempool %p using start=%p end=%p size=%zu for memory" + " region", + dev->data->port_id, (void *)mp, (void *)start, (void *)end, + (size_t)(end - start)); mr->mr = ibv_reg_mr(priv->pd, (void *)start, end - start, IBV_ACCESS_LOCAL_WRITE); + if (!mr->mr) { + rte_errno = ENOMEM; + return NULL; + } mr->mp = mp; mr->lkey = rte_cpu_to_be_32(mr->mr->lkey); rte_atomic32_inc(&mr->refcnt); - DEBUG("%p: new Memory Region %p refcnt: %d", (void *)priv, - (void *)mr, rte_atomic32_read(&mr->refcnt)); + DRV_LOG(DEBUG, "port %u new memory Region %p refcnt: %d", + dev->data->port_id, (void *)mr, rte_atomic32_read(&mr->refcnt)); LIST_INSERT_HEAD(&priv->mr, mr, next); return mr; } @@ -322,16 +323,18 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp) /** * Search the memory region object in the memory region list. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param mp * Pointer to the memory pool to register. + * * @return * The memory region on success. */ -struct mlx5_mr* -priv_mr_get(struct priv *priv, struct rte_mempool *mp) +struct mlx5_mr * +mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp) { + struct priv *priv = dev->data->dev_private; struct mlx5_mr *mr; assert(mp); @@ -340,8 +343,9 @@ priv_mr_get(struct priv *priv, struct rte_mempool *mp) LIST_FOREACH(mr, &priv->mr, next) { if (mr->mp == mp) { rte_atomic32_inc(&mr->refcnt); - DEBUG("Memory Region %p refcnt: %d", - (void *)mr, rte_atomic32_read(&mr->refcnt)); + DRV_LOG(DEBUG, "port %u memory region %p refcnt: %d", + dev->data->port_id, (void *)mr, + rte_atomic32_read(&mr->refcnt)); return mr; } } @@ -355,41 +359,42 @@ priv_mr_get(struct priv *priv, struct rte_mempool *mp) * Pointer to memory region to release. * * @return - * 0 on success, errno on failure. + * 1 while a reference on it exists, 0 when freed. */ int -priv_mr_release(struct priv *priv, struct mlx5_mr *mr) +mlx5_mr_release(struct mlx5_mr *mr) { - (void)priv; assert(mr); - DEBUG("Memory Region %p refcnt: %d", - (void *)mr, rte_atomic32_read(&mr->refcnt)); + DRV_LOG(DEBUG, "memory region %p refcnt: %d", (void *)mr, + rte_atomic32_read(&mr->refcnt)); if (rte_atomic32_dec_and_test(&mr->refcnt)) { claim_zero(ibv_dereg_mr(mr->mr)); LIST_REMOVE(mr, next); rte_free(mr); return 0; } - return EBUSY; + return 1; } /** * Verify the flow list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * - * @return the number of object not released. + * @return + * The number of object not released. */ int -priv_mr_verify(struct priv *priv) +mlx5_mr_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; int ret = 0; struct mlx5_mr *mr; LIST_FOREACH(mr, &priv->mr, next) { - DEBUG("%p: mr %p still referenced", (void *)priv, - (void *)mr); + DRV_LOG(DEBUG, "port %u memory region %p still referenced", + dev->data->port_id, (void *)mr); ++ret; } return ret; diff --git a/dpdk/drivers/net/mlx5/mlx5_rss.c b/dpdk/drivers/net/mlx5/mlx5_rss.c index f47bda66..029e0ec4 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rss.c +++ b/dpdk/drivers/net/mlx5/mlx5_rss.c @@ -63,35 +63,48 @@ * RSS configuration data. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_rss_hash_update(struct rte_eth_dev *dev, struct rte_eth_rss_conf *rss_conf) { struct priv *priv = dev->data->dev_private; - int ret = 0; + unsigned int i; + unsigned int idx; - priv_lock(priv); if (rss_conf->rss_hf & MLX5_RSS_HF_MASK) { - ret = -EINVAL; - goto out; + rte_errno = EINVAL; + return -rte_errno; } if (rss_conf->rss_key && rss_conf->rss_key_len) { + if (rss_conf->rss_key_len != rss_hash_default_key_len) { + DRV_LOG(ERR, + "port %u RSS key len must be %zu Bytes long", + dev->data->port_id, rss_hash_default_key_len); + rte_errno = EINVAL; + return -rte_errno; + } priv->rss_conf.rss_key = rte_realloc(priv->rss_conf.rss_key, rss_conf->rss_key_len, 0); if (!priv->rss_conf.rss_key) { - ret = -ENOMEM; - goto out; + rte_errno = ENOMEM; + return -rte_errno; } memcpy(priv->rss_conf.rss_key, rss_conf->rss_key, rss_conf->rss_key_len); priv->rss_conf.rss_key_len = rss_conf->rss_key_len; } priv->rss_conf.rss_hf = rss_conf->rss_hf; -out: - priv_unlock(priv); - return ret; + /* Enable the RSS hash in all Rx queues. */ + for (i = 0, idx = 0; idx != priv->rxqs_n; ++i) { + if (!(*priv->rxqs)[i]) + continue; + (*priv->rxqs)[i]->rss_hash = !!rss_conf->rss_hf && + !!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS); + ++idx; + } + return 0; } /** @@ -103,7 +116,7 @@ out: * RSS configuration data. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_rss_hash_conf_get(struct rte_eth_dev *dev, @@ -111,9 +124,10 @@ mlx5_rss_hash_conf_get(struct rte_eth_dev *dev, { struct priv *priv = dev->data->dev_private; - if (!rss_conf) - return -EINVAL; - priv_lock(priv); + if (!rss_conf) { + rte_errno = EINVAL; + return -rte_errno; + } if (rss_conf->rss_key && (rss_conf->rss_key_len >= priv->rss_conf.rss_key_len)) { memcpy(rss_conf->rss_key, priv->rss_conf.rss_key, @@ -121,24 +135,24 @@ mlx5_rss_hash_conf_get(struct rte_eth_dev *dev, } rss_conf->rss_key_len = priv->rss_conf.rss_key_len; rss_conf->rss_hf = priv->rss_conf.rss_hf; - priv_unlock(priv); return 0; } /** * Allocate/reallocate RETA index table. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @praram reta_size * The size of the array to allocate. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size) +mlx5_rss_reta_index_resize(struct rte_eth_dev *dev, unsigned int reta_size) { + struct priv *priv = dev->data->dev_private; void *mem; unsigned int old_size = priv->reta_idx_n; @@ -147,11 +161,12 @@ priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size) mem = rte_realloc(priv->reta_idx, reta_size * sizeof((*priv->reta_idx)[0]), 0); - if (!mem) - return ENOMEM; + if (!mem) { + rte_errno = ENOMEM; + return -rte_errno; + } priv->reta_idx = mem; priv->reta_idx_n = reta_size; - if (old_size < reta_size) memset(&(*priv->reta_idx)[old_size], 0, (reta_size - old_size) * @@ -159,78 +174,6 @@ priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size) return 0; } -/** - * Query RETA table. - * - * @param priv - * Pointer to private structure. - * @param[in, out] reta_conf - * Pointer to the first RETA configuration structure. - * @param reta_size - * Number of entries. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_dev_rss_reta_query(struct priv *priv, - struct rte_eth_rss_reta_entry64 *reta_conf, - unsigned int reta_size) -{ - unsigned int idx; - unsigned int i; - - if (!reta_size || reta_size > priv->reta_idx_n) - return EINVAL; - /* Fill each entry of the table even if its bit is not set. */ - for (idx = 0, i = 0; (i != reta_size); ++i) { - idx = i / RTE_RETA_GROUP_SIZE; - reta_conf[idx].reta[i % RTE_RETA_GROUP_SIZE] = - (*priv->reta_idx)[i]; - } - return 0; -} - -/** - * Update RETA table. - * - * @param priv - * Pointer to private structure. - * @param[in] reta_conf - * Pointer to the first RETA configuration structure. - * @param reta_size - * Number of entries. - * - * @return - * 0 on success, errno value on failure. - */ -static int -priv_dev_rss_reta_update(struct priv *priv, - struct rte_eth_rss_reta_entry64 *reta_conf, - unsigned int reta_size) -{ - unsigned int idx; - unsigned int i; - unsigned int pos; - int ret; - - if (!reta_size) - return EINVAL; - ret = priv_rss_reta_index_resize(priv, reta_size); - if (ret) - return ret; - - for (idx = 0, i = 0; (i != reta_size); ++i) { - idx = i / RTE_RETA_GROUP_SIZE; - pos = i % RTE_RETA_GROUP_SIZE; - if (((reta_conf[idx].mask >> i) & 0x1) == 0) - continue; - assert(reta_conf[idx].reta[pos] < priv->rxqs_n); - (*priv->reta_idx)[i] = reta_conf[idx].reta[pos]; - } - return 0; -} - /** * DPDK callback to get the RETA indirection table. * @@ -242,20 +185,28 @@ priv_dev_rss_reta_update(struct priv *priv, * Size of the RETA table. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_dev_rss_reta_query(struct rte_eth_dev *dev, struct rte_eth_rss_reta_entry64 *reta_conf, uint16_t reta_size) { - int ret; struct priv *priv = dev->data->dev_private; + unsigned int idx; + unsigned int i; - priv_lock(priv); - ret = priv_dev_rss_reta_query(priv, reta_conf, reta_size); - priv_unlock(priv); - return -ret; + if (!reta_size || reta_size > priv->reta_idx_n) { + rte_errno = EINVAL; + return -rte_errno; + } + /* Fill each entry of the table even if its bit is not set. */ + for (idx = 0, i = 0; (i != reta_size); ++i) { + idx = i / RTE_RETA_GROUP_SIZE; + reta_conf[idx].reta[i % RTE_RETA_GROUP_SIZE] = + (*priv->reta_idx)[i]; + } + return 0; } /** @@ -269,7 +220,7 @@ mlx5_dev_rss_reta_query(struct rte_eth_dev *dev, * Size of the RETA table. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_dev_rss_reta_update(struct rte_eth_dev *dev, @@ -278,13 +229,28 @@ mlx5_dev_rss_reta_update(struct rte_eth_dev *dev, { int ret; struct priv *priv = dev->data->dev_private; + unsigned int idx; + unsigned int i; + unsigned int pos; - priv_lock(priv); - ret = priv_dev_rss_reta_update(priv, reta_conf, reta_size); - priv_unlock(priv); + if (!reta_size) { + rte_errno = EINVAL; + return -rte_errno; + } + ret = mlx5_rss_reta_index_resize(dev, reta_size); + if (ret) + return ret; + for (idx = 0, i = 0; (i != reta_size); ++i) { + idx = i / RTE_RETA_GROUP_SIZE; + pos = i % RTE_RETA_GROUP_SIZE; + if (((reta_conf[idx].mask >> i) & 0x1) == 0) + continue; + assert(reta_conf[idx].reta[pos] < priv->rxqs_n); + (*priv->reta_idx)[i] = reta_conf[idx].reta[pos]; + } if (dev->data->dev_started) { mlx5_dev_stop(dev); - mlx5_dev_start(dev); + return mlx5_dev_start(dev); } - return -ret; + return 0; } diff --git a/dpdk/drivers/net/mlx5/mlx5_rxmode.c b/dpdk/drivers/net/mlx5/mlx5_rxmode.c index 6fb245ba..617138c1 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxmode.c +++ b/dpdk/drivers/net/mlx5/mlx5_rxmode.c @@ -60,8 +60,21 @@ void mlx5_promiscuous_enable(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; + int ret; + dev->data->promiscuous = 1; - mlx5_traffic_restart(dev); + if (priv->isolated) { + DRV_LOG(WARNING, + "port %u cannot enable promiscuous mode" + " in flow isolation mode", + dev->data->port_id); + return; + } + ret = mlx5_traffic_restart(dev); + if (ret) + DRV_LOG(ERR, "port %u cannot enable promiscuous mode: %s", + dev->data->port_id, strerror(rte_errno)); } /** @@ -73,8 +86,13 @@ mlx5_promiscuous_enable(struct rte_eth_dev *dev) void mlx5_promiscuous_disable(struct rte_eth_dev *dev) { + int ret; + dev->data->promiscuous = 0; - mlx5_traffic_restart(dev); + ret = mlx5_traffic_restart(dev); + if (ret) + DRV_LOG(ERR, "port %u cannot disable promiscuous mode: %s", + dev->data->port_id, strerror(rte_errno)); } /** @@ -86,8 +104,21 @@ mlx5_promiscuous_disable(struct rte_eth_dev *dev) void mlx5_allmulticast_enable(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; + int ret; + dev->data->all_multicast = 1; - mlx5_traffic_restart(dev); + if (priv->isolated) { + DRV_LOG(WARNING, + "port %u cannot enable allmulticast mode" + " in flow isolation mode", + dev->data->port_id); + return; + } + ret = mlx5_traffic_restart(dev); + if (ret) + DRV_LOG(ERR, "port %u cannot enable allmulicast mode: %s", + dev->data->port_id, strerror(rte_errno)); } /** @@ -99,6 +130,11 @@ mlx5_allmulticast_enable(struct rte_eth_dev *dev) void mlx5_allmulticast_disable(struct rte_eth_dev *dev) { + int ret; + dev->data->all_multicast = 0; - mlx5_traffic_restart(dev); + ret = mlx5_traffic_restart(dev); + if (ret) + DRV_LOG(ERR, "port %u cannot disable allmulicast mode: %s", + dev->data->port_id, strerror(rte_errno)); } diff --git a/dpdk/drivers/net/mlx5/mlx5_rxq.c b/dpdk/drivers/net/mlx5/mlx5_rxq.c index 20f3ec6c..dcc5a87b 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxq.c +++ b/dpdk/drivers/net/mlx5/mlx5_rxq.c @@ -88,7 +88,7 @@ const size_t rss_hash_default_key_len = sizeof(rss_hash_default_key); * Pointer to RX queue structure. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl) @@ -96,7 +96,7 @@ rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl) const unsigned int sges_n = 1 << rxq_ctrl->rxq.sges_n; unsigned int elts_n = 1 << rxq_ctrl->rxq.elts_n; unsigned int i; - int ret = 0; + int err; /* Iterate on segments. */ for (i = 0; (i != elts_n); ++i) { @@ -104,8 +104,9 @@ rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl) buf = rte_pktmbuf_alloc(rxq_ctrl->rxq.mp); if (buf == NULL) { - ERROR("%p: empty mbuf pool", (void *)rxq_ctrl); - ret = ENOMEM; + DRV_LOG(ERR, "port %u empty mbuf pool", + PORT_ID(rxq_ctrl->priv)); + rte_errno = ENOMEM; goto error; } /* Headroom is reserved by rte_pktmbuf_alloc(). */ @@ -124,7 +125,7 @@ rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl) (*rxq_ctrl->rxq.elts)[i] = buf; } /* If Rx vector is activated. */ - if (rxq_check_vec_support(&rxq_ctrl->rxq) > 0) { + if (mlx5_rxq_check_vec_support(&rxq_ctrl->rxq) > 0) { struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq; struct rte_mbuf *mbuf_init = &rxq->fake_mbuf; int j; @@ -145,20 +146,24 @@ rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl) for (j = 0; j < MLX5_VPMD_DESCS_PER_LOOP; ++j) (*rxq->elts)[elts_n + j] = &rxq->fake_mbuf; } - DEBUG("%p: allocated and configured %u segments (max %u packets)", - (void *)rxq_ctrl, elts_n, elts_n / (1 << rxq_ctrl->rxq.sges_n)); - assert(ret == 0); + DRV_LOG(DEBUG, + "port %u Rx queue %u allocated and configured %u segments" + " (max %u packets)", + PORT_ID(rxq_ctrl->priv), rxq_ctrl->idx, elts_n, + elts_n / (1 << rxq_ctrl->rxq.sges_n)); return 0; error: + err = rte_errno; /* Save rte_errno before cleanup. */ elts_n = i; for (i = 0; (i != elts_n); ++i) { if ((*rxq_ctrl->rxq.elts)[i] != NULL) rte_pktmbuf_free_seg((*rxq_ctrl->rxq.elts)[i]); (*rxq_ctrl->rxq.elts)[i] = NULL; } - DEBUG("%p: failed, freed everything", (void *)rxq_ctrl); - assert(ret > 0); - return ret; + DRV_LOG(DEBUG, "port %u Rx queue %u failed, freed everything", + PORT_ID(rxq_ctrl->priv), rxq_ctrl->idx); + rte_errno = err; /* Restore rte_errno. */ + return -rte_errno; } /** @@ -176,14 +181,15 @@ rxq_free_elts(struct mlx5_rxq_ctrl *rxq_ctrl) uint16_t used = q_n - (rxq->rq_ci - rxq->rq_pi); uint16_t i; - DEBUG("%p: freeing WRs", (void *)rxq_ctrl); + DRV_LOG(DEBUG, "port %u Rx queue %u freeing WRs", + PORT_ID(rxq_ctrl->priv), rxq_ctrl->idx); if (rxq->elts == NULL) return; /** * Some mbuf in the Ring belongs to the application. They cannot be * freed. */ - if (rxq_check_vec_support(rxq) > 0) { + if (mlx5_rxq_check_vec_support(rxq) > 0) { for (i = 0; i < used; ++i) (*rxq->elts)[(rxq->rq_ci + i) & q_mask] = NULL; rxq->rq_pi = rxq->rq_ci; @@ -206,9 +212,10 @@ rxq_free_elts(struct mlx5_rxq_ctrl *rxq_ctrl) void mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *rxq_ctrl) { - DEBUG("cleaning up %p", (void *)rxq_ctrl); + DRV_LOG(DEBUG, "port %u cleaning up Rx queue %u", + PORT_ID(rxq_ctrl->priv), rxq_ctrl->idx); if (rxq_ctrl->ibv) - mlx5_priv_rxq_ibv_release(rxq_ctrl->priv, rxq_ctrl->ibv); + mlx5_rxq_ibv_release(rxq_ctrl->ibv); memset(rxq_ctrl, 0, sizeof(*rxq_ctrl)); } @@ -228,55 +235,52 @@ mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *rxq_ctrl) * Memory pool for buffer allocations. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, - unsigned int socket, const struct rte_eth_rxconf *conf, + unsigned int socket, + const struct rte_eth_rxconf *conf __rte_unused, struct rte_mempool *mp) { struct priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx]; struct mlx5_rxq_ctrl *rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); - int ret = 0; - (void)conf; - priv_lock(priv); if (!rte_is_power_of_2(desc)) { desc = 1 << log2above(desc); - WARN("%p: increased number of descriptors in RX queue %u" - " to the next power of two (%d)", - (void *)dev, idx, desc); + DRV_LOG(WARNING, + "port %u increased number of descriptors in Rx queue %u" + " to the next power of two (%d)", + dev->data->port_id, idx, desc); } - DEBUG("%p: configuring queue %u for %u descriptors", - (void *)dev, idx, desc); + DRV_LOG(DEBUG, "port %u configuring Rx queue %u for %u descriptors", + dev->data->port_id, idx, desc); if (idx >= priv->rxqs_n) { - ERROR("%p: queue index out of range (%u >= %u)", - (void *)dev, idx, priv->rxqs_n); - priv_unlock(priv); - return -EOVERFLOW; + DRV_LOG(ERR, "port %u Rx queue index out of range (%u >= %u)", + dev->data->port_id, idx, priv->rxqs_n); + rte_errno = EOVERFLOW; + return -rte_errno; } - if (!mlx5_priv_rxq_releasable(priv, idx)) { - ret = EBUSY; - ERROR("%p: unable to release queue index %u", - (void *)dev, idx); - goto out; + if (!mlx5_rxq_releasable(dev, idx)) { + DRV_LOG(ERR, "port %u unable to release queue index %u", + dev->data->port_id, idx); + rte_errno = EBUSY; + return -rte_errno; } - mlx5_priv_rxq_release(priv, idx); - rxq_ctrl = mlx5_priv_rxq_new(priv, idx, desc, socket, mp); + mlx5_rxq_release(dev, idx); + rxq_ctrl = mlx5_rxq_new(dev, idx, desc, socket, mp); if (!rxq_ctrl) { - ERROR("%p: unable to allocate queue index %u", - (void *)dev, idx); - ret = ENOMEM; - goto out; + DRV_LOG(ERR, "port %u unable to allocate queue index %u", + dev->data->port_id, idx); + rte_errno = ENOMEM; + return -rte_errno; } - DEBUG("%p: adding RX queue %p to list", - (void *)dev, (void *)rxq_ctrl); + DRV_LOG(DEBUG, "port %u adding Rx queue %u to list", + dev->data->port_id, idx); (*priv->rxqs)[idx] = &rxq_ctrl->rxq; -out: - priv_unlock(priv); - return -ret; + return 0; } /** @@ -296,45 +300,48 @@ mlx5_rx_queue_release(void *dpdk_rxq) return; rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); priv = rxq_ctrl->priv; - priv_lock(priv); - if (!mlx5_priv_rxq_releasable(priv, rxq_ctrl->rxq.stats.idx)) - rte_panic("Rx queue %p is still used by a flow and cannot be" - " removed\n", (void *)rxq_ctrl); - mlx5_priv_rxq_release(priv, rxq_ctrl->rxq.stats.idx); - priv_unlock(priv); + if (!mlx5_rxq_releasable(ETH_DEV(priv), rxq_ctrl->rxq.stats.idx)) + rte_panic("port %u Rx queue %u is still used by a flow and" + " cannot be removed\n", + PORT_ID(priv), rxq_ctrl->idx); + mlx5_rxq_release(ETH_DEV(priv), rxq_ctrl->rxq.stats.idx); } /** * Allocate queue vector and fill epoll fd list for Rx interrupts. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return - * 0 on success, negative on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_rx_intr_vec_enable(struct priv *priv) +mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; unsigned int i; unsigned int rxqs_n = priv->rxqs_n; unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); unsigned int count = 0; - struct rte_intr_handle *intr_handle = priv->dev->intr_handle; + struct rte_intr_handle *intr_handle = dev->intr_handle; - if (!priv->dev->data->dev_conf.intr_conf.rxq) + if (!dev->data->dev_conf.intr_conf.rxq) return 0; - priv_rx_intr_vec_disable(priv); + mlx5_rx_intr_vec_disable(dev); intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0])); if (intr_handle->intr_vec == NULL) { - ERROR("failed to allocate memory for interrupt vector," - " Rx interrupts will not be supported"); - return -ENOMEM; + DRV_LOG(ERR, + "port %u failed to allocate memory for interrupt" + " vector, Rx interrupts will not be supported", + dev->data->port_id); + rte_errno = ENOMEM; + return -rte_errno; } intr_handle->type = RTE_INTR_HANDLE_EXT; for (i = 0; i != n; ++i) { /* This rxq ibv must not be released in this function. */ - struct mlx5_rxq_ibv *rxq_ibv = mlx5_priv_rxq_ibv_get(priv, i); + struct mlx5_rxq_ibv *rxq_ibv = mlx5_rxq_ibv_get(dev, i); int fd; int flags; int rc; @@ -348,27 +355,34 @@ priv_rx_intr_vec_enable(struct priv *priv) continue; } if (count >= RTE_MAX_RXTX_INTR_VEC_ID) { - ERROR("too many Rx queues for interrupt vector size" - " (%d), Rx interrupts cannot be enabled", - RTE_MAX_RXTX_INTR_VEC_ID); - priv_rx_intr_vec_disable(priv); - return -1; + DRV_LOG(ERR, + "port %u too many Rx queues for interrupt" + " vector size (%d), Rx interrupts cannot be" + " enabled", + dev->data->port_id, RTE_MAX_RXTX_INTR_VEC_ID); + mlx5_rx_intr_vec_disable(dev); + rte_errno = ENOMEM; + return -rte_errno; } fd = rxq_ibv->channel->fd; flags = fcntl(fd, F_GETFL); rc = fcntl(fd, F_SETFL, flags | O_NONBLOCK); if (rc < 0) { - ERROR("failed to make Rx interrupt file descriptor" - " %d non-blocking for queue index %d", fd, i); - priv_rx_intr_vec_disable(priv); - return -1; + rte_errno = errno; + DRV_LOG(ERR, + "port %u failed to make Rx interrupt file" + " descriptor %d non-blocking for queue index" + " %d", + dev->data->port_id, fd, i); + mlx5_rx_intr_vec_disable(dev); + return -rte_errno; } intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count; intr_handle->efds[count] = fd; count++; } if (!count) - priv_rx_intr_vec_disable(priv); + mlx5_rx_intr_vec_disable(dev); else intr_handle->nb_efd = count; return 0; @@ -377,18 +391,19 @@ priv_rx_intr_vec_enable(struct priv *priv) /** * Clean up Rx interrupts handler. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. */ void -priv_rx_intr_vec_disable(struct priv *priv) +mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev) { - struct rte_intr_handle *intr_handle = priv->dev->intr_handle; + struct priv *priv = dev->data->dev_private; + struct rte_intr_handle *intr_handle = dev->intr_handle; unsigned int i; unsigned int rxqs_n = priv->rxqs_n; unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); - if (!priv->dev->data->dev_conf.intr_conf.rxq) + if (!dev->data->dev_conf.intr_conf.rxq) return; if (!intr_handle->intr_vec) goto free; @@ -405,7 +420,7 @@ priv_rx_intr_vec_disable(struct priv *priv) */ rxq_data = (*priv->rxqs)[i]; rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); - mlx5_priv_rxq_ibv_release(priv, rxq_ctrl->ibv); + mlx5_rxq_ibv_release(rxq_ctrl->ibv); } free: rte_intr_free_epoll_fd(intr_handle); @@ -449,39 +464,33 @@ mlx5_arm_cq(struct mlx5_rxq_data *rxq, int sq_n_rxq) * Rx queue number. * * @return - * 0 on success, negative on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id) { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq_data; struct mlx5_rxq_ctrl *rxq_ctrl; - int ret = 0; - priv_lock(priv); rxq_data = (*priv->rxqs)[rx_queue_id]; if (!rxq_data) { - ret = EINVAL; - goto exit; + rte_errno = EINVAL; + return -rte_errno; } rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); if (rxq_ctrl->irq) { struct mlx5_rxq_ibv *rxq_ibv; - rxq_ibv = mlx5_priv_rxq_ibv_get(priv, rx_queue_id); + rxq_ibv = mlx5_rxq_ibv_get(dev, rx_queue_id); if (!rxq_ibv) { - ret = EINVAL; - goto exit; + rte_errno = EINVAL; + return -rte_errno; } mlx5_arm_cq(rxq_data, rxq_data->cq_arm_sn); - mlx5_priv_rxq_ibv_release(priv, rxq_ibv); + mlx5_rxq_ibv_release(rxq_ibv); } -exit: - priv_unlock(priv); - if (ret) - WARN("unable to arm interrupt on rx queue %d", rx_queue_id); - return -ret; + return 0; } /** @@ -493,64 +502,65 @@ exit: * Rx queue number. * * @return - * 0 on success, negative on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id) { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq_data; struct mlx5_rxq_ctrl *rxq_ctrl; struct mlx5_rxq_ibv *rxq_ibv = NULL; struct ibv_cq *ev_cq; void *ev_ctx; - int ret = 0; + int ret; - priv_lock(priv); rxq_data = (*priv->rxqs)[rx_queue_id]; if (!rxq_data) { - ret = EINVAL; - goto exit; + rte_errno = EINVAL; + return -rte_errno; } rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); if (!rxq_ctrl->irq) - goto exit; - rxq_ibv = mlx5_priv_rxq_ibv_get(priv, rx_queue_id); + return 0; + rxq_ibv = mlx5_rxq_ibv_get(dev, rx_queue_id); if (!rxq_ibv) { - ret = EINVAL; - goto exit; + rte_errno = EINVAL; + return -rte_errno; } ret = ibv_get_cq_event(rxq_ibv->channel, &ev_cq, &ev_ctx); if (ret || ev_cq != rxq_ibv->cq) { - ret = EINVAL; + rte_errno = EINVAL; goto exit; } rxq_data->cq_arm_sn++; ibv_ack_cq_events(rxq_ibv->cq, 1); + return 0; exit: + ret = rte_errno; /* Save rte_errno before cleanup. */ if (rxq_ibv) - mlx5_priv_rxq_ibv_release(priv, rxq_ibv); - priv_unlock(priv); - if (ret) - WARN("unable to disable interrupt on rx queue %d", - rx_queue_id); - return -ret; + mlx5_rxq_ibv_release(rxq_ibv); + DRV_LOG(WARNING, "port %u unable to disable interrupt on Rx queue %d", + dev->data->port_id, rx_queue_id); + rte_errno = ret; /* Restore rte_errno. */ + return -rte_errno; } /** * Create the Rx queue Verbs object. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * Queue index in DPDK Rx queue array * * @return - * The Verbs object initialised if it can be created. + * The Verbs object initialised, NULL otherwise and rte_errno is set. */ -struct mlx5_rxq_ibv* -mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) +struct mlx5_rxq_ibv * +mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; struct mlx5_rxq_ctrl *rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); @@ -573,28 +583,34 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) assert(rxq_data); assert(!rxq_ctrl->ibv); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_RX_QUEUE; + priv->verbs_alloc_ctx.obj = rxq_ctrl; tmpl = rte_calloc_socket(__func__, 1, sizeof(*tmpl), 0, rxq_ctrl->socket); if (!tmpl) { - ERROR("%p: cannot allocate verbs resources", - (void *)rxq_ctrl); + DRV_LOG(ERR, + "port %u Rx queue %u cannot allocate verbs resources", + dev->data->port_id, rxq_ctrl->idx); + rte_errno = ENOMEM; goto error; } tmpl->rxq_ctrl = rxq_ctrl; /* Use the entire RX mempool as the memory region. */ - tmpl->mr = priv_mr_get(priv, rxq_data->mp); + tmpl->mr = mlx5_mr_get(dev, rxq_data->mp); if (!tmpl->mr) { - tmpl->mr = priv_mr_new(priv, rxq_data->mp); + tmpl->mr = mlx5_mr_new(dev, rxq_data->mp); if (!tmpl->mr) { - ERROR("%p: MR creation failure", (void *)rxq_ctrl); + DRV_LOG(ERR, "port %u: memeroy region creation failure", + dev->data->port_id); goto error; } } if (rxq_ctrl->irq) { tmpl->channel = ibv_create_comp_channel(priv->ctx); if (!tmpl->channel) { - ERROR("%p: Comp Channel creation failure", - (void *)rxq_ctrl); + DRV_LOG(ERR, "port %u: comp channel creation failure", + dev->data->port_id); + rte_errno = ENOMEM; goto error; } } @@ -614,21 +630,26 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) * For vectorized Rx, it must not be doubled in order to * make cq_ci and rq_ci aligned. */ - if (rxq_check_vec_support(rxq_data) < 0) + if (mlx5_rxq_check_vec_support(rxq_data) < 0) attr.cq.ibv.cqe *= 2; } else if (priv->cqe_comp && rxq_data->hw_timestamp) { - DEBUG("Rx CQE compression is disabled for HW timestamp"); + DRV_LOG(DEBUG, + "port %u Rx CQE compression is disabled for HW" + " timestamp", + dev->data->port_id); } tmpl->cq = ibv_cq_ex_to_cq(mlx5dv_create_cq(priv->ctx, &attr.cq.ibv, &attr.cq.mlx5)); if (tmpl->cq == NULL) { - ERROR("%p: CQ creation failure", (void *)rxq_ctrl); + DRV_LOG(ERR, "port %u Rx queue %u CQ creation failure", + dev->data->port_id, idx); + rte_errno = ENOMEM; goto error; } - DEBUG("priv->device_attr.max_qp_wr is %d", - priv->device_attr.orig_attr.max_qp_wr); - DEBUG("priv->device_attr.max_sge is %d", - priv->device_attr.orig_attr.max_sge); + DRV_LOG(DEBUG, "port %u priv->device_attr.max_qp_wr is %d", + dev->data->port_id, priv->device_attr.orig_attr.max_qp_wr); + DRV_LOG(DEBUG, "port %u priv->device_attr.max_sge is %d", + dev->data->port_id, priv->device_attr.orig_attr.max_sge); attr.wq = (struct ibv_wq_init_attr){ .wq_context = NULL, /* Could be useful in the future. */ .wq_type = IBV_WQT_RQ, @@ -658,7 +679,9 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) #endif tmpl->wq = ibv_create_wq(priv->ctx, &attr.wq); if (tmpl->wq == NULL) { - ERROR("%p: WQ creation failure", (void *)rxq_ctrl); + DRV_LOG(ERR, "port %u Rx queue %u WQ creation failure", + dev->data->port_id, idx); + rte_errno = ENOMEM; goto error; } /* @@ -668,11 +691,14 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) if (((int)attr.wq.max_wr != ((1 << rxq_data->elts_n) >> rxq_data->sges_n)) || ((int)attr.wq.max_sge != (1 << rxq_data->sges_n))) { - ERROR("%p: requested %u*%u but got %u*%u WRs*SGEs", - (void *)rxq_ctrl, - ((1 << rxq_data->elts_n) >> rxq_data->sges_n), - (1 << rxq_data->sges_n), - attr.wq.max_wr, attr.wq.max_sge); + DRV_LOG(ERR, + "port %u Rx queue %u requested %u*%u but got %u*%u" + " WRs*SGEs", + dev->data->port_id, idx, + ((1 << rxq_data->elts_n) >> rxq_data->sges_n), + (1 << rxq_data->sges_n), + attr.wq.max_wr, attr.wq.max_sge); + rte_errno = EINVAL; goto error; } /* Change queue state to ready. */ @@ -682,8 +708,10 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) }; ret = ibv_modify_wq(tmpl->wq, &mod); if (ret) { - ERROR("%p: WQ state to IBV_WQS_RDY failed", - (void *)rxq_ctrl); + DRV_LOG(ERR, + "port %u Rx queue %u WQ state to IBV_WQS_RDY failed", + dev->data->port_id, idx); + rte_errno = ret; goto error; } obj.cq.in = tmpl->cq; @@ -691,11 +719,16 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) obj.rwq.in = tmpl->wq; obj.rwq.out = &rwq; ret = mlx5dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_RWQ); - if (ret != 0) + if (ret) { + rte_errno = ret; goto error; + } if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) { - ERROR("Wrong MLX5_CQE_SIZE environment variable value: " - "it should be set to %u", RTE_CACHE_LINE_SIZE); + DRV_LOG(ERR, + "port %u wrong MLX5_CQE_SIZE environment variable" + " value: it should be set to %u", + dev->data->port_id, RTE_CACHE_LINE_SIZE); + rte_errno = EINVAL; goto error; } /* Fill the rings. */ @@ -731,13 +764,16 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx) rxq_data->rq_ci = (1 << rxq_data->elts_n) >> rxq_data->sges_n; rte_wmb(); *rxq_data->rq_db = rte_cpu_to_be_32(rxq_data->rq_ci); - DEBUG("%p: rxq updated with %p", (void *)rxq_ctrl, (void *)&tmpl); + DRV_LOG(DEBUG, "port %u rxq %u updated with %p", dev->data->port_id, + idx, (void *)&tmpl); rte_atomic32_inc(&tmpl->refcnt); - DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv, - (void *)tmpl, rte_atomic32_read(&tmpl->refcnt)); + DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d", + dev->data->port_id, idx, rte_atomic32_read(&tmpl->refcnt)); LIST_INSERT_HEAD(&priv->rxqsibv, tmpl, next); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; return tmpl; error: + ret = rte_errno; /* Save rte_errno before cleanup. */ if (tmpl->wq) claim_zero(ibv_destroy_wq(tmpl->wq)); if (tmpl->cq) @@ -745,24 +781,27 @@ error: if (tmpl->channel) claim_zero(ibv_destroy_comp_channel(tmpl->channel)); if (tmpl->mr) - priv_mr_release(priv, tmpl->mr); + mlx5_mr_release(tmpl->mr); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; + rte_errno = ret; /* Restore rte_errno. */ return NULL; } /** * Get an Rx queue Verbs object. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * Queue index in DPDK Rx queue array * * @return * The Verbs object if it exists. */ -struct mlx5_rxq_ibv* -mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx) +struct mlx5_rxq_ibv * +mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx]; struct mlx5_rxq_ctrl *rxq_ctrl; @@ -772,11 +811,11 @@ mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx) return NULL; rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq); if (rxq_ctrl->ibv) { - priv_mr_get(priv, rxq_data->mp); + mlx5_mr_get(dev, rxq_data->mp); rte_atomic32_inc(&rxq_ctrl->ibv->refcnt); - DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv, - (void *)rxq_ctrl->ibv, - rte_atomic32_read(&rxq_ctrl->ibv->refcnt)); + DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d", + dev->data->port_id, rxq_ctrl->idx, + rte_atomic32_read(&rxq_ctrl->ibv->refcnt)); } return rxq_ctrl->ibv; } @@ -784,16 +823,14 @@ mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx) /** * Release an Rx verbs queue object. * - * @param priv - * Pointer to private structure. * @param rxq_ibv * Verbs Rx queue object. * * @return - * 0 on success, errno value on failure. + * 1 while a reference on it exists, 0 when freed. */ int -mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv) +mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv) { int ret; @@ -801,11 +838,12 @@ mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv) assert(rxq_ibv->wq); assert(rxq_ibv->cq); assert(rxq_ibv->mr); - ret = priv_mr_release(priv, rxq_ibv->mr); + ret = mlx5_mr_release(rxq_ibv->mr); if (!ret) rxq_ibv->mr = NULL; - DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv, - (void *)rxq_ibv, rte_atomic32_read(&rxq_ibv->refcnt)); + DRV_LOG(DEBUG, "port %u Verbs Rx queue %u: refcnt %d", + PORT_ID(rxq_ibv->rxq_ctrl->priv), + rxq_ibv->rxq_ctrl->idx, rte_atomic32_read(&rxq_ibv->refcnt)); if (rte_atomic32_dec_and_test(&rxq_ibv->refcnt)) { rxq_free_elts(rxq_ibv->rxq_ctrl); claim_zero(ibv_destroy_wq(rxq_ibv->wq)); @@ -816,26 +854,28 @@ mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv) rte_free(rxq_ibv); return 0; } - return EBUSY; + return 1; } /** * Verify the Verbs Rx queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * - * @return the number of object not released. + * @return + * The number of object not released. */ int -mlx5_priv_rxq_ibv_verify(struct priv *priv) +mlx5_rxq_ibv_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; int ret = 0; struct mlx5_rxq_ibv *rxq_ibv; LIST_FOREACH(rxq_ibv, &priv->rxqsibv, next) { - DEBUG("%p: Verbs Rx queue %p still referenced", (void *)priv, - (void *)rxq_ibv); + DRV_LOG(DEBUG, "port %u Verbs Rx queue %u still referenced", + dev->data->port_id, rxq_ibv->rxq_ctrl->idx); ++ret; } return ret; @@ -844,15 +884,12 @@ mlx5_priv_rxq_ibv_verify(struct priv *priv) /** * Return true if a single reference exists on the object. * - * @param priv - * Pointer to private structure. * @param rxq_ibv * Verbs Rx queue object. */ int -mlx5_priv_rxq_ibv_releasable(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv) +mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv) { - (void)priv; assert(rxq_ibv); return (rte_atomic32_read(&rxq_ibv->refcnt) == 1); } @@ -860,8 +897,8 @@ mlx5_priv_rxq_ibv_releasable(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv) /** * Create a DPDK Rx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * @param desc @@ -870,13 +907,13 @@ mlx5_priv_rxq_ibv_releasable(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv) * NUMA socket on which memory must be allocated. * * @return - * A DPDK queue object on success. + * A DPDK queue object on success, NULL otherwise and rte_errno is set. */ -struct mlx5_rxq_ctrl* -mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc, - unsigned int socket, struct rte_mempool *mp) +struct mlx5_rxq_ctrl * +mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, struct rte_mempool *mp) { - struct rte_eth_dev *dev = priv->dev; + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *tmpl; const uint16_t desc_n = desc + priv->rx_vec_en * MLX5_VPMD_DESCS_PER_LOOP; @@ -886,10 +923,12 @@ mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc, sizeof(*tmpl) + desc_n * sizeof(struct rte_mbuf *), 0, socket); - if (!tmpl) + if (!tmpl) { + rte_errno = ENOMEM; return NULL; + } tmpl->socket = socket; - if (priv->dev->data->dev_conf.intr_conf.rxq) + if (dev->data->dev_conf.intr_conf.rxq) tmpl->irq = 1; /* Enable scattered packets support for this queue if necessary. */ assert(mb_len >= RTE_PKTMBUF_HEADROOM); @@ -912,29 +951,34 @@ mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc, size = mb_len * (1 << tmpl->rxq.sges_n); size -= RTE_PKTMBUF_HEADROOM; if (size < dev->data->dev_conf.rxmode.max_rx_pkt_len) { - ERROR("%p: too many SGEs (%u) needed to handle" - " requested maximum packet size %u", - (void *)dev, - 1 << sges_n, - dev->data->dev_conf.rxmode.max_rx_pkt_len); + DRV_LOG(ERR, + "port %u too many SGEs (%u) needed to handle" + " requested maximum packet size %u", + dev->data->port_id, + 1 << sges_n, + dev->data->dev_conf.rxmode.max_rx_pkt_len); + rte_errno = EOVERFLOW; goto error; } } else { - WARN("%p: the requested maximum Rx packet size (%u) is" - " larger than a single mbuf (%u) and scattered" - " mode has not been requested", - (void *)dev, - dev->data->dev_conf.rxmode.max_rx_pkt_len, - mb_len - RTE_PKTMBUF_HEADROOM); + DRV_LOG(WARNING, + "port %u the requested maximum Rx packet size (%u) is" + " larger than a single mbuf (%u) and scattered mode has" + " not been requested", + dev->data->port_id, + dev->data->dev_conf.rxmode.max_rx_pkt_len, + mb_len - RTE_PKTMBUF_HEADROOM); } - DEBUG("%p: maximum number of segments per packet: %u", - (void *)dev, 1 << tmpl->rxq.sges_n); + DRV_LOG(DEBUG, "port %u maximum number of segments per packet: %u", + dev->data->port_id, 1 << tmpl->rxq.sges_n); if (desc % (1 << tmpl->rxq.sges_n)) { - ERROR("%p: number of RX queue descriptors (%u) is not a" - " multiple of SGEs per packet (%u)", - (void *)dev, - desc, - 1 << tmpl->rxq.sges_n); + DRV_LOG(ERR, + "port %u number of Rx queue descriptors (%u) is not a" + " multiple of SGEs per packet (%u)", + dev->data->port_id, + desc, + 1 << tmpl->rxq.sges_n); + rte_errno = EINVAL; goto error; } /* Toggle RX checksum offload if hardware supports it. */ @@ -954,19 +998,22 @@ mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc, } else if (priv->hw_fcs_strip) { tmpl->rxq.crc_present = 1; } else { - WARN("%p: CRC stripping has been disabled but will still" - " be performed by hardware, make sure MLNX_OFED and" - " firmware are up to date", - (void *)dev); + DRV_LOG(WARNING, + "port %u CRC stripping has been disabled but will" + " still be performed by hardware, make sure MLNX_OFED" + " and firmware are up to date", + dev->data->port_id); tmpl->rxq.crc_present = 0; } - DEBUG("%p: CRC stripping is %s, %u bytes will be subtracted from" - " incoming frames to hide it", - (void *)dev, - tmpl->rxq.crc_present ? "disabled" : "enabled", - tmpl->rxq.crc_present << 2); + DRV_LOG(DEBUG, + "port %u CRC stripping is %s, %u bytes will be subtracted from" + " incoming frames to hide it", + dev->data->port_id, + tmpl->rxq.crc_present ? "disabled" : "enabled", + tmpl->rxq.crc_present << 2); /* Save port ID. */ - tmpl->rxq.rss_hash = priv->rxqs_n > 1; + tmpl->rxq.rss_hash = !!priv->rss_conf.rss_hf && + (!!(dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS)); tmpl->rxq.port_id = dev->data->port_id; tmpl->priv = priv; tmpl->rxq.mp = mp; @@ -974,9 +1021,10 @@ mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc, tmpl->rxq.elts_n = log2above(desc); tmpl->rxq.elts = (struct rte_mbuf *(*)[1 << tmpl->rxq.elts_n])(tmpl + 1); + tmpl->idx = idx; rte_atomic32_inc(&tmpl->refcnt); - DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv, - (void *)tmpl, rte_atomic32_read(&tmpl->refcnt)); + DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d", dev->data->port_id, + idx, rte_atomic32_read(&tmpl->refcnt)); LIST_INSERT_HEAD(&priv->rxqsctrl, tmpl, next); return tmpl; error: @@ -987,28 +1035,29 @@ error: /** * Get a Rx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * * @return - * A pointer to the queue if it exists. + * A pointer to the queue if it exists, NULL otherwise. */ -struct mlx5_rxq_ctrl* -mlx5_priv_rxq_get(struct priv *priv, uint16_t idx) +struct mlx5_rxq_ctrl * +mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl = NULL; if ((*priv->rxqs)[idx]) { rxq_ctrl = container_of((*priv->rxqs)[idx], struct mlx5_rxq_ctrl, rxq); - - mlx5_priv_rxq_ibv_get(priv, idx); + mlx5_rxq_ibv_get(dev, idx); rte_atomic32_inc(&rxq_ctrl->refcnt); - DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv, - (void *)rxq_ctrl, rte_atomic32_read(&rxq_ctrl->refcnt)); + DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d", + dev->data->port_id, rxq_ctrl->idx, + rte_atomic32_read(&rxq_ctrl->refcnt)); } return rxq_ctrl; } @@ -1016,59 +1065,59 @@ mlx5_priv_rxq_get(struct priv *priv, uint16_t idx) /** * Release a Rx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * * @return - * 0 on success, errno value on failure. + * 1 while a reference on it exists, 0 when freed. */ int -mlx5_priv_rxq_release(struct priv *priv, uint16_t idx) +mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl; if (!(*priv->rxqs)[idx]) return 0; rxq_ctrl = container_of((*priv->rxqs)[idx], struct mlx5_rxq_ctrl, rxq); assert(rxq_ctrl->priv); - if (rxq_ctrl->ibv) { - int ret; - - ret = mlx5_priv_rxq_ibv_release(rxq_ctrl->priv, rxq_ctrl->ibv); - if (!ret) - rxq_ctrl->ibv = NULL; - } - DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv, - (void *)rxq_ctrl, rte_atomic32_read(&rxq_ctrl->refcnt)); + if (rxq_ctrl->ibv && !mlx5_rxq_ibv_release(rxq_ctrl->ibv)) + rxq_ctrl->ibv = NULL; + DRV_LOG(DEBUG, "port %u Rx queue %u: refcnt %d", dev->data->port_id, + rxq_ctrl->idx, rte_atomic32_read(&rxq_ctrl->refcnt)); if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) { LIST_REMOVE(rxq_ctrl, next); rte_free(rxq_ctrl); (*priv->rxqs)[idx] = NULL; return 0; } - return EBUSY; + return 1; } /** * Verify if the queue can be released. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * * @return - * 1 if the queue can be released. + * 1 if the queue can be released, negative errno otherwise and rte_errno is + * set. */ int -mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx) +mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl; - if (!(*priv->rxqs)[idx]) - return -1; + if (!(*priv->rxqs)[idx]) { + rte_errno = EINVAL; + return -rte_errno; + } rxq_ctrl = container_of((*priv->rxqs)[idx], struct mlx5_rxq_ctrl, rxq); return (rte_atomic32_read(&rxq_ctrl->refcnt) == 1); } @@ -1076,20 +1125,22 @@ mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx) /** * Verify the Rx Queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * - * @return the number of object not released. + * @return + * The number of object not released. */ int -mlx5_priv_rxq_verify(struct priv *priv) +mlx5_rxq_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_rxq_ctrl *rxq_ctrl; int ret = 0; LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) { - DEBUG("%p: Rx Queue %p still referenced", (void *)priv, - (void *)rxq_ctrl); + DRV_LOG(DEBUG, "port %u Rx Queue %u still referenced", + dev->data->port_id, rxq_ctrl->idx); ++ret; } return ret; @@ -1098,20 +1149,21 @@ mlx5_priv_rxq_verify(struct priv *priv) /** * Create an indirection table. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param queues * Queues entering in the indirection table. * @param queues_n * Number of queues in the array. * * @return - * A new indirection table. + * The Verbs object initialised, NULL otherwise and rte_errno is set. */ -struct mlx5_ind_table_ibv* -mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[], - uint16_t queues_n) +struct mlx5_ind_table_ibv * +mlx5_ind_table_ibv_new(struct rte_eth_dev *dev, uint16_t queues[], + uint16_t queues_n) { + struct priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl; const unsigned int wq_n = rte_is_power_of_2(queues_n) ? log2above(queues_n) : @@ -1122,11 +1174,12 @@ mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[], ind_tbl = rte_calloc(__func__, 1, sizeof(*ind_tbl) + queues_n * sizeof(uint16_t), 0); - if (!ind_tbl) + if (!ind_tbl) { + rte_errno = ENOMEM; return NULL; + } for (i = 0; i != queues_n; ++i) { - struct mlx5_rxq_ctrl *rxq = - mlx5_priv_rxq_get(priv, queues[i]); + struct mlx5_rxq_ctrl *rxq = mlx5_rxq_get(dev, queues[i]); if (!rxq) goto error; @@ -1144,24 +1197,28 @@ mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[], .ind_tbl = wq, .comp_mask = 0, }); - if (!ind_tbl->ind_table) + if (!ind_tbl->ind_table) { + rte_errno = errno; goto error; + } rte_atomic32_inc(&ind_tbl->refcnt); LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next); - DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv, - (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt)); + DRV_LOG(DEBUG, "port %u indirection table %p: refcnt %d", + dev->data->port_id, (void *)ind_tbl, + rte_atomic32_read(&ind_tbl->refcnt)); return ind_tbl; error: rte_free(ind_tbl); - DEBUG("%p cannot create indirection table", (void *)priv); + DRV_LOG(DEBUG, "port %u cannot create indirection table", + dev->data->port_id); return NULL; } /** * Get an indirection table. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param queues * Queues entering in the indirection table. * @param queues_n @@ -1170,10 +1227,11 @@ error: * @return * An indirection table if found. */ -struct mlx5_ind_table_ibv* -mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[], - uint16_t queues_n) +struct mlx5_ind_table_ibv * +mlx5_ind_table_ibv_get(struct rte_eth_dev *dev, uint16_t queues[], + uint16_t queues_n) { + struct priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl; LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) { @@ -1187,10 +1245,11 @@ mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[], unsigned int i; rte_atomic32_inc(&ind_tbl->refcnt); - DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv, - (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt)); + DRV_LOG(DEBUG, "port %u indirection table %p: refcnt %d", + dev->data->port_id, (void *)ind_tbl, + rte_atomic32_read(&ind_tbl->refcnt)); for (i = 0; i != ind_tbl->queues_n; ++i) - mlx5_priv_rxq_get(priv, ind_tbl->queues[i]); + mlx5_rxq_get(dev, ind_tbl->queues[i]); } return ind_tbl; } @@ -1198,51 +1257,55 @@ mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[], /** * Release an indirection table. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param ind_table * Indirection table to release. * * @return - * 0 on success, errno value on failure. + * 1 while a reference on it exists, 0 when freed. */ int -mlx5_priv_ind_table_ibv_release(struct priv *priv, - struct mlx5_ind_table_ibv *ind_tbl) +mlx5_ind_table_ibv_release(struct rte_eth_dev *dev, + struct mlx5_ind_table_ibv *ind_tbl) { unsigned int i; - DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv, - (void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt)); + DRV_LOG(DEBUG, "port %u indirection table %p: refcnt %d", + dev->data->port_id, (void *)ind_tbl, + rte_atomic32_read(&ind_tbl->refcnt)); if (rte_atomic32_dec_and_test(&ind_tbl->refcnt)) claim_zero(ibv_destroy_rwq_ind_table(ind_tbl->ind_table)); for (i = 0; i != ind_tbl->queues_n; ++i) - claim_nonzero(mlx5_priv_rxq_release(priv, ind_tbl->queues[i])); + claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i])); if (!rte_atomic32_read(&ind_tbl->refcnt)) { LIST_REMOVE(ind_tbl, next); rte_free(ind_tbl); return 0; } - return EBUSY; + return 1; } /** * Verify the Rx Queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * - * @return the number of object not released. + * @return + * The number of object not released. */ int -mlx5_priv_ind_table_ibv_verify(struct priv *priv) +mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_ind_table_ibv *ind_tbl; int ret = 0; LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) { - DEBUG("%p: Verbs indirection table %p still referenced", - (void *)priv, (void *)ind_tbl); + DRV_LOG(DEBUG, + "port %u Verbs indirection table %p still referenced", + dev->data->port_id, (void *)ind_tbl); ++ret; } return ret; @@ -1251,8 +1314,8 @@ mlx5_priv_ind_table_ibv_verify(struct priv *priv) /** * Create an Rx Hash queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param rss_key * RSS key for the Rx hash queue. * @param rss_key_len @@ -1266,22 +1329,26 @@ mlx5_priv_ind_table_ibv_verify(struct priv *priv) * Number of queues. * * @return - * An hash Rx queue on success. + * The Verbs object initialised, NULL otherwise and rte_errno is set. */ -struct mlx5_hrxq* -mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, - uint64_t hash_fields, uint16_t queues[], uint16_t queues_n) +struct mlx5_hrxq * +mlx5_hrxq_new(struct rte_eth_dev *dev, uint8_t *rss_key, uint8_t rss_key_len, + uint64_t hash_fields, uint16_t queues[], uint16_t queues_n) { + struct priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; struct mlx5_ind_table_ibv *ind_tbl; struct ibv_qp *qp; + int err; queues_n = hash_fields ? queues_n : 1; - ind_tbl = mlx5_priv_ind_table_ibv_get(priv, queues, queues_n); - if (!ind_tbl) - ind_tbl = mlx5_priv_ind_table_ibv_new(priv, queues, queues_n); + ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n); if (!ind_tbl) + ind_tbl = mlx5_ind_table_ibv_new(dev, queues, queues_n); + if (!ind_tbl) { + rte_errno = ENOMEM; return NULL; + } qp = ibv_create_qp_ex( priv->ctx, &(struct ibv_qp_init_attr_ex){ @@ -1299,8 +1366,10 @@ mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, .rwq_ind_tbl = ind_tbl->ind_table, .pd = priv->pd, }); - if (!qp) + if (!qp) { + rte_errno = errno; goto error; + } hrxq = rte_calloc(__func__, 1, sizeof(*hrxq) + rss_key_len, 0); if (!hrxq) goto error; @@ -1311,21 +1380,24 @@ mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, memcpy(hrxq->rss_key, rss_key, rss_key_len); rte_atomic32_inc(&hrxq->refcnt); LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next); - DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv, - (void *)hrxq, rte_atomic32_read(&hrxq->refcnt)); + DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d", + dev->data->port_id, (void *)hrxq, + rte_atomic32_read(&hrxq->refcnt)); return hrxq; error: - mlx5_priv_ind_table_ibv_release(priv, ind_tbl); + err = rte_errno; /* Save rte_errno before cleanup. */ + mlx5_ind_table_ibv_release(dev, ind_tbl); if (qp) claim_zero(ibv_destroy_qp(qp)); + rte_errno = err; /* Restore rte_errno. */ return NULL; } /** * Get an Rx Hash queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param rss_conf * RSS configuration for the Rx hash queue. * @param queues @@ -1337,10 +1409,11 @@ error: * @return * An hash Rx queue on success. */ -struct mlx5_hrxq* -mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, - uint64_t hash_fields, uint16_t queues[], uint16_t queues_n) +struct mlx5_hrxq * +mlx5_hrxq_get(struct rte_eth_dev *dev, uint8_t *rss_key, uint8_t rss_key_len, + uint64_t hash_fields, uint16_t queues[], uint16_t queues_n) { + struct priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; queues_n = hash_fields ? queues_n : 1; @@ -1353,16 +1426,17 @@ mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, continue; if (hrxq->hash_fields != hash_fields) continue; - ind_tbl = mlx5_priv_ind_table_ibv_get(priv, queues, queues_n); + ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n); if (!ind_tbl) continue; if (ind_tbl != hrxq->ind_table) { - mlx5_priv_ind_table_ibv_release(priv, ind_tbl); + mlx5_ind_table_ibv_release(dev, ind_tbl); continue; } rte_atomic32_inc(&hrxq->refcnt); - DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv, - (void *)hrxq, rte_atomic32_read(&hrxq->refcnt)); + DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d", + dev->data->port_id, (void *)hrxq, + rte_atomic32_read(&hrxq->refcnt)); return hrxq; } return NULL; @@ -1371,47 +1445,51 @@ mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len, /** * Release the hash Rx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param hrxq * Pointer to Hash Rx queue to release. * * @return - * 0 on success, errno value on failure. + * 1 while a reference on it exists, 0 when freed. */ int -mlx5_priv_hrxq_release(struct priv *priv, struct mlx5_hrxq *hrxq) +mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq) { - DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv, - (void *)hrxq, rte_atomic32_read(&hrxq->refcnt)); + DRV_LOG(DEBUG, "port %u hash Rx queue %p: refcnt %d", + dev->data->port_id, (void *)hrxq, + rte_atomic32_read(&hrxq->refcnt)); if (rte_atomic32_dec_and_test(&hrxq->refcnt)) { claim_zero(ibv_destroy_qp(hrxq->qp)); - mlx5_priv_ind_table_ibv_release(priv, hrxq->ind_table); + mlx5_ind_table_ibv_release(dev, hrxq->ind_table); LIST_REMOVE(hrxq, next); rte_free(hrxq); return 0; } - claim_nonzero(mlx5_priv_ind_table_ibv_release(priv, hrxq->ind_table)); - return EBUSY; + claim_nonzero(mlx5_ind_table_ibv_release(dev, hrxq->ind_table)); + return 1; } /** * Verify the Rx Queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * - * @return the number of object not released. + * @return + * The number of object not released. */ int -mlx5_priv_hrxq_ibv_verify(struct priv *priv) +mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_hrxq *hrxq; int ret = 0; LIST_FOREACH(hrxq, &priv->hrxqs, next) { - DEBUG("%p: Verbs Hash Rx queue %p still referenced", - (void *)priv, (void *)hrxq); + DRV_LOG(DEBUG, + "port %u Verbs hash Rx queue %p still referenced", + dev->data->port_id, (void *)hrxq); ++ret; } return ret; diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx.c b/dpdk/drivers/net/mlx5/mlx5_rxtx.c index 32bfa307..1bbce3b7 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxtx.c +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx.c @@ -114,6 +114,14 @@ mlx5_set_ptype_table(void) RTE_PTYPE_L4_TCP; (*p)[0x06] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP; + (*p)[0x0d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x0e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x11] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x12] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; /* UDP */ (*p)[0x09] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP; @@ -132,6 +140,14 @@ mlx5_set_ptype_table(void) RTE_PTYPE_L4_TCP; (*p)[0x86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP; + (*p)[0x8d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x8e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x91] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; + (*p)[0x92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_L4_TCP; (*p)[0x89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP; (*p)[0x8a] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | @@ -169,12 +185,36 @@ mlx5_set_ptype_table(void) (*p)[0x46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP; + (*p)[0x4d] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; + (*p)[0x4e] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; + (*p)[0x51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; + (*p)[0x52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; (*p)[0xc5] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP; (*p)[0xc6] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L4_TCP; + (*p)[0xcd] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; + (*p)[0xce] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; + (*p)[0xd1] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; + (*p)[0xd2] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | + RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN | + RTE_PTYPE_INNER_L4_TCP; /* Tunneled - UDP */ (*p)[0x49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN | @@ -370,7 +410,6 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) unsigned int ds = 0; unsigned int sg = 0; /* counter of additional segs attached. */ uintptr_t addr; - uint64_t naddr; uint16_t pkt_inline_sz = MLX5_WQE_DWORD_SIZE + 2; uint16_t tso_header_sz = 0; uint16_t ehdr; @@ -594,12 +633,12 @@ mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) ds = 3; use_dseg: /* Add the remaining packet as a simple ds. */ - naddr = rte_cpu_to_be_64(addr); + addr = rte_cpu_to_be_64(addr); *dseg = (rte_v128u32_t){ rte_cpu_to_be_32(length), mlx5_tx_mb2mr(txq, buf), - naddr, - naddr >> 32, + addr, + addr >> 32, }; ++ds; if (!segs_n) @@ -633,12 +672,12 @@ next_seg: total_length += length; #endif /* Store segment information. */ - naddr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, uintptr_t)); + addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, uintptr_t)); *dseg = (rte_v128u32_t){ rte_cpu_to_be_32(length), mlx5_tx_mb2mr(txq, buf), - naddr, - naddr >> 32, + addr, + addr >> 32, }; (*txq->elts)[++elts_head & elts_m] = buf; ++sg; @@ -701,6 +740,8 @@ next_wqe: /* Check whether completion threshold has been reached. */ comp = txq->elts_comp + i + j + k; if (comp >= MLX5_TX_COMP_THRESH) { + /* A CQE slot must always be available. */ + assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci)); /* Request completion on last WQE. */ last_wqe->ctrl2 = rte_cpu_to_be_32(8); /* Save elts_head in unused "immediate" field of WQE. */ @@ -912,6 +953,8 @@ mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) if (comp >= MLX5_TX_COMP_THRESH) { volatile struct mlx5_wqe *wqe = mpw.wqe; + /* A CQE slot must always be available. */ + assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci)); /* Request completion on last WQE. */ wqe->ctrl[2] = rte_cpu_to_be_32(8); /* Save elts_head in unused "immediate" field of WQE. */ @@ -1204,6 +1247,8 @@ mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, if (comp >= MLX5_TX_COMP_THRESH) { volatile struct mlx5_wqe *wqe = mpw.wqe; + /* A CQE slot must always be available. */ + assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci)); /* Request completion on last WQE. */ wqe->ctrl[2] = rte_cpu_to_be_32(8); /* Save elts_head in unused "immediate" field of WQE. */ @@ -1331,16 +1376,12 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) /* Start processing. */ mlx5_tx_complete(txq); max_elts = (elts_n - (elts_head - txq->elts_tail)); - /* A CQE slot must always be available. */ - assert((1u << txq->cqe_n) - (txq->cq_pi - txq->cq_ci)); max_wqe = (1u << txq->wqe_n) - (txq->wqe_ci - txq->wqe_pi); if (unlikely(!max_wqe)) return 0; do { struct rte_mbuf *buf = *(pkts++); uintptr_t addr; - uint64_t naddr; - unsigned int n; unsigned int do_inline = 0; /* Whether inline is possible. */ uint32_t length; unsigned int segs_n = buf->nb_segs; @@ -1459,7 +1500,7 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) max_wqe--; else max_wqe -= 2; - } else if (do_inline) { + } else if (max_inline && do_inline) { /* Inline packet into WQE. */ unsigned int max; @@ -1517,16 +1558,13 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) ((uintptr_t)mpw.data.raw + inl_pad); (*txq->elts)[elts_head++ & elts_m] = buf; - addr = rte_pktmbuf_mtod(buf, uintptr_t); - for (n = 0; n * RTE_CACHE_LINE_SIZE < length; n++) - rte_prefetch2((void *)(addr + - n * RTE_CACHE_LINE_SIZE)); - naddr = rte_cpu_to_be_64(addr); + addr = rte_cpu_to_be_64(rte_pktmbuf_mtod(buf, + uintptr_t)); *dseg = (rte_v128u32_t) { rte_cpu_to_be_32(length), mlx5_tx_mb2mr(txq, buf), - naddr, - naddr >> 32, + addr, + addr >> 32, }; mpw.data.raw = (volatile void *)(dseg + 1); mpw.total_len += (inl_pad + sizeof(*dseg)); @@ -1550,13 +1588,14 @@ mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) (1 << txq->wqe_n) / MLX5_TX_COMP_THRESH_INLINE_DIV) { volatile struct mlx5_wqe *wqe = mpw.wqe; + /* A CQE slot must always be available. */ + assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci)); /* Request completion on last WQE. */ wqe->ctrl[2] = rte_cpu_to_be_32(8); /* Save elts_head in unused "immediate" field of WQE. */ wqe->ctrl[3] = elts_head; txq->elts_comp = 0; txq->mpw_comp = txq->wqe_ci; - txq->cq_pi++; } else { txq->elts_comp += j; } @@ -1677,6 +1716,7 @@ mlx5_rx_poll_len(struct mlx5_rxq_data *rxq, volatile struct mlx5_cqe *cqe, return 0; ++rxq->cq_ci; op_own = cqe->op_own; + rte_io_rmb(); if (MLX5_CQE_FORMAT(op_own) == MLX5_COMPRESSED) { volatile struct mlx5_mini_cqe8 (*mc)[8] = (volatile struct mlx5_mini_cqe8 (*)[8]) @@ -1934,11 +1974,10 @@ skip: * Number of packets successfully transmitted (<= pkts_n). */ uint16_t -removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) +removed_tx_burst(void *dpdk_txq __rte_unused, + struct rte_mbuf **pkts __rte_unused, + uint16_t pkts_n __rte_unused) { - (void)dpdk_txq; - (void)pkts; - (void)pkts_n; return 0; } @@ -1959,11 +1998,10 @@ removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) * Number of packets successfully received (<= pkts_n). */ uint16_t -removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) +removed_rx_burst(void *dpdk_txq __rte_unused, + struct rte_mbuf **pkts __rte_unused, + uint16_t pkts_n __rte_unused) { - (void)dpdk_rxq; - (void)pkts; - (void)pkts_n; return 0; } @@ -1975,56 +2013,49 @@ removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) */ uint16_t __attribute__((weak)) -mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) +mlx5_tx_burst_raw_vec(void *dpdk_txq __rte_unused, + struct rte_mbuf **pkts __rte_unused, + uint16_t pkts_n __rte_unused) { - (void)dpdk_txq; - (void)pkts; - (void)pkts_n; return 0; } uint16_t __attribute__((weak)) -mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) +mlx5_tx_burst_vec(void *dpdk_txq __rte_unused, + struct rte_mbuf **pkts __rte_unused, + uint16_t pkts_n __rte_unused) { - (void)dpdk_txq; - (void)pkts; - (void)pkts_n; return 0; } uint16_t __attribute__((weak)) -mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) +mlx5_rx_burst_vec(void *dpdk_txq __rte_unused, + struct rte_mbuf **pkts __rte_unused, + uint16_t pkts_n __rte_unused) { - (void)dpdk_rxq; - (void)pkts; - (void)pkts_n; return 0; } int __attribute__((weak)) -priv_check_raw_vec_tx_support(struct priv *priv) +mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev __rte_unused) { - (void)priv; return -ENOTSUP; } int __attribute__((weak)) -priv_check_vec_tx_support(struct priv *priv) +mlx5_check_vec_tx_support(struct rte_eth_dev *dev __rte_unused) { - (void)priv; return -ENOTSUP; } int __attribute__((weak)) -rxq_check_vec_support(struct mlx5_rxq_data *rxq) +mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused) { - (void)rxq; return -ENOTSUP; } int __attribute__((weak)) -priv_check_vec_rx_support(struct priv *priv) +mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused) { - (void)priv; return -ENOTSUP; } diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx.h b/dpdk/drivers/net/mlx5/mlx5_rxtx.h index de5b769e..dac3b39f 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxtx.h +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx.h @@ -154,6 +154,7 @@ struct mlx5_rxq_ctrl { struct mlx5_rxq_data rxq; /* Data path structure. */ unsigned int socket; /* CPU socket ID for allocations. */ unsigned int irq:1; /* Whether IRQ is enabled. */ + uint16_t idx; /* Queue index. */ }; /* Indirection table. */ @@ -184,7 +185,9 @@ struct mlx5_txq_data { uint16_t elts_comp; /* Counter since last completion request. */ uint16_t mpw_comp; /* WQ index since last completion request. */ uint16_t cq_ci; /* Consumer index for completion queue. */ +#ifndef NDEBUG uint16_t cq_pi; /* Producer index for completion queue. */ +#endif uint16_t wqe_ci; /* Consumer index for work queue. */ uint16_t wqe_pi; /* Producer index for work queue. */ uint16_t elts_n:4; /* (*elts)[] length (in log2). */ @@ -204,7 +207,7 @@ struct mlx5_txq_data { volatile void *wqes; /* Work queue (use volatile to write into). */ volatile uint32_t *qp_db; /* Work queue doorbell. */ volatile uint32_t *cq_db; /* Completion queue doorbell. */ - volatile void *bf_reg; /* Blueflame register. */ + volatile void *bf_reg; /* Blueflame register remapped. */ struct mlx5_mr *mp2mr[MLX5_PMD_TX_MP_CACHE]; /* MR translation table. */ struct rte_mbuf *(*elts)[]; /* TX elements. */ struct mlx5_txq_stats stats; /* TX queue counters. */ @@ -214,6 +217,7 @@ struct mlx5_txq_data { struct mlx5_txq_ibv { LIST_ENTRY(mlx5_txq_ibv) next; /* Pointer to the next element. */ rte_atomic32_t refcnt; /* Reference counter. */ + struct mlx5_txq_ctrl *txq_ctrl; /* Pointer to the control queue. */ struct ibv_cq *cq; /* Completion Queue. */ struct ibv_qp *qp; /* Queue Pair. */ }; @@ -229,6 +233,8 @@ struct mlx5_txq_ctrl { struct mlx5_txq_ibv *ibv; /* Verbs queue object. */ struct mlx5_txq_data txq; /* Data path structure. */ off_t uar_mmap_offset; /* UAR mmap offset for non-primary process. */ + volatile void *bf_reg_orig; /* Blueflame register from verbs. */ + uint16_t idx; /* Queue index. */ }; /* mlx5_rxq.c */ @@ -236,93 +242,104 @@ struct mlx5_txq_ctrl { extern uint8_t rss_hash_default_key[]; extern const size_t rss_hash_default_key_len; -void mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *); -int mlx5_rx_queue_setup(struct rte_eth_dev *, uint16_t, uint16_t, unsigned int, - const struct rte_eth_rxconf *, struct rte_mempool *); -void mlx5_rx_queue_release(void *); -int priv_rx_intr_vec_enable(struct priv *priv); -void priv_rx_intr_vec_disable(struct priv *priv); +void mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *rxq_ctrl); +int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, const struct rte_eth_rxconf *conf, + struct rte_mempool *mp); +void mlx5_rx_queue_release(void *dpdk_rxq); +int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev); +void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev); int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id); int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id); -struct mlx5_rxq_ibv *mlx5_priv_rxq_ibv_new(struct priv *, uint16_t); -struct mlx5_rxq_ibv *mlx5_priv_rxq_ibv_get(struct priv *, uint16_t); -int mlx5_priv_rxq_ibv_release(struct priv *, struct mlx5_rxq_ibv *); -int mlx5_priv_rxq_ibv_releasable(struct priv *, struct mlx5_rxq_ibv *); -int mlx5_priv_rxq_ibv_verify(struct priv *); -struct mlx5_rxq_ctrl *mlx5_priv_rxq_new(struct priv *, uint16_t, - uint16_t, unsigned int, - struct rte_mempool *); -struct mlx5_rxq_ctrl *mlx5_priv_rxq_get(struct priv *, uint16_t); -int mlx5_priv_rxq_release(struct priv *, uint16_t); -int mlx5_priv_rxq_releasable(struct priv *, uint16_t); -int mlx5_priv_rxq_verify(struct priv *); -int rxq_alloc_elts(struct mlx5_rxq_ctrl *); -struct mlx5_ind_table_ibv *mlx5_priv_ind_table_ibv_new(struct priv *, - uint16_t [], - uint16_t); -struct mlx5_ind_table_ibv *mlx5_priv_ind_table_ibv_get(struct priv *, - uint16_t [], - uint16_t); -int mlx5_priv_ind_table_ibv_release(struct priv *, struct mlx5_ind_table_ibv *); -int mlx5_priv_ind_table_ibv_verify(struct priv *); -struct mlx5_hrxq *mlx5_priv_hrxq_new(struct priv *, uint8_t *, uint8_t, - uint64_t, uint16_t [], uint16_t); -struct mlx5_hrxq *mlx5_priv_hrxq_get(struct priv *, uint8_t *, uint8_t, - uint64_t, uint16_t [], uint16_t); -int mlx5_priv_hrxq_release(struct priv *, struct mlx5_hrxq *); -int mlx5_priv_hrxq_ibv_verify(struct priv *); +struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx); +struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv); +int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv); +int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev); +struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, + uint16_t desc, unsigned int socket, + struct rte_mempool *mp); +struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_rxq_verify(struct rte_eth_dev *dev); +int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl); +struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_new(struct rte_eth_dev *dev, + uint16_t queues[], + uint16_t queues_n); +struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev, + uint16_t queues[], + uint16_t queues_n); +int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev, + struct mlx5_ind_table_ibv *ind_tbl); +int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev); +struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev, uint8_t *rss_key, + uint8_t rss_key_len, uint64_t hash_fields, + uint16_t queues[], uint16_t queues_n); +struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev, uint8_t *rss_key, + uint8_t rss_key_len, uint64_t hash_fields, + uint16_t queues[], uint16_t queues_n); +int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq); +int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev); /* mlx5_txq.c */ -int mlx5_tx_queue_setup(struct rte_eth_dev *, uint16_t, uint16_t, unsigned int, - const struct rte_eth_txconf *); -void mlx5_tx_queue_release(void *); -int priv_tx_uar_remap(struct priv *priv, int fd); -struct mlx5_txq_ibv *mlx5_priv_txq_ibv_new(struct priv *, uint16_t); -struct mlx5_txq_ibv *mlx5_priv_txq_ibv_get(struct priv *, uint16_t); -int mlx5_priv_txq_ibv_release(struct priv *, struct mlx5_txq_ibv *); -int mlx5_priv_txq_ibv_releasable(struct priv *, struct mlx5_txq_ibv *); -int mlx5_priv_txq_ibv_verify(struct priv *); -struct mlx5_txq_ctrl *mlx5_priv_txq_new(struct priv *, uint16_t, - uint16_t, unsigned int, - const struct rte_eth_txconf *); -struct mlx5_txq_ctrl *mlx5_priv_txq_get(struct priv *, uint16_t); -int mlx5_priv_txq_release(struct priv *, uint16_t); -int mlx5_priv_txq_releasable(struct priv *, uint16_t); -int mlx5_priv_txq_verify(struct priv *); -void txq_alloc_elts(struct mlx5_txq_ctrl *); +int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, const struct rte_eth_txconf *conf); +void mlx5_tx_queue_release(void *dpdk_txq); +int mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd); +struct mlx5_txq_ibv *mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx); +struct mlx5_txq_ibv *mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv); +int mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv); +int mlx5_txq_ibv_verify(struct rte_eth_dev *dev); +struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, + uint16_t desc, unsigned int socket, + const struct rte_eth_txconf *conf); +struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx); +int mlx5_txq_verify(struct rte_eth_dev *dev); +void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl); /* mlx5_rxtx.c */ extern uint32_t mlx5_ptype_table[]; void mlx5_set_ptype_table(void); -uint16_t mlx5_tx_burst(void *, struct rte_mbuf **, uint16_t); -uint16_t mlx5_tx_burst_mpw(void *, struct rte_mbuf **, uint16_t); -uint16_t mlx5_tx_burst_mpw_inline(void *, struct rte_mbuf **, uint16_t); -uint16_t mlx5_tx_burst_empw(void *, struct rte_mbuf **, uint16_t); -uint16_t mlx5_rx_burst(void *, struct rte_mbuf **, uint16_t); -uint16_t removed_tx_burst(void *, struct rte_mbuf **, uint16_t); -uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t); -int mlx5_rx_descriptor_status(void *, uint16_t); -int mlx5_tx_descriptor_status(void *, uint16_t); +uint16_t mlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n); +uint16_t mlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n); +uint16_t mlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n); +uint16_t mlx5_tx_burst_empw(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n); +uint16_t mlx5_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); +uint16_t removed_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n); +uint16_t removed_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, + uint16_t pkts_n); +int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset); +int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset); /* Vectorized version of mlx5_rxtx.c */ -int priv_check_raw_vec_tx_support(struct priv *); -int priv_check_vec_tx_support(struct priv *); -int rxq_check_vec_support(struct mlx5_rxq_data *); -int priv_check_vec_rx_support(struct priv *); -uint16_t mlx5_tx_burst_raw_vec(void *, struct rte_mbuf **, uint16_t); -uint16_t mlx5_tx_burst_vec(void *, struct rte_mbuf **, uint16_t); -uint16_t mlx5_rx_burst_vec(void *, struct rte_mbuf **, uint16_t); +int mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev); +int mlx5_check_vec_tx_support(struct rte_eth_dev *dev); +int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data); +int mlx5_check_vec_rx_support(struct rte_eth_dev *dev); +uint16_t mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n); +uint16_t mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n); +uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts, + uint16_t pkts_n); /* mlx5_mr.c */ -void mlx5_mp2mr_iter(struct rte_mempool *, void *); -struct mlx5_mr *priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *, - struct rte_mempool *, unsigned int); -struct mlx5_mr *mlx5_txq_mp2mr_reg(struct mlx5_txq_data *, struct rte_mempool *, - unsigned int); +void mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg); +struct mlx5_mr *mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, + struct rte_mempool *mp, unsigned int idx); #ifndef NDEBUG /** @@ -385,9 +402,10 @@ check_cqe(volatile struct mlx5_cqe *cqe, (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR)) return 0; if (!check_cqe_seen(cqe)) { - ERROR("unexpected CQE error %u (0x%02x)" - " syndrome 0x%02x", - op_code, op_code, syndrome); + DRV_LOG(ERR, + "unexpected CQE error %u (0x%02x) syndrome" + " 0x%02x", + op_code, op_code, syndrome); rte_hexdump(stderr, "MLX5 Error CQE:", (const void *)((uintptr_t)err_cqe), sizeof(*err_cqe)); @@ -396,8 +414,8 @@ check_cqe(volatile struct mlx5_cqe *cqe, } else if ((op_code != MLX5_CQE_RESP_SEND) && (op_code != MLX5_CQE_REQ)) { if (!check_cqe_seen(cqe)) { - ERROR("unexpected CQE opcode %u (0x%02x)", - op_code, op_code); + DRV_LOG(ERR, "unexpected CQE opcode %u (0x%02x)", + op_code, op_code); rte_hexdump(stderr, "MLX5 CQE:", (const void *)((uintptr_t)cqe), sizeof(*cqe)); @@ -457,7 +475,7 @@ mlx5_tx_complete(struct mlx5_txq_data *txq) if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) || (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) { if (!check_cqe_seen(cqe)) { - ERROR("unexpected error CQE, TX stopped"); + DRV_LOG(ERR, "unexpected error CQE, Tx stopped"); rte_hexdump(stderr, "MLX5 TXQ:", (const void *)((uintptr_t)txq->wqes), ((1 << txq->wqe_n) * @@ -558,8 +576,6 @@ mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb) if (txq->mp2mr[i]->start <= addr && txq->mp2mr[i]->end > addr) { assert(txq->mp2mr[i]->lkey != (uint32_t)-1); - assert(rte_cpu_to_be_32(txq->mp2mr[i]->mr->lkey) == - txq->mp2mr[i]->lkey); txq->mr_cache_idx = i; return txq->mp2mr[i]->lkey; } @@ -573,6 +589,11 @@ mlx5_tx_mb2mr(struct mlx5_txq_data *txq, struct rte_mbuf *mb) rte_atomic32_inc(&mr->refcnt); txq->mr_cache_idx = i >= RTE_DIM(txq->mp2mr) ? i - 1 : i; return mr->lkey; + } else { + struct rte_mempool *mp = mlx5_tx_mb2mp(mb); + + DRV_LOG(WARNING, "failed to register mempool 0x%p(%s)", + (void *)mp, mp->name); } return (uint32_t)-1; } diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c index 101aa156..982b8f1f 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c @@ -276,15 +276,16 @@ mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) /** * Check Tx queue flags are set for raw vectorized Tx. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -priv_check_raw_vec_tx_support(struct priv *priv) +mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; uint16_t i; /* All the configured queues should support. */ @@ -303,15 +304,17 @@ priv_check_raw_vec_tx_support(struct priv *priv) /** * Check a device can support vectorized TX. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -priv_check_vec_tx_support(struct priv *priv) +mlx5_check_vec_tx_support(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; + if (!priv->tx_vec_en || priv->txqs_n > MLX5_VPMD_MIN_TXQS || priv->mps != MLX5_MPW_ENHANCED || @@ -330,7 +333,7 @@ priv_check_vec_tx_support(struct priv *priv) * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -rxq_check_vec_support(struct mlx5_rxq_data *rxq) +mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq) { struct mlx5_rxq_ctrl *ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq); @@ -343,15 +346,16 @@ rxq_check_vec_support(struct mlx5_rxq_data *rxq) /** * Check a device can support vectorized RX. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return * 1 if supported, negative errno value if not. */ int __attribute__((cold)) -priv_check_vec_rx_support(struct priv *priv) +mlx5_check_vec_rx_support(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; uint16_t i; if (!priv->rx_vec_en) @@ -362,7 +366,7 @@ priv_check_vec_rx_support(struct priv *priv) if (!rxq) continue; - if (rxq_check_vec_support(rxq) < 0) + if (mlx5_rxq_check_vec_support(rxq) < 0) break; } if (i != priv->rxqs_n) diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h index 1f08ed0b..d504e2ae 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h @@ -106,9 +106,9 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq, uint16_t n) volatile struct mlx5_wqe_data_seg *wq = &(*rxq->wqes)[elts_idx]; unsigned int i; - assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH); + assert(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n)); assert(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi))); - assert(MLX5_VPMD_RXQ_RPLNSH_THRESH > MLX5_VPMD_DESCS_PER_LOOP); + assert(MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n) > MLX5_VPMD_DESCS_PER_LOOP); /* Not to cross queue end. */ n = RTE_MIN(n - MLX5_VPMD_DESCS_PER_LOOP, q_n - elts_idx); if (rte_mempool_get_bulk(rxq->mp, (void *)elts, n) < 0) { diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h index 06f83ef1..e748615e 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h @@ -193,8 +193,8 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, vst1q_u8((void *)t_wqe, ctrl); /* Fill ESEG in the header. */ vst1q_u16((void *)(t_wqe + 1), - (uint16x8_t) { 0, 0, cs_flags, rte_cpu_to_be_16(len), - 0, 0, 0, 0 }); + ((uint16x8_t) { 0, 0, cs_flags, rte_cpu_to_be_16(len), + 0, 0, 0, 0 })); txq->wqe_ci = wqe_ci; } if (!n) @@ -202,10 +202,11 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, txq->elts_comp += (uint16_t)(elts_head - txq->elts_head); txq->elts_head = elts_head; if (txq->elts_comp >= MLX5_TX_COMP_THRESH) { + /* A CQE slot must always be available. */ + assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci)); wqe->ctrl[2] = rte_cpu_to_be_32(8); wqe->ctrl[3] = txq->elts_head; txq->elts_comp = 0; - ++txq->cq_pi; } #ifdef MLX5_PMD_SOFT_COUNTERS txq->stats.opackets += n; @@ -304,9 +305,10 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n, if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) { txq->elts_comp += pkts_n; } else { + /* A CQE slot must always be available. */ + assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci)); /* Request a completion. */ txq->elts_comp = 0; - ++txq->cq_pi; comp_req = 8; } /* Fill CTRL in the header. */ @@ -320,10 +322,10 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n, vst1q_u8((void *)t_wqe, ctrl); /* Fill ESEG in the header. */ vst1q_u8((void *)(t_wqe + 1), - (uint8x16_t) { 0, 0, 0, 0, - cs_flags, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, 0, 0 }); + ((uint8x16_t) { 0, 0, 0, 0, + cs_flags, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, 0, 0 })); #ifdef MLX5_PMD_SOFT_COUNTERS txq->stats.opackets += pkts_n; #endif @@ -754,7 +756,7 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, * N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished). */ repl_n = q_n - (rxq->rq_ci - rxq->rq_pi); - if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH) + if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n)) mlx5_rx_replenish_bulk_mbuf(rxq, repl_n); /* See if there're unreturned mbufs from compressed CQE. */ rcvd_pkt = rxq->cq_ci - rxq->rq_pi; @@ -806,6 +808,7 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, uint16x4_t mask; uint16x4_t byte_cnt; uint32x4_t ptype_info, flow_tag; + register uint64x2_t c0, c1, c2, c3; uint8_t *p0, *p1, *p2, *p3; uint8_t *e0 = (void *)&elts[pos]->pkt_len; uint8_t *e1 = (void *)&elts[pos + 1]->pkt_len; @@ -822,6 +825,16 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, p1 = p0 + (pkts_n - pos > 1) * sizeof(struct mlx5_cqe); p2 = p1 + (pkts_n - pos > 2) * sizeof(struct mlx5_cqe); p3 = p2 + (pkts_n - pos > 3) * sizeof(struct mlx5_cqe); + /* B.0 (CQE 3) load a block having op_own. */ + c3 = vld1q_u64((uint64_t *)(p3 + 48)); + /* B.0 (CQE 2) load a block having op_own. */ + c2 = vld1q_u64((uint64_t *)(p2 + 48)); + /* B.0 (CQE 1) load a block having op_own. */ + c1 = vld1q_u64((uint64_t *)(p1 + 48)); + /* B.0 (CQE 0) load a block having op_own. */ + c0 = vld1q_u64((uint64_t *)(p0 + 48)); + /* Synchronize for loading the rest of blocks. */ + rte_io_rmb(); /* Prefetch next 4 CQEs. */ if (pkts_n - pos >= 2 * MLX5_VPMD_DESCS_PER_LOOP) { unsigned int next = pos + MLX5_VPMD_DESCS_PER_LOOP; @@ -831,50 +844,46 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, rte_prefetch_non_temporal(&cq[next + 3]); } __asm__ volatile ( - /* B.1 (CQE 3) load a block having op_own. */ - "ld1 {v19.16b}, [%[p3]] \n\t" - "sub %[p3], %[p3], #48 \n\t" - /* B.2 (CQE 3) load the rest blocks. */ + /* B.1 (CQE 3) load the rest of blocks. */ "ld1 {v16.16b - v18.16b}, [%[p3]] \n\t" + /* B.2 (CQE 3) move the block having op_own. */ + "mov v19.16b, %[c3].16b \n\t" /* B.3 (CQE 3) extract 16B fields. */ "tbl v23.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t" + /* B.1 (CQE 2) load the rest of blocks. */ + "ld1 {v16.16b - v18.16b}, [%[p2]] \n\t" /* B.4 (CQE 3) adjust CRC length. */ "sub v23.8h, v23.8h, %[crc_adj].8h \n\t" - /* B.1 (CQE 2) load a block having op_own. */ - "ld1 {v19.16b}, [%[p2]] \n\t" - "sub %[p2], %[p2], #48 \n\t" /* C.1 (CQE 3) generate final structure for mbuf. */ "tbl v15.16b, {v23.16b}, %[mb_shuf_m].16b \n\t" - /* B.2 (CQE 2) load the rest blocks. */ - "ld1 {v16.16b - v18.16b}, [%[p2]] \n\t" + /* B.2 (CQE 2) move the block having op_own. */ + "mov v19.16b, %[c2].16b \n\t" /* B.3 (CQE 2) extract 16B fields. */ "tbl v22.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t" + /* B.1 (CQE 1) load the rest of blocks. */ + "ld1 {v16.16b - v18.16b}, [%[p1]] \n\t" /* B.4 (CQE 2) adjust CRC length. */ "sub v22.8h, v22.8h, %[crc_adj].8h \n\t" - /* B.1 (CQE 1) load a block having op_own. */ - "ld1 {v19.16b}, [%[p1]] \n\t" - "sub %[p1], %[p1], #48 \n\t" /* C.1 (CQE 2) generate final structure for mbuf. */ "tbl v14.16b, {v22.16b}, %[mb_shuf_m].16b \n\t" - /* B.2 (CQE 1) load the rest blocks. */ - "ld1 {v16.16b - v18.16b}, [%[p1]] \n\t" + /* B.2 (CQE 1) move the block having op_own. */ + "mov v19.16b, %[c1].16b \n\t" /* B.3 (CQE 1) extract 16B fields. */ "tbl v21.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t" + /* B.1 (CQE 0) load the rest of blocks. */ + "ld1 {v16.16b - v18.16b}, [%[p0]] \n\t" /* B.4 (CQE 1) adjust CRC length. */ "sub v21.8h, v21.8h, %[crc_adj].8h \n\t" - /* B.1 (CQE 0) load a block having op_own. */ - "ld1 {v19.16b}, [%[p0]] \n\t" - "sub %[p0], %[p0], #48 \n\t" /* C.1 (CQE 1) generate final structure for mbuf. */ "tbl v13.16b, {v21.16b}, %[mb_shuf_m].16b \n\t" - /* B.2 (CQE 0) load the rest blocks. */ - "ld1 {v16.16b - v18.16b}, [%[p0]] \n\t" + /* B.2 (CQE 0) move the block having op_own. */ + "mov v19.16b, %[c0].16b \n\t" + /* A.1 load mbuf pointers. */ + "ld1 {v24.2d - v25.2d}, [%[elts_p]] \n\t" /* B.3 (CQE 0) extract 16B fields. */ "tbl v20.16b, {v16.16b - v19.16b}, %[cqe_shuf_m].16b \n\t" /* B.4 (CQE 0) adjust CRC length. */ "sub v20.8h, v20.8h, %[crc_adj].8h \n\t" - /* A.1 load mbuf pointers. */ - "ld1 {v24.2d - v25.2d}, [%[elts_p]] \n\t" /* D.1 extract op_own byte. */ "tbl %[op_own].8b, {v20.16b - v23.16b}, %[owner_shuf_m].8b \n\t" /* C.2 (CQE 3) adjust flow mark. */ @@ -909,9 +918,9 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, [byte_cnt]"=&w"(byte_cnt), [ptype_info]"=&w"(ptype_info), [flow_tag]"=&w"(flow_tag) - :[p3]"r"(p3 + 48), [p2]"r"(p2 + 48), - [p1]"r"(p1 + 48), [p0]"r"(p0 + 48), + :[p3]"r"(p3), [p2]"r"(p2), [p1]"r"(p1), [p0]"r"(p0), [e3]"r"(e3), [e2]"r"(e2), [e1]"r"(e1), [e0]"r"(e0), + [c3]"w"(c3), [c2]"w"(c2), [c1]"w"(c1), [c0]"w"(c0), [elts_p]"r"(elts_p), [pkts_p]"r"(pkts_p), [cqe_shuf_m]"w"(cqe_shuf_m), diff --git a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h index 7ef2c59e..7e8c9b88 100644 --- a/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h +++ b/dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h @@ -203,10 +203,11 @@ txq_scatter_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, txq->elts_comp += (uint16_t)(elts_head - txq->elts_head); txq->elts_head = elts_head; if (txq->elts_comp >= MLX5_TX_COMP_THRESH) { + /* A CQE slot must always be available. */ + assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci)); wqe->ctrl[2] = rte_cpu_to_be_32(8); wqe->ctrl[3] = txq->elts_head; txq->elts_comp = 0; - ++txq->cq_pi; } #ifdef MLX5_PMD_SOFT_COUNTERS txq->stats.opackets += n; @@ -305,9 +306,10 @@ txq_burst_v(struct mlx5_txq_data *txq, struct rte_mbuf **pkts, uint16_t pkts_n, if (txq->elts_comp + pkts_n < MLX5_TX_COMP_THRESH) { txq->elts_comp += pkts_n; } else { + /* A CQE slot must always be available. */ + assert((1u << txq->cqe_n) - (txq->cq_pi++ - txq->cq_ci)); /* Request a completion. */ txq->elts_comp = 0; - ++txq->cq_pi; comp_req = 8; } /* Fill CTRL in the header. */ @@ -735,7 +737,7 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, * N - (rq_ci - rq_pi) := # of buffers consumed (to be replenished). */ repl_n = q_n - (rxq->rq_ci - rxq->rq_pi); - if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH) + if (repl_n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n)) mlx5_rx_replenish_bulk_mbuf(rxq, repl_n); /* See if there're unreturned mbufs from compressed CQE. */ rcvd_pkt = rxq->cq_ci - rxq->rq_pi; @@ -825,7 +827,7 @@ rxq_burst_v(struct mlx5_rxq_data *rxq, struct rte_mbuf **pkts, uint16_t pkts_n, /* B.2 copy mbuf pointers. */ _mm_storeu_si128((__m128i *)&pkts[pos], mbp1); _mm_storeu_si128((__m128i *)&pkts[pos + 2], mbp2); - rte_compiler_barrier(); + rte_io_rmb(); /* C.1 load remained CQE data and extract necessary fields. */ cqe_tmp2 = _mm_load_si128((__m128i *)&cq[pos + p3]); cqe_tmp1 = _mm_load_si128((__m128i *)&cq[pos + p2]); diff --git a/dpdk/drivers/net/mlx5/mlx5_socket.c b/dpdk/drivers/net/mlx5/mlx5_socket.c index 5cd1ab80..97f90c07 100644 --- a/dpdk/drivers/net/mlx5/mlx5_socket.c +++ b/dpdk/drivers/net/mlx5/mlx5_socket.c @@ -45,92 +45,105 @@ /** * Initialise the socket to communicate with the secondary process * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_socket_init(struct priv *priv) +mlx5_socket_init(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct sockaddr_un sun = { .sun_family = AF_UNIX, }; int ret; int flags; - struct stat file_stat; + /* + * Close the last socket that was used to communicate + * with the secondary process + */ + if (priv->primary_socket) + mlx5_socket_uninit(dev); /* * Initialise the socket to communicate with the secondary * process. */ ret = socket(AF_UNIX, SOCK_STREAM, 0); if (ret < 0) { - WARN("secondary process not supported: %s", strerror(errno)); - return ret; + rte_errno = errno; + DRV_LOG(WARNING, "port %u secondary process not supported: %s", + dev->data->port_id, strerror(errno)); + goto error; } priv->primary_socket = ret; flags = fcntl(priv->primary_socket, F_GETFL, 0); - if (flags == -1) - goto out; + if (flags == -1) { + rte_errno = errno; + goto error; + } ret = fcntl(priv->primary_socket, F_SETFL, flags | O_NONBLOCK); - if (ret < 0) - goto out; + if (ret < 0) { + rte_errno = errno; + goto error; + } snprintf(sun.sun_path, sizeof(sun.sun_path), "/var/tmp/%s_%d", MLX5_DRIVER_NAME, priv->primary_socket); - ret = stat(sun.sun_path, &file_stat); - if (!ret) - claim_zero(remove(sun.sun_path)); + remove(sun.sun_path); ret = bind(priv->primary_socket, (const struct sockaddr *)&sun, sizeof(sun)); if (ret < 0) { - WARN("cannot bind socket, secondary process not supported: %s", - strerror(errno)); + rte_errno = errno; + DRV_LOG(WARNING, + "port %u cannot bind socket, secondary process not" + " supported: %s", + dev->data->port_id, strerror(errno)); goto close; } ret = listen(priv->primary_socket, 0); if (ret < 0) { - WARN("Secondary process not supported: %s", strerror(errno)); + rte_errno = errno; + DRV_LOG(WARNING, "port %u secondary process not supported: %s", + dev->data->port_id, strerror(errno)); goto close; } - return ret; + return 0; close: remove(sun.sun_path); -out: +error: claim_zero(close(priv->primary_socket)); priv->primary_socket = 0; - return -(ret); + return -rte_errno; } /** * Un-Initialise the socket to communicate with the secondary process * - * @param[in] priv - * Pointer to private structure. - * - * @return - * 0 on success, errno value on failure. + * @param[in] dev */ -int -priv_socket_uninit(struct priv *priv) +void +mlx5_socket_uninit(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; + MKSTR(path, "/var/tmp/%s_%d", MLX5_DRIVER_NAME, priv->primary_socket); claim_zero(close(priv->primary_socket)); priv->primary_socket = 0; claim_zero(remove(path)); - return 0; } /** * Handle socket interrupts. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. */ void -priv_socket_handle(struct priv *priv) +mlx5_socket_handle(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; int conn_sock; int ret = 0; struct cmsghdr *cmsg = NULL; @@ -152,25 +165,30 @@ priv_socket_handle(struct priv *priv) /* Accept the connection from the client. */ conn_sock = accept(priv->primary_socket, NULL, NULL); if (conn_sock < 0) { - WARN("connection failed: %s", strerror(errno)); + DRV_LOG(WARNING, "port %u connection failed: %s", + dev->data->port_id, strerror(errno)); return; } ret = setsockopt(conn_sock, SOL_SOCKET, SO_PASSCRED, &(int){1}, sizeof(int)); if (ret < 0) { - WARN("cannot change socket options"); - goto out; + ret = errno; + DRV_LOG(WARNING, "port %u cannot change socket options: %s", + dev->data->port_id, strerror(rte_errno)); + goto error; } ret = recvmsg(conn_sock, &msg, MSG_WAITALL); if (ret < 0) { - WARN("received an empty message: %s", strerror(errno)); - goto out; + ret = errno; + DRV_LOG(WARNING, "port %u received an empty message: %s", + dev->data->port_id, strerror(rte_errno)); + goto error; } /* Expect to receive credentials only. */ cmsg = CMSG_FIRSTHDR(&msg); if (cmsg == NULL) { - WARN("no message"); - goto out; + DRV_LOG(WARNING, "port %u no message", dev->data->port_id); + goto error; } if ((cmsg->cmsg_type == SCM_CREDENTIALS) && (cmsg->cmsg_len >= sizeof(*cred))) { @@ -179,14 +197,16 @@ priv_socket_handle(struct priv *priv) } cmsg = CMSG_NXTHDR(&msg, cmsg); if (cmsg != NULL) { - WARN("Message wrongly formatted"); - goto out; + DRV_LOG(WARNING, "port %u message wrongly formatted", + dev->data->port_id); + goto error; } /* Make sure all the ancillary data was received and valid. */ if ((cred == NULL) || (cred->uid != getuid()) || (cred->gid != getgid())) { - WARN("wrong credentials"); - goto out; + DRV_LOG(WARNING, "port %u wrong credentials", + dev->data->port_id); + goto error; } /* Set-up the ancillary data. */ cmsg = CMSG_FIRSTHDR(&msg); @@ -198,27 +218,29 @@ priv_socket_handle(struct priv *priv) *fd = priv->ctx->cmd_fd; ret = sendmsg(conn_sock, &msg, 0); if (ret < 0) - WARN("cannot send response"); -out: + DRV_LOG(WARNING, "port %u cannot send response", + dev->data->port_id); +error: close(conn_sock); } /** * Connect to the primary process. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet structure. * * @return - * fd on success, negative errno value on failure. + * fd on success, negative errno value otherwise and rte_errno is set. */ int -priv_socket_connect(struct priv *priv) +mlx5_socket_connect(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct sockaddr_un sun = { .sun_family = AF_UNIX, }; - int socket_fd; + int socket_fd = -1; int *fd = NULL; int ret; struct ucred *cred; @@ -238,57 +260,75 @@ priv_socket_connect(struct priv *priv) ret = socket(AF_UNIX, SOCK_STREAM, 0); if (ret < 0) { - WARN("cannot connect to primary"); - return ret; + rte_errno = errno; + DRV_LOG(WARNING, "port %u cannot connect to primary", + dev->data->port_id); + goto error; } socket_fd = ret; snprintf(sun.sun_path, sizeof(sun.sun_path), "/var/tmp/%s_%d", MLX5_DRIVER_NAME, priv->primary_socket); ret = connect(socket_fd, (const struct sockaddr *)&sun, sizeof(sun)); if (ret < 0) { - WARN("cannot connect to primary"); - goto out; + rte_errno = errno; + DRV_LOG(WARNING, "port %u cannot connect to primary", + dev->data->port_id); + goto error; } cmsg = CMSG_FIRSTHDR(&msg); if (cmsg == NULL) { - DEBUG("cannot get first message"); - goto out; + rte_errno = EINVAL; + DRV_LOG(DEBUG, "port %u cannot get first message", + dev->data->port_id); + goto error; } cmsg->cmsg_level = SOL_SOCKET; cmsg->cmsg_type = SCM_CREDENTIALS; cmsg->cmsg_len = CMSG_LEN(sizeof(*cred)); cred = (struct ucred *)CMSG_DATA(cmsg); if (cred == NULL) { - DEBUG("no credentials received"); - goto out; + rte_errno = EINVAL; + DRV_LOG(DEBUG, "port %u no credentials received", + dev->data->port_id); + goto error; } cred->pid = getpid(); cred->uid = getuid(); cred->gid = getgid(); ret = sendmsg(socket_fd, &msg, MSG_DONTWAIT); if (ret < 0) { - WARN("cannot send credentials to primary: %s", - strerror(errno)); - goto out; + rte_errno = errno; + DRV_LOG(WARNING, + "port %u cannot send credentials to primary: %s", + dev->data->port_id, strerror(errno)); + goto error; } ret = recvmsg(socket_fd, &msg, MSG_WAITALL); if (ret <= 0) { - WARN("no message from primary: %s", strerror(errno)); - goto out; + rte_errno = errno; + DRV_LOG(WARNING, "port %u no message from primary: %s", + dev->data->port_id, strerror(errno)); + goto error; } cmsg = CMSG_FIRSTHDR(&msg); if (cmsg == NULL) { - WARN("No file descriptor received"); - goto out; + rte_errno = EINVAL; + DRV_LOG(WARNING, "port %u no file descriptor received", + dev->data->port_id); + goto error; } fd = (int *)CMSG_DATA(cmsg); - if (*fd <= 0) { - WARN("no file descriptor received: %s", strerror(errno)); - ret = *fd; - goto out; + if (*fd < 0) { + DRV_LOG(WARNING, "port %u no file descriptor received: %s", + dev->data->port_id, strerror(errno)); + rte_errno = *fd; + goto error; } ret = *fd; -out: close(socket_fd); return ret; +error: + if (socket_fd != -1) + close(socket_fd); + return -rte_errno; } diff --git a/dpdk/drivers/net/mlx5/mlx5_stats.c b/dpdk/drivers/net/mlx5/mlx5_stats.c index 2427585f..345ed707 100644 --- a/dpdk/drivers/net/mlx5/mlx5_stats.c +++ b/dpdk/drivers/net/mlx5/mlx5_stats.c @@ -31,8 +31,11 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include #include #include +#include +#include #include #include @@ -47,6 +50,7 @@ struct mlx5_counter_ctrl { char dpdk_name[RTE_ETH_XSTATS_NAME_SIZE]; /* Name of the counter on the device table. */ char ctr_name[RTE_ETH_XSTATS_NAME_SIZE]; + uint32_t ib:1; /**< Nonzero for IB counters. */ }; static const struct mlx5_counter_ctrl mlx5_counters_init[] = { @@ -121,6 +125,7 @@ static const struct mlx5_counter_ctrl mlx5_counters_init[] = { { .dpdk_name = "rx_out_of_buffer", .ctr_name = "out_of_buffer", + .ib = 1, }, }; @@ -129,39 +134,56 @@ static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); /** * Read device counters table. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param[out] stats * Counters table output buffer. * * @return - * 0 on success and stats is filled, negative on error. + * 0 on success and stats is filled, negative errno value otherwise and + * rte_errno is set. */ static int -priv_read_dev_counters(struct priv *priv, uint64_t *stats) +mlx5_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) { + struct priv *priv = dev->data->dev_private; struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; unsigned int i; struct ifreq ifr; unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t); unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz]; struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf; + int ret; et_stats->cmd = ETHTOOL_GSTATS; et_stats->n_stats = xstats_ctrl->stats_n; ifr.ifr_data = (caddr_t)et_stats; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) { - WARN("unable to read statistic values from device"); - return -1; + ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); + if (ret) { + DRV_LOG(WARNING, + "port %u unable to read statistic values from device", + dev->data->port_id); + return ret; } for (i = 0; i != xstats_n; ++i) { - if (priv_is_ib_cntr(mlx5_counters_init[i].ctr_name)) - priv_get_cntr_sysfs(priv, - mlx5_counters_init[i].ctr_name, - &stats[i]); - else + if (mlx5_counters_init[i].ib) { + FILE *file; + MKSTR(path, "%s/ports/1/hw_counters/%s", + priv->ibdev_path, + mlx5_counters_init[i].ctr_name); + + file = fopen(path, "rb"); + if (file) { + int n = fscanf(file, "%" SCNu64, &stats[i]); + + fclose(file); + if (n != 1) + stats[i] = 0; + } + } else { stats[i] = (uint64_t) et_stats->data[xstats_ctrl->dev_table_idx[i]]; + } } return 0; } @@ -169,22 +191,26 @@ priv_read_dev_counters(struct priv *priv, uint64_t *stats) /** * Query the number of statistics provided by ETHTOOL. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * * @return - * Number of statistics on success, -1 on error. + * Number of statistics on success, negative errno value otherwise and + * rte_errno is set. */ static int -priv_ethtool_get_stats_n(struct priv *priv) { +mlx5_ethtool_get_stats_n(struct rte_eth_dev *dev) { struct ethtool_drvinfo drvinfo; struct ifreq ifr; + int ret; drvinfo.cmd = ETHTOOL_GDRVINFO; ifr.ifr_data = (caddr_t)&drvinfo; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) { - WARN("unable to query number of statistics"); - return -1; + ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); + if (ret) { + DRV_LOG(WARNING, "port %u unable to query number of statistics", + dev->data->port_id); + return ret; } return drvinfo.n_stats; } @@ -192,12 +218,13 @@ priv_ethtool_get_stats_n(struct priv *priv) { /** * Init the structures to read device counters. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. */ void -priv_xstats_init(struct priv *priv) +mlx5_xstats_init(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; unsigned int i; unsigned int j; @@ -205,12 +232,15 @@ priv_xstats_init(struct priv *priv) struct ethtool_gstrings *strings = NULL; unsigned int dev_stats_n; unsigned int str_sz; + int ret; - dev_stats_n = priv_ethtool_get_stats_n(priv); - if (dev_stats_n < 1) { - WARN("no extended statistics available"); + ret = mlx5_ethtool_get_stats_n(dev); + if (ret < 0) { + DRV_LOG(WARNING, "port %u no extended statistics available", + dev->data->port_id); return; } + dev_stats_n = ret; xstats_ctrl->stats_n = dev_stats_n; /* Allocate memory to grab stat names and values. */ str_sz = dev_stats_n * ETH_GSTRING_LEN; @@ -218,15 +248,18 @@ priv_xstats_init(struct priv *priv) rte_malloc("xstats_strings", str_sz + sizeof(struct ethtool_gstrings), 0); if (!strings) { - WARN("unable to allocate memory for xstats"); + DRV_LOG(WARNING, "port %u unable to allocate memory for xstats", + dev->data->port_id); return; } strings->cmd = ETHTOOL_GSTRINGS; strings->string_set = ETH_SS_STATS; strings->len = dev_stats_n; ifr.ifr_data = (caddr_t)strings; - if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) { - WARN("unable to get statistic names"); + ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); + if (ret) { + DRV_LOG(WARNING, "port %u unable to get statistic names", + dev->data->port_id); goto free; } for (j = 0; j != xstats_n; ++j) @@ -244,68 +277,67 @@ priv_xstats_init(struct priv *priv) } } for (j = 0; j != xstats_n; ++j) { - if (priv_is_ib_cntr(mlx5_counters_init[j].ctr_name)) + if (mlx5_counters_init[j].ib) continue; if (xstats_ctrl->dev_table_idx[j] >= dev_stats_n) { - WARN("counter \"%s\" is not recognized", - mlx5_counters_init[j].dpdk_name); + DRV_LOG(WARNING, + "port %u counter \"%s\" is not recognized", + dev->data->port_id, + mlx5_counters_init[j].dpdk_name); goto free; } } /* Copy to base at first time. */ assert(xstats_n <= MLX5_MAX_XSTATS); - priv_read_dev_counters(priv, xstats_ctrl->base); + ret = mlx5_read_dev_counters(dev, xstats_ctrl->base); + if (ret) + DRV_LOG(ERR, "port %u cannot read device counters: %s", + dev->data->port_id, strerror(rte_errno)); free: rte_free(strings); } /** - * Get device extended statistics. + * DPDK callback to get extended device statistics. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param[out] stats * Pointer to rte extended stats table. + * @param n + * The size of the stats table. * * @return * Number of extended stats on success and stats is filled, - * negative on error. + * negative on error and rte_errno is set. */ -static int -priv_xstats_get(struct priv *priv, struct rte_eth_xstat *stats) +int +mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats, + unsigned int n) { - struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + struct priv *priv = dev->data->dev_private; unsigned int i; - unsigned int n = xstats_n; uint64_t counters[n]; - if (priv_read_dev_counters(priv, counters) < 0) - return -1; - for (i = 0; i != xstats_n; ++i) { - stats[i].id = i; - stats[i].value = (counters[i] - xstats_ctrl->base[i]); + if (n >= xstats_n && stats) { + struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; + int stats_n; + int ret; + + stats_n = mlx5_ethtool_get_stats_n(dev); + if (stats_n < 0) + return stats_n; + if (xstats_ctrl->stats_n != stats_n) + mlx5_xstats_init(dev); + ret = mlx5_read_dev_counters(dev, counters); + if (ret) + return ret; + for (i = 0; i != xstats_n; ++i) { + stats[i].id = i; + stats[i].value = (counters[i] - xstats_ctrl->base[i]); + } } - return n; -} - -/** - * Reset device extended statistics. - * - * @param priv - * Pointer to private structure. - */ -static void -priv_xstats_reset(struct priv *priv) -{ - struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; - unsigned int i; - unsigned int n = xstats_n; - uint64_t counters[n]; - - if (priv_read_dev_counters(priv, counters) < 0) - return; - for (i = 0; i != n; ++i) - xstats_ctrl->base[i] = counters[i]; + return xstats_n; } /** @@ -315,16 +347,19 @@ priv_xstats_reset(struct priv *priv) * Pointer to Ethernet device structure. * @param[out] stats * Stats structure output buffer. + * + * @return + * 0 on success and stats is filled, negative errno value otherwise and + * rte_errno is set. */ int mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; struct rte_eth_stats tmp = {0}; unsigned int i; unsigned int idx; - priv_lock(priv); /* Add software counters. */ for (i = 0; (i != priv->rxqs_n); ++i) { struct mlx5_rxq_data *rxq = (*priv->rxqs)[i]; @@ -370,7 +405,6 @@ mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) /* FIXME: retrieve and add hardware counters. */ #endif *stats = tmp; - priv_unlock(priv); return 0; } @@ -387,7 +421,6 @@ mlx5_stats_reset(struct rte_eth_dev *dev) unsigned int i; unsigned int idx; - priv_lock(priv); for (i = 0; (i != priv->rxqs_n); ++i) { if ((*priv->rxqs)[i] == NULL) continue; @@ -405,45 +438,6 @@ mlx5_stats_reset(struct rte_eth_dev *dev) #ifndef MLX5_PMD_SOFT_COUNTERS /* FIXME: reset hardware counters. */ #endif - priv_unlock(priv); -} - -/** - * DPDK callback to get extended device statistics. - * - * @param dev - * Pointer to Ethernet device structure. - * @param[out] stats - * Stats table output buffer. - * @param n - * The size of the stats table. - * - * @return - * Number of xstats on success, negative on failure. - */ -int -mlx5_xstats_get(struct rte_eth_dev *dev, - struct rte_eth_xstat *stats, unsigned int n) -{ - struct priv *priv = mlx5_get_priv(dev); - int ret = xstats_n; - - if (n >= xstats_n && stats) { - struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; - int stats_n; - - priv_lock(priv); - stats_n = priv_ethtool_get_stats_n(priv); - if (stats_n < 0) { - priv_unlock(priv); - return -1; - } - if (xstats_ctrl->stats_n != stats_n) - priv_xstats_init(priv); - ret = priv_xstats_get(priv, stats); - priv_unlock(priv); - } - return ret; } /** @@ -455,19 +449,30 @@ mlx5_xstats_get(struct rte_eth_dev *dev, void mlx5_xstats_reset(struct rte_eth_dev *dev) { - struct priv *priv = mlx5_get_priv(dev); + struct priv *priv = dev->data->dev_private; struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; int stats_n; + unsigned int i; + unsigned int n = xstats_n; + uint64_t counters[n]; + int ret; - priv_lock(priv); - stats_n = priv_ethtool_get_stats_n(priv); - if (stats_n < 0) - goto unlock; + stats_n = mlx5_ethtool_get_stats_n(dev); + if (stats_n < 0) { + DRV_LOG(ERR, "port %u cannot get stats: %s", dev->data->port_id, + strerror(-stats_n)); + return; + } if (xstats_ctrl->stats_n != stats_n) - priv_xstats_init(priv); - priv_xstats_reset(priv); -unlock: - priv_unlock(priv); + mlx5_xstats_init(dev); + ret = mlx5_read_dev_counters(dev, counters); + if (ret) { + DRV_LOG(ERR, "port %u cannot read device counters: %s", + dev->data->port_id, strerror(rte_errno)); + return; + } + for (i = 0; i != n; ++i) + xstats_ctrl->base[i] = counters[i]; } /** @@ -484,21 +489,18 @@ unlock: * Number of xstats names. */ int -mlx5_xstats_get_names(struct rte_eth_dev *dev, - struct rte_eth_xstat_name *xstats_names, unsigned int n) +mlx5_xstats_get_names(struct rte_eth_dev *dev __rte_unused, + struct rte_eth_xstat_name *xstats_names, unsigned int n) { - struct priv *priv = mlx5_get_priv(dev); unsigned int i; if (n >= xstats_n && xstats_names) { - priv_lock(priv); for (i = 0; i != xstats_n; ++i) { strncpy(xstats_names[i].name, mlx5_counters_init[i].dpdk_name, RTE_ETH_XSTATS_NAME_SIZE); xstats_names[i].name[RTE_ETH_XSTATS_NAME_SIZE - 1] = 0; } - priv_unlock(priv); } return xstats_n; } diff --git a/dpdk/drivers/net/mlx5/mlx5_trigger.c b/dpdk/drivers/net/mlx5/mlx5_trigger.c index d682ea2c..9a1d6f95 100644 --- a/dpdk/drivers/net/mlx5/mlx5_trigger.c +++ b/dpdk/drivers/net/mlx5/mlx5_trigger.c @@ -41,80 +41,125 @@ #include "mlx5_rxtx.h" #include "mlx5_utils.h" +/** + * Stop traffic on Tx queues. + * + * @param dev + * Pointer to Ethernet device structure. + */ static void -priv_txq_stop(struct priv *priv) +mlx5_txq_stop(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; unsigned int i; for (i = 0; i != priv->txqs_n; ++i) - mlx5_priv_txq_release(priv, i); + mlx5_txq_release(dev, i); } +/** + * Start traffic on Tx queues. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ static int -priv_txq_start(struct priv *priv) +mlx5_txq_start(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; unsigned int i; - int ret = 0; + int ret; - /* Add memory regions to Tx queues. */ for (i = 0; i != priv->txqs_n; ++i) { unsigned int idx = 0; struct mlx5_mr *mr; - struct mlx5_txq_ctrl *txq_ctrl = mlx5_priv_txq_get(priv, i); + struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i); if (!txq_ctrl) continue; LIST_FOREACH(mr, &priv->mr, next) { - priv_txq_mp2mr_reg(priv, &txq_ctrl->txq, mr->mp, idx++); + mlx5_txq_mp2mr_reg(&txq_ctrl->txq, mr->mp, idx++); if (idx == MLX5_PMD_TX_MP_CACHE) break; } txq_alloc_elts(txq_ctrl); - txq_ctrl->ibv = mlx5_priv_txq_ibv_new(priv, i); + txq_ctrl->ibv = mlx5_txq_ibv_new(dev, i); if (!txq_ctrl->ibv) { - ret = ENOMEM; + rte_errno = ENOMEM; goto error; } } - return -ret; + ret = mlx5_tx_uar_remap(dev, priv->ctx->cmd_fd); + if (ret) { + /* Adjust index for rollback. */ + i = priv->txqs_n - 1; + goto error; + } + return 0; error: - priv_txq_stop(priv); - return -ret; + ret = rte_errno; /* Save rte_errno before cleanup. */ + do { + mlx5_txq_release(dev, i); + } while (i-- != 0); + rte_errno = ret; /* Restore rte_errno. */ + return -rte_errno; } +/** + * Stop traffic on Rx queues. + * + * @param dev + * Pointer to Ethernet device structure. + */ static void -priv_rxq_stop(struct priv *priv) +mlx5_rxq_stop(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; unsigned int i; for (i = 0; i != priv->rxqs_n; ++i) - mlx5_priv_rxq_release(priv, i); + mlx5_rxq_release(dev, i); } +/** + * Start traffic on Rx queues. + * + * @param dev + * Pointer to Ethernet device structure. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ static int -priv_rxq_start(struct priv *priv) +mlx5_rxq_start(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; unsigned int i; int ret = 0; for (i = 0; i != priv->rxqs_n; ++i) { - struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_priv_rxq_get(priv, i); + struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i); if (!rxq_ctrl) continue; ret = rxq_alloc_elts(rxq_ctrl); if (ret) goto error; - rxq_ctrl->ibv = mlx5_priv_rxq_ibv_new(priv, i); - if (!rxq_ctrl->ibv) { - ret = ENOMEM; + rxq_ctrl->ibv = mlx5_rxq_ibv_new(dev, i); + if (!rxq_ctrl->ibv) goto error; - } } - return -ret; + return 0; error: - priv_rxq_stop(priv); - return -ret; + ret = rte_errno; /* Save rte_errno before cleanup. */ + do { + mlx5_rxq_release(dev, i); + } while (i-- != 0); + rte_errno = ret; /* Restore rte_errno. */ + return -rte_errno; } /** @@ -126,68 +171,73 @@ error: * Pointer to Ethernet device structure. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_dev_start(struct rte_eth_dev *dev) { struct priv *priv = dev->data->dev_private; struct mlx5_mr *mr = NULL; - int err; + int ret; - dev->data->dev_started = 1; - priv_lock(priv); - err = priv_flow_create_drop_queue(priv); - if (err) { - ERROR("%p: Drop queue allocation failed: %s", - (void *)dev, strerror(err)); + DRV_LOG(DEBUG, "port %u starting device", dev->data->port_id); + ret = mlx5_flow_create_drop_queue(dev); + if (ret) { + DRV_LOG(ERR, "port %u drop queue allocation failed: %s", + dev->data->port_id, strerror(rte_errno)); goto error; } - DEBUG("%p: allocating and configuring hash RX queues", (void *)dev); rte_mempool_walk(mlx5_mp2mr_iter, priv); - err = priv_txq_start(priv); - if (err) { - ERROR("%p: TXQ allocation failed: %s", - (void *)dev, strerror(err)); + ret = mlx5_txq_start(dev); + if (ret) { + DRV_LOG(ERR, "port %u Tx queue allocation failed: %s", + dev->data->port_id, strerror(rte_errno)); + return -rte_errno; + } + ret = mlx5_rxq_start(dev); + if (ret) { + DRV_LOG(ERR, "port %u Rx queue allocation failed: %s", + dev->data->port_id, strerror(rte_errno)); + mlx5_txq_stop(dev); + return -rte_errno; + } + dev->data->dev_started = 1; + ret = mlx5_rx_intr_vec_enable(dev); + if (ret) { + DRV_LOG(ERR, "port %u Rx interrupt vector creation failed", + dev->data->port_id); goto error; } - err = priv_rxq_start(priv); - if (err) { - ERROR("%p: RXQ allocation failed: %s", - (void *)dev, strerror(err)); + mlx5_xstats_init(dev); + ret = mlx5_traffic_enable(dev); + if (ret) { + DRV_LOG(DEBUG, "port %u failed to set defaults flows", + dev->data->port_id); goto error; } - err = priv_rx_intr_vec_enable(priv); - if (err) { - ERROR("%p: RX interrupt vector creation failed", - (void *)priv); + ret = mlx5_flow_start(dev, &priv->flows); + if (ret) { + DRV_LOG(DEBUG, "port %u failed to set flows", + dev->data->port_id); goto error; } - priv_xstats_init(priv); - /* Update link status and Tx/Rx callbacks for the first time. */ - memset(&dev->data->dev_link, 0, sizeof(struct rte_eth_link)); - INFO("Forcing port %u link to be up", dev->data->port_id); - err = priv_force_link_status_change(priv, ETH_LINK_UP); - if (err) { - DEBUG("Failed to set port %u link to be up", - dev->data->port_id); - goto error; - } - priv_dev_interrupt_handler_install(priv, dev); - priv_unlock(priv); + dev->tx_pkt_burst = mlx5_select_tx_function(dev); + dev->rx_pkt_burst = mlx5_select_rx_function(dev); + mlx5_dev_interrupt_handler_install(dev); return 0; error: + ret = rte_errno; /* Save rte_errno before cleanup. */ /* Rollback. */ dev->data->dev_started = 0; for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr)) - priv_mr_release(priv, mr); - priv_flow_stop(priv, &priv->flows); - priv_dev_traffic_disable(priv, dev); - priv_txq_stop(priv); - priv_rxq_stop(priv); - priv_flow_delete_drop_queue(priv); - priv_unlock(priv); - return err; + mlx5_mr_release(mr); + mlx5_flow_stop(dev, &priv->flows); + mlx5_traffic_disable(dev); + mlx5_txq_stop(dev); + mlx5_rxq_stop(dev); + mlx5_flow_delete_drop_queue(dev); + rte_errno = ret; /* Restore rte_errno. */ + return -rte_errno; } /** @@ -204,40 +254,39 @@ mlx5_dev_stop(struct rte_eth_dev *dev) struct priv *priv = dev->data->dev_private; struct mlx5_mr *mr; - priv_lock(priv); dev->data->dev_started = 0; /* Prevent crashes when queues are still in use. */ dev->rx_pkt_burst = removed_rx_burst; dev->tx_pkt_burst = removed_tx_burst; rte_wmb(); usleep(1000 * priv->rxqs_n); - DEBUG("%p: cleaning up and destroying hash RX queues", (void *)dev); - priv_flow_stop(priv, &priv->flows); - priv_dev_traffic_disable(priv, dev); - priv_rx_intr_vec_disable(priv); - priv_dev_interrupt_handler_uninstall(priv, dev); - priv_txq_stop(priv); - priv_rxq_stop(priv); + DRV_LOG(DEBUG, "port %u stopping device", dev->data->port_id); + mlx5_flow_stop(dev, &priv->flows); + mlx5_traffic_disable(dev); + mlx5_rx_intr_vec_disable(dev); + mlx5_dev_interrupt_handler_uninstall(dev); + mlx5_txq_stop(dev); + mlx5_rxq_stop(dev); for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr)) - priv_mr_release(priv, mr); - priv_flow_delete_drop_queue(priv); - priv_unlock(priv); + mlx5_mr_release(mr); + mlx5_flow_delete_drop_queue(dev); } /** * Enable traffic flows configured by control plane * - * @param priv + * @param dev * Pointer to Ethernet device private data. * @param dev * Pointer to Ethernet device structure. * * @return - * 0 on success. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev) +mlx5_traffic_enable(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct rte_flow_item_eth bcast = { .dst.addr_bytes = "\xff\xff\xff\xff\xff\xff", }; @@ -270,8 +319,9 @@ priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev) .type = 0, }; - claim_zero(mlx5_ctrl_flow(dev, &promisc, &promisc)); - return 0; + ret = mlx5_ctrl_flow(dev, &promisc, &promisc); + if (ret) + goto error; } if (dev->data->all_multicast) { struct rte_flow_item_eth multicast = { @@ -280,7 +330,9 @@ priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev) .type = 0, }; - claim_zero(mlx5_ctrl_flow(dev, &multicast, &multicast)); + ret = mlx5_ctrl_flow(dev, &multicast, &multicast); + if (ret) + goto error; } else { /* Add broadcast/multicast flows. */ for (i = 0; i != vlan_filter_n; ++i) { @@ -289,9 +341,8 @@ priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev) struct rte_flow_item_vlan vlan_spec = { .tci = rte_cpu_to_be_16(vlan), }; - struct rte_flow_item_vlan vlan_mask = { - .tci = 0xffff, - }; + struct rte_flow_item_vlan vlan_mask = + rte_flow_item_vlan_mask; ret = mlx5_ctrl_flow_vlan(dev, &bcast, &bcast, &vlan_spec, &vlan_mask); @@ -328,9 +379,8 @@ priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev) struct rte_flow_item_vlan vlan_spec = { .tci = rte_cpu_to_be_16(vlan), }; - struct rte_flow_item_vlan vlan_mask = { - .tci = 0xffff, - }; + struct rte_flow_item_vlan vlan_mask = + rte_flow_item_vlan_mask; ret = mlx5_ctrl_flow_vlan(dev, &unicast, &unicast_mask, @@ -340,74 +390,49 @@ priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev) goto error; } if (!vlan_filter_n) { - ret = mlx5_ctrl_flow(dev, &unicast, - &unicast_mask); + ret = mlx5_ctrl_flow(dev, &unicast, &unicast_mask); if (ret) goto error; } } return 0; error: - return rte_errno; + ret = rte_errno; /* Save rte_errno before cleanup. */ + mlx5_flow_list_flush(dev, &priv->ctrl_flows); + rte_errno = ret; /* Restore rte_errno. */ + return -rte_errno; } /** * Disable traffic flows configured by control plane * - * @param priv - * Pointer to Ethernet device private data. * @param dev - * Pointer to Ethernet device structure. - * - * @return - * 0 on success. + * Pointer to Ethernet device private data. */ -int -priv_dev_traffic_disable(struct priv *priv, struct rte_eth_dev *dev) +void +mlx5_traffic_disable(struct rte_eth_dev *dev) { - (void)dev; - priv_flow_flush(priv, &priv->ctrl_flows); - return 0; -} + struct priv *priv = dev->data->dev_private; -/** - * Restart traffic flows configured by control plane - * - * @param priv - * Pointer to Ethernet device private data. - * @param dev - * Pointer to Ethernet device structure. - * - * @return - * 0 on success. - */ -int -priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev) -{ - if (dev->data->dev_started) { - priv_dev_traffic_disable(priv, dev); - priv_dev_traffic_enable(priv, dev); - } - return 0; + mlx5_flow_list_flush(dev, &priv->ctrl_flows); } /** * Restart traffic flows configured by control plane * * @param dev - * Pointer to Ethernet device structure. + * Pointer to Ethernet device private data. * * @return - * 0 on success. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_traffic_restart(struct rte_eth_dev *dev) { - struct priv *priv = dev->data->dev_private; - - priv_lock(priv); - priv_dev_traffic_restart(priv, dev); - priv_unlock(priv); + if (dev->data->dev_started) { + mlx5_traffic_disable(dev); + return mlx5_traffic_enable(dev); + } return 0; } diff --git a/dpdk/drivers/net/mlx5/mlx5_txq.c b/dpdk/drivers/net/mlx5/mlx5_txq.c index 7ca99f5a..760ac92d 100644 --- a/dpdk/drivers/net/mlx5/mlx5_txq.c +++ b/dpdk/drivers/net/mlx5/mlx5_txq.c @@ -74,7 +74,8 @@ txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl) for (i = 0; (i != elts_n); ++i) (*txq_ctrl->txq.elts)[i] = NULL; - DEBUG("%p: allocated and configured %u WRs", (void *)txq_ctrl, elts_n); + DRV_LOG(DEBUG, "port %u Tx queue %u allocated and configured %u WRs", + PORT_ID(txq_ctrl->priv), txq_ctrl->idx, elts_n); txq_ctrl->txq.elts_head = 0; txq_ctrl->txq.elts_tail = 0; txq_ctrl->txq.elts_comp = 0; @@ -95,7 +96,8 @@ txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl) uint16_t elts_tail = txq_ctrl->txq.elts_tail; struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts; - DEBUG("%p: freeing WRs", (void *)txq_ctrl); + DRV_LOG(DEBUG, "port %u Tx queue %u freeing WRs", + PORT_ID(txq_ctrl->priv), txq_ctrl->idx); txq_ctrl->txq.elts_head = 0; txq_ctrl->txq.elts_tail = 0; txq_ctrl->txq.elts_comp = 0; @@ -130,7 +132,7 @@ txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl) * Thresholds parameters. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, @@ -140,50 +142,47 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, struct mlx5_txq_data *txq = (*priv->txqs)[idx]; struct mlx5_txq_ctrl *txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); - int ret = 0; - priv_lock(priv); if (desc <= MLX5_TX_COMP_THRESH) { - WARN("%p: number of descriptors requested for TX queue %u" - " must be higher than MLX5_TX_COMP_THRESH, using" - " %u instead of %u", - (void *)dev, idx, MLX5_TX_COMP_THRESH + 1, desc); + DRV_LOG(WARNING, + "port %u number of descriptors requested for Tx queue" + " %u must be higher than MLX5_TX_COMP_THRESH, using %u" + " instead of %u", + dev->data->port_id, idx, MLX5_TX_COMP_THRESH + 1, desc); desc = MLX5_TX_COMP_THRESH + 1; } if (!rte_is_power_of_2(desc)) { desc = 1 << log2above(desc); - WARN("%p: increased number of descriptors in TX queue %u" - " to the next power of two (%d)", - (void *)dev, idx, desc); + DRV_LOG(WARNING, + "port %u increased number of descriptors in Tx queue" + " %u to the next power of two (%d)", + dev->data->port_id, idx, desc); } - DEBUG("%p: configuring queue %u for %u descriptors", - (void *)dev, idx, desc); + DRV_LOG(DEBUG, "port %u configuring queue %u for %u descriptors", + dev->data->port_id, idx, desc); if (idx >= priv->txqs_n) { - ERROR("%p: queue index out of range (%u >= %u)", - (void *)dev, idx, priv->txqs_n); - priv_unlock(priv); - return -EOVERFLOW; + DRV_LOG(ERR, "port %u Tx queue index out of range (%u >= %u)", + dev->data->port_id, idx, priv->txqs_n); + rte_errno = EOVERFLOW; + return -rte_errno; } - if (!mlx5_priv_txq_releasable(priv, idx)) { - ret = EBUSY; - ERROR("%p: unable to release queue index %u", - (void *)dev, idx); - goto out; + if (!mlx5_txq_releasable(dev, idx)) { + rte_errno = EBUSY; + DRV_LOG(ERR, "port %u unable to release queue index %u", + dev->data->port_id, idx); + return -rte_errno; } - mlx5_priv_txq_release(priv, idx); - txq_ctrl = mlx5_priv_txq_new(priv, idx, desc, socket, conf); + mlx5_txq_release(dev, idx); + txq_ctrl = mlx5_txq_new(dev, idx, desc, socket, conf); if (!txq_ctrl) { - ERROR("%p: unable to allocate queue index %u", - (void *)dev, idx); - ret = ENOMEM; - goto out; + DRV_LOG(ERR, "port %u unable to allocate queue index %u", + dev->data->port_id, idx); + return -rte_errno; } - DEBUG("%p: adding TX queue %p to list", - (void *)dev, (void *)txq_ctrl); + DRV_LOG(DEBUG, "port %u adding Tx queue %u to list", + dev->data->port_id, idx); (*priv->txqs)[idx] = &txq_ctrl->txq; -out: - priv_unlock(priv); - return -ret; + return 0; } /** @@ -204,37 +203,40 @@ mlx5_tx_queue_release(void *dpdk_txq) return; txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); priv = txq_ctrl->priv; - priv_lock(priv); for (i = 0; (i != priv->txqs_n); ++i) if ((*priv->txqs)[i] == txq) { - DEBUG("%p: removing TX queue %p from list", - (void *)priv->dev, (void *)txq_ctrl); - mlx5_priv_txq_release(priv, i); + mlx5_txq_release(ETH_DEV(priv), i); + DRV_LOG(DEBUG, "port %u removing Tx queue %u from list", + PORT_ID(priv), txq_ctrl->idx); break; } - priv_unlock(priv); } /** - * Map locally UAR used in Tx queues for BlueFlame doorbell. + * Mmap TX UAR(HW doorbell) pages into reserved UAR address space. + * Both primary and secondary process do mmap to make UAR address + * aligned. * - * @param[in] priv - * Pointer to private structure. + * @param[in] dev + * Pointer to Ethernet device. * @param fd * Verbs file descriptor to map UAR pages. * * @return - * 0 on success, errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int -priv_tx_uar_remap(struct priv *priv, int fd) +mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd) { + struct priv *priv = dev->data->dev_private; unsigned int i, j; uintptr_t pages[priv->txqs_n]; unsigned int pages_n = 0; uintptr_t uar_va; + uintptr_t off; void *addr; + void *ret; struct mlx5_txq_data *txq; struct mlx5_txq_ctrl *txq_ctrl; int already_mapped; @@ -251,8 +253,11 @@ priv_tx_uar_remap(struct priv *priv, int fd) continue; txq = (*priv->txqs)[i]; txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq); - uar_va = (uintptr_t)txq_ctrl->txq.bf_reg; - uar_va = RTE_ALIGN_FLOOR(uar_va, page_size); + assert(txq_ctrl->idx == (uint16_t)i); + /* UAR addr form verbs used to find dup and offset in page. */ + uar_va = (uintptr_t)txq_ctrl->bf_reg_orig; + off = uar_va & (page_size - 1); /* offset in page. */ + uar_va = RTE_ALIGN_FLOOR(uar_va, page_size); /* page addr. */ already_mapped = 0; for (j = 0; j != pages_n; ++j) { if (pages[j] == uar_va) { @@ -260,16 +265,32 @@ priv_tx_uar_remap(struct priv *priv, int fd) break; } } - if (already_mapped) - continue; - pages[pages_n++] = uar_va; - addr = mmap((void *)uar_va, page_size, - PROT_WRITE, MAP_FIXED | MAP_SHARED, fd, - txq_ctrl->uar_mmap_offset); - if (addr != (void *)uar_va) { - ERROR("call to mmap failed on UAR for txq %d\n", i); - return -1; + /* new address in reserved UAR address space. */ + addr = RTE_PTR_ADD(priv->uar_base, + uar_va & (MLX5_UAR_SIZE - 1)); + if (!already_mapped) { + pages[pages_n++] = uar_va; + /* fixed mmap to specified address in reserved + * address space. + */ + ret = mmap(addr, page_size, + PROT_WRITE, MAP_FIXED | MAP_SHARED, fd, + txq_ctrl->uar_mmap_offset); + if (ret != addr) { + /* fixed mmap have to return same address */ + DRV_LOG(ERR, + "port %u call to mmap failed on UAR" + " for txq %u", + dev->data->port_id, txq_ctrl->idx); + rte_errno = ENXIO; + return -rte_errno; + } } + if (rte_eal_process_type() == RTE_PROC_PRIMARY) /* save once */ + txq_ctrl->txq.bf_reg = RTE_PTR_ADD((void *)addr, off); + else + assert(txq_ctrl->txq.bf_reg == + RTE_PTR_ADD((void *)addr, off)); } return 0; } @@ -277,17 +298,18 @@ priv_tx_uar_remap(struct priv *priv, int fd) /** * Create the Tx queue Verbs object. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * Queue index in DPDK Rx queue array * * @return - * The Verbs object initialised if it can be created. + * The Verbs object initialised, NULL otherwise and rte_errno is set. */ -struct mlx5_txq_ibv* -mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) +struct mlx5_txq_ibv * +mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_txq_data *txq_data = (*priv->txqs)[idx]; struct mlx5_txq_ctrl *txq_ctrl = container_of(txq_data, struct mlx5_txq_ctrl, txq); @@ -307,9 +329,14 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) int ret = 0; assert(txq_data); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_TX_QUEUE; + priv->verbs_alloc_ctx.obj = txq_ctrl; if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) { - ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set"); - goto error; + DRV_LOG(ERR, + "port %u MLX5_ENABLE_CQE_COMPRESSION must never be set", + dev->data->port_id); + rte_errno = EINVAL; + return NULL; } memset(&tmpl, 0, sizeof(struct mlx5_txq_ibv)); /* MRs will be registered in mp2mr[] later. */ @@ -322,7 +349,9 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) cqe_n += MLX5_TX_COMP_THRESH_INLINE_DIV; tmpl.cq = ibv_create_cq(priv->ctx, cqe_n, NULL, NULL, 0); if (tmpl.cq == NULL) { - ERROR("%p: CQ creation failure", (void *)txq_ctrl); + DRV_LOG(ERR, "port %u Tx queue %u CQ creation failure", + dev->data->port_id, idx); + rte_errno = errno; goto error; } attr.init = (struct ibv_qp_init_attr_ex){ @@ -363,7 +392,9 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) } tmpl.qp = ibv_create_qp_ex(priv->ctx, &attr.init); if (tmpl.qp == NULL) { - ERROR("%p: QP creation failure", (void *)txq_ctrl); + DRV_LOG(ERR, "port %u Tx queue %u QP creation failure", + dev->data->port_id, idx); + rte_errno = errno; goto error; } attr.mod = (struct ibv_qp_attr){ @@ -374,7 +405,10 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) }; ret = ibv_modify_qp(tmpl.qp, &attr.mod, (IBV_QP_STATE | IBV_QP_PORT)); if (ret) { - ERROR("%p: QP state to IBV_QPS_INIT failed", (void *)txq_ctrl); + DRV_LOG(ERR, + "port %u Tx queue %u QP state to IBV_QPS_INIT failed", + dev->data->port_id, idx); + rte_errno = errno; goto error; } attr.mod = (struct ibv_qp_attr){ @@ -382,19 +416,27 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) }; ret = ibv_modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE); if (ret) { - ERROR("%p: QP state to IBV_QPS_RTR failed", (void *)txq_ctrl); + DRV_LOG(ERR, + "port %u Tx queue %u QP state to IBV_QPS_RTR failed", + dev->data->port_id, idx); + rte_errno = errno; goto error; } attr.mod.qp_state = IBV_QPS_RTS; ret = ibv_modify_qp(tmpl.qp, &attr.mod, IBV_QP_STATE); if (ret) { - ERROR("%p: QP state to IBV_QPS_RTS failed", (void *)txq_ctrl); + DRV_LOG(ERR, + "port %u Tx queue %u QP state to IBV_QPS_RTS failed", + dev->data->port_id, idx); + rte_errno = errno; goto error; } txq_ibv = rte_calloc_socket(__func__, 1, sizeof(struct mlx5_txq_ibv), 0, txq_ctrl->socket); if (!txq_ibv) { - ERROR("%p: cannot allocate memory", (void *)txq_ctrl); + DRV_LOG(ERR, "port %u Tx queue %u cannot allocate memory", + dev->data->port_id, idx); + rte_errno = ENOMEM; goto error; } obj.cq.in = tmpl.cq; @@ -402,11 +444,16 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) obj.qp.in = tmpl.qp; obj.qp.out = &qp; ret = mlx5dv_init_obj(&obj, MLX5DV_OBJ_CQ | MLX5DV_OBJ_QP); - if (ret != 0) + if (ret != 0) { + rte_errno = errno; goto error; + } if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) { - ERROR("Wrong MLX5_CQE_SIZE environment variable value: " - "it should be set to %u", RTE_CACHE_LINE_SIZE); + DRV_LOG(ERR, + "port %u wrong MLX5_CQE_SIZE environment variable" + " value: it should be set to %u", + dev->data->port_id, RTE_CACHE_LINE_SIZE); + rte_errno = EINVAL; goto error; } txq_data->cqe_n = log2above(cq_info.cqe_cnt); @@ -414,13 +461,15 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) txq_data->wqes = qp.sq.buf; txq_data->wqe_n = log2above(qp.sq.wqe_cnt); txq_data->qp_db = &qp.dbrec[MLX5_SND_DBR]; - txq_data->bf_reg = qp.bf.reg; + txq_ctrl->bf_reg_orig = qp.bf.reg; txq_data->cq_db = cq_info.dbrec; txq_data->cqes = (volatile struct mlx5_cqe (*)[]) (uintptr_t)cq_info.buf; txq_data->cq_ci = 0; +#ifndef NDEBUG txq_data->cq_pi = 0; +#endif txq_data->wqe_ci = 0; txq_data->wqe_pi = 0; txq_ibv->qp = tmpl.qp; @@ -429,35 +478,45 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx) if (qp.comp_mask & MLX5DV_QP_MASK_UAR_MMAP_OFFSET) { txq_ctrl->uar_mmap_offset = qp.uar_mmap_offset; } else { - ERROR("Failed to retrieve UAR info, invalid libmlx5.so version"); + DRV_LOG(ERR, + "port %u failed to retrieve UAR info, invalid" + " libmlx5.so", + dev->data->port_id); + rte_errno = EINVAL; goto error; } - DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, - (void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt)); + DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d", + dev->data->port_id, idx, rte_atomic32_read(&txq_ibv->refcnt)); LIST_INSERT_HEAD(&priv->txqsibv, txq_ibv, next); + txq_ibv->txq_ctrl = txq_ctrl; + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; return txq_ibv; error: + ret = rte_errno; /* Save rte_errno before cleanup. */ if (tmpl.cq) claim_zero(ibv_destroy_cq(tmpl.cq)); if (tmpl.qp) claim_zero(ibv_destroy_qp(tmpl.qp)); + priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE; + rte_errno = ret; /* Restore rte_errno. */ return NULL; } /** * Get an Tx queue Verbs object. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * Queue index in DPDK Rx queue array * * @return * The Verbs object if it exists. */ -struct mlx5_txq_ibv* -mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx) +struct mlx5_txq_ibv * +mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *txq_ctrl; if (idx >= priv->txqs_n) @@ -467,8 +526,8 @@ mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx) txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); if (txq_ctrl->ibv) { rte_atomic32_inc(&txq_ctrl->ibv->refcnt); - DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, - (void *)txq_ctrl->ibv, + DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d", + dev->data->port_id, txq_ctrl->idx, rte_atomic32_read(&txq_ctrl->ibv->refcnt)); } return txq_ctrl->ibv; @@ -477,21 +536,19 @@ mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx) /** * Release an Tx verbs queue object. * - * @param priv - * Pointer to private structure. * @param txq_ibv * Verbs Tx queue object. * * @return - * 0 on success, errno on failure. + * 1 while a reference on it exists, 0 when freed. */ int -mlx5_priv_txq_ibv_release(struct priv *priv, struct mlx5_txq_ibv *txq_ibv) +mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv) { - (void)priv; assert(txq_ibv); - DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv, - (void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt)); + DRV_LOG(DEBUG, "port %u Verbs Tx queue %u: refcnt %d", + PORT_ID(txq_ibv->txq_ctrl->priv), + txq_ibv->txq_ctrl->idx, rte_atomic32_read(&txq_ibv->refcnt)); if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) { claim_zero(ibv_destroy_qp(txq_ibv->qp)); claim_zero(ibv_destroy_cq(txq_ibv->cq)); @@ -499,21 +556,18 @@ mlx5_priv_txq_ibv_release(struct priv *priv, struct mlx5_txq_ibv *txq_ibv) rte_free(txq_ibv); return 0; } - return EBUSY; + return 1; } /** * Return true if a single reference exists on the object. * - * @param priv - * Pointer to private structure. * @param txq_ibv * Verbs Tx queue object. */ int -mlx5_priv_txq_ibv_releasable(struct priv *priv, struct mlx5_txq_ibv *txq_ibv) +mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv) { - (void)priv; assert(txq_ibv); return (rte_atomic32_read(&txq_ibv->refcnt) == 1); } @@ -521,20 +575,22 @@ mlx5_priv_txq_ibv_releasable(struct priv *priv, struct mlx5_txq_ibv *txq_ibv) /** * Verify the Verbs Tx queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * - * @return the number of object not released. + * @return + * The number of object not released. */ int -mlx5_priv_txq_ibv_verify(struct priv *priv) +mlx5_txq_ibv_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; int ret = 0; struct mlx5_txq_ibv *txq_ibv; LIST_FOREACH(txq_ibv, &priv->txqsibv, next) { - DEBUG("%p: Verbs Tx queue %p still referenced", (void *)priv, - (void *)txq_ibv); + DRV_LOG(DEBUG, "port %u Verbs Tx queue %u still referenced", + dev->data->port_id, txq_ibv->txq_ctrl->idx); ++ret; } return ret; @@ -543,8 +599,8 @@ mlx5_priv_txq_ibv_verify(struct priv *priv) /** * Create a DPDK Tx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * @param desc @@ -555,13 +611,13 @@ mlx5_priv_txq_ibv_verify(struct priv *priv) * Thresholds parameters. * * @return - * A DPDK queue object on success. + * A DPDK queue object on success, NULL otherwise and rte_errno is set. */ -struct mlx5_txq_ctrl* -mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, - unsigned int socket, - const struct rte_eth_txconf *conf) +struct mlx5_txq_ctrl * +mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, + unsigned int socket, const struct rte_eth_txconf *conf) { + struct priv *priv = dev->data->dev_private; const unsigned int max_tso_inline = ((MLX5_MAX_TSO_HEADER + (RTE_CACHE_LINE_SIZE - 1)) / RTE_CACHE_LINE_SIZE); @@ -571,20 +627,23 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, sizeof(*tmpl) + desc * sizeof(struct rte_mbuf *), 0, socket); - if (!tmpl) + if (!tmpl) { + rte_errno = ENOMEM; return NULL; + } assert(desc > MLX5_TX_COMP_THRESH); tmpl->txq.flags = conf->txq_flags; tmpl->priv = priv; tmpl->socket = socket; tmpl->txq.elts_n = log2above(desc); + tmpl->idx = idx; if (priv->mps == MLX5_MPW_ENHANCED) tmpl->txq.mpw_hdr_dseg = priv->mpw_hdr_dseg; /* MRs will be registered in mp2mr[] later. */ - DEBUG("priv->device_attr.max_qp_wr is %d", - priv->device_attr.orig_attr.max_qp_wr); - DEBUG("priv->device_attr.max_sge is %d", - priv->device_attr.orig_attr.max_sge); + DRV_LOG(DEBUG, "port %u priv->device_attr.max_qp_wr is %d", + dev->data->port_id, priv->device_attr.orig_attr.max_qp_wr); + DRV_LOG(DEBUG, "port %u priv->device_attr.max_sge is %d", + dev->data->port_id, priv->device_attr.orig_attr.max_sge); if (priv->txq_inline && (priv->txqs_n >= priv->txqs_inline)) { unsigned int ds_cnt; @@ -605,17 +664,6 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, priv->inline_max_packet_sz) + (RTE_CACHE_LINE_SIZE - 1)) / RTE_CACHE_LINE_SIZE) * RTE_CACHE_LINE_SIZE; - } else if (priv->tso) { - int inline_diff = tmpl->txq.max_inline - max_tso_inline; - - /* - * Adjust inline value as Verbs aggregates - * tso_inline and txq_inline fields. - */ - tmpl->max_inline_data = inline_diff > 0 ? - inline_diff * - RTE_CACHE_LINE_SIZE : - 0; } else { tmpl->max_inline_data = tmpl->txq.max_inline * RTE_CACHE_LINE_SIZE; @@ -635,9 +683,10 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, max_inline = max_inline - (max_inline % RTE_CACHE_LINE_SIZE); - WARN("txq inline is too large (%d) setting it to " - "the maximum possible: %d\n", - priv->txq_inline, max_inline); + DRV_LOG(WARNING, + "port %u txq inline is too large (%d) setting it" + " to the maximum possible: %d\n", + PORT_ID(priv), priv->txq_inline, max_inline); tmpl->txq.max_inline = max_inline / RTE_CACHE_LINE_SIZE; } } @@ -653,8 +702,8 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, (struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])(tmpl + 1); tmpl->txq.stats.idx = idx; rte_atomic32_inc(&tmpl->refcnt); - DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, - (void *)tmpl, rte_atomic32_read(&tmpl->refcnt)); + DRV_LOG(DEBUG, "port %u Tx queue %u: refcnt %d", dev->data->port_id, + idx, rte_atomic32_read(&tmpl->refcnt)); LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next); return tmpl; } @@ -662,17 +711,18 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc, /** * Get a Tx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * * @return * A pointer to the queue if it exists. */ -struct mlx5_txq_ctrl* -mlx5_priv_txq_get(struct priv *priv, uint16_t idx) +struct mlx5_txq_ctrl * +mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *ctrl = NULL; if ((*priv->txqs)[idx]) { @@ -680,19 +730,17 @@ mlx5_priv_txq_get(struct priv *priv, uint16_t idx) txq); unsigned int i; - mlx5_priv_txq_ibv_get(priv, idx); + mlx5_txq_ibv_get(dev, idx); for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) { - struct mlx5_mr *mr = NULL; - - (void)mr; - if (ctrl->txq.mp2mr[i]) { - mr = priv_mr_get(priv, ctrl->txq.mp2mr[i]->mp); - assert(mr); - } + if (ctrl->txq.mp2mr[i]) + claim_nonzero + (mlx5_mr_get(dev, + ctrl->txq.mp2mr[i]->mp)); } rte_atomic32_inc(&ctrl->refcnt); - DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, - (void *)ctrl, rte_atomic32_read(&ctrl->refcnt)); + DRV_LOG(DEBUG, "port %u Tx queue %u refcnt %d", + dev->data->port_id, + ctrl->idx, rte_atomic32_read(&ctrl->refcnt)); } return ctrl; } @@ -700,38 +748,38 @@ mlx5_priv_txq_get(struct priv *priv, uint16_t idx) /** * Release a Tx queue. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * * @return - * 0 on success, errno on failure. + * 1 while a reference on it exists, 0 when freed. */ int -mlx5_priv_txq_release(struct priv *priv, uint16_t idx) +mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; unsigned int i; struct mlx5_txq_ctrl *txq; + size_t page_size = sysconf(_SC_PAGESIZE); if (!(*priv->txqs)[idx]) return 0; txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq); - DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv, - (void *)txq, rte_atomic32_read(&txq->refcnt)); - if (txq->ibv) { - int ret; - - ret = mlx5_priv_txq_ibv_release(priv, txq->ibv); - if (!ret) - txq->ibv = NULL; - } + DRV_LOG(DEBUG, "port %u Tx queue %u: refcnt %d", dev->data->port_id, + txq->idx, rte_atomic32_read(&txq->refcnt)); + if (txq->ibv && !mlx5_txq_ibv_release(txq->ibv)) + txq->ibv = NULL; for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) { if (txq->txq.mp2mr[i]) { - priv_mr_release(priv, txq->txq.mp2mr[i]); + mlx5_mr_release(txq->txq.mp2mr[i]); txq->txq.mp2mr[i] = NULL; } } + if (priv->uar_base) + munmap((void *)RTE_ALIGN_FLOOR((uintptr_t)txq->txq.bf_reg, + page_size), page_size); if (rte_atomic32_dec_and_test(&txq->refcnt)) { txq_free_elts(txq); LIST_REMOVE(txq, next); @@ -739,14 +787,14 @@ mlx5_priv_txq_release(struct priv *priv, uint16_t idx) (*priv->txqs)[idx] = NULL; return 0; } - return EBUSY; + return 1; } /** * Verify if the queue can be released. * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * @param idx * TX queue index. * @@ -754,8 +802,9 @@ mlx5_priv_txq_release(struct priv *priv, uint16_t idx) * 1 if the queue can be released. */ int -mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx) +mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx) { + struct priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *txq; if (!(*priv->txqs)[idx]) @@ -767,20 +816,22 @@ mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx) /** * Verify the Tx Queue list is empty * - * @param priv - * Pointer to private structure. + * @param dev + * Pointer to Ethernet device. * - * @return the number of object not released. + * @return + * The number of object not released. */ int -mlx5_priv_txq_verify(struct priv *priv) +mlx5_txq_verify(struct rte_eth_dev *dev) { + struct priv *priv = dev->data->dev_private; struct mlx5_txq_ctrl *txq; int ret = 0; LIST_FOREACH(txq, &priv->txqsctrl, next) { - DEBUG("%p: Tx Queue %p still referenced", (void *)priv, - (void *)txq); + DRV_LOG(DEBUG, "port %u Tx queue %u still referenced", + dev->data->port_id, txq->idx); ++ret; } return ret; diff --git a/dpdk/drivers/net/mlx5/mlx5_utils.h b/dpdk/drivers/net/mlx5/mlx5_utils.h index 2fbd10b1..e48352f3 100644 --- a/dpdk/drivers/net/mlx5/mlx5_utils.h +++ b/dpdk/drivers/net/mlx5/mlx5_utils.h @@ -89,14 +89,21 @@ pmd_drv_log_basename(const char *s) return s; } +extern int mlx5_logtype; + +#define PMD_DRV_LOG___(level, ...) \ + rte_log(RTE_LOG_ ## level, \ + mlx5_logtype, \ + RTE_FMT(MLX5_DRIVER_NAME ": " \ + RTE_FMT_HEAD(__VA_ARGS__,), \ + RTE_FMT_TAIL(__VA_ARGS__,))) + /* * When debugging is enabled (NDEBUG not defined), file, line and function * information replace the driver name (MLX5_DRIVER_NAME) in log messages. */ #ifndef NDEBUG -#define PMD_DRV_LOG___(level, ...) \ - ERRNO_SAFE(RTE_LOG(level, PMD, __VA_ARGS__)) #define PMD_DRV_LOG__(level, ...) \ PMD_DRV_LOG___(level, "%s:%u: %s(): " __VA_ARGS__) #define PMD_DRV_LOG_(level, s, ...) \ @@ -108,9 +115,6 @@ pmd_drv_log_basename(const char *s) __VA_ARGS__) #else /* NDEBUG */ - -#define PMD_DRV_LOG___(level, ...) \ - ERRNO_SAFE(RTE_LOG(level, PMD, MLX5_DRIVER_NAME ": " __VA_ARGS__)) #define PMD_DRV_LOG__(level, ...) \ PMD_DRV_LOG___(level, __VA_ARGS__) #define PMD_DRV_LOG_(level, s, ...) \ @@ -119,18 +123,15 @@ pmd_drv_log_basename(const char *s) #endif /* NDEBUG */ /* Generic printf()-like logging macro with automatic line feed. */ -#define PMD_DRV_LOG(level, ...) \ +#define DRV_LOG(level, ...) \ PMD_DRV_LOG_(level, \ __VA_ARGS__ PMD_DRV_LOG_STRIP PMD_DRV_LOG_OPAREN, \ PMD_DRV_LOG_CPAREN) -/* - * Like assert(), DEBUG() becomes a no-op and claim_zero() does not perform - * any check when debugging is disabled. - */ +/* claim_zero() does not perform any check when debugging is disabled. */ #ifndef NDEBUG -#define DEBUG(...) PMD_DRV_LOG(DEBUG, __VA_ARGS__) +#define DEBUG(...) DRV_LOG(DEBUG, __VA_ARGS__) #define claim_zero(...) assert((__VA_ARGS__) == 0) #define claim_nonzero(...) assert((__VA_ARGS__) != 0) @@ -142,9 +143,9 @@ pmd_drv_log_basename(const char *s) #endif /* NDEBUG */ -#define INFO(...) PMD_DRV_LOG(INFO, __VA_ARGS__) -#define WARN(...) PMD_DRV_LOG(WARNING, __VA_ARGS__) -#define ERROR(...) PMD_DRV_LOG(ERR, __VA_ARGS__) +#define INFO(...) DRV_LOG(INFO, __VA_ARGS__) +#define WARN(...) DRV_LOG(WARNING, __VA_ARGS__) +#define ERROR(...) DRV_LOG(ERR, __VA_ARGS__) /* Convenience macros for accessing mbuf fields. */ #define NEXT(m) ((m)->next) diff --git a/dpdk/drivers/net/mlx5/mlx5_vlan.c b/dpdk/drivers/net/mlx5/mlx5_vlan.c index 198a69e3..dbfa8a0c 100644 --- a/dpdk/drivers/net/mlx5/mlx5_vlan.c +++ b/dpdk/drivers/net/mlx5/mlx5_vlan.c @@ -54,26 +54,24 @@ * Toggle filter. * * @return - * 0 on success, negative errno value on failure. + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) { struct priv *priv = dev->data->dev_private; unsigned int i; - int ret = 0; - priv_lock(priv); - DEBUG("%p: %s VLAN filter ID %" PRIu16, - (void *)dev, (on ? "enable" : "disable"), vlan_id); + DRV_LOG(DEBUG, "port %u %s VLAN filter ID %" PRIu16, + dev->data->port_id, (on ? "enable" : "disable"), vlan_id); assert(priv->vlan_filter_n <= RTE_DIM(priv->vlan_filter)); for (i = 0; (i != priv->vlan_filter_n); ++i) if (priv->vlan_filter[i] == vlan_id) break; /* Check if there's room for another VLAN filter. */ if (i == RTE_DIM(priv->vlan_filter)) { - ret = -ENOMEM; - goto out; + rte_errno = ENOMEM; + return -rte_errno; } if (i < priv->vlan_filter_n) { assert(priv->vlan_filter_n != 0); @@ -96,57 +94,10 @@ mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) priv->vlan_filter[priv->vlan_filter_n] = vlan_id; ++priv->vlan_filter_n; } - if (dev->data->dev_started) - priv_dev_traffic_restart(priv, dev); out: - priv_unlock(priv); - return ret; -} - -/** - * Set/reset VLAN stripping for a specific queue. - * - * @param priv - * Pointer to private structure. - * @param idx - * RX queue index. - * @param on - * Enable/disable VLAN stripping. - */ -static void -priv_vlan_strip_queue_set(struct priv *priv, uint16_t idx, int on) -{ - struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx]; - struct mlx5_rxq_ctrl *rxq_ctrl = - container_of(rxq, struct mlx5_rxq_ctrl, rxq); - struct ibv_wq_attr mod; - uint16_t vlan_offloads = - (on ? IBV_WQ_FLAGS_CVLAN_STRIPPING : 0) | - 0; - int err; - - DEBUG("set VLAN offloads 0x%x for port %d queue %d", - vlan_offloads, rxq->port_id, idx); - if (!rxq_ctrl->ibv) { - /* Update related bits in RX queue. */ - rxq->vlan_strip = !!on; - return; - } - mod = (struct ibv_wq_attr){ - .attr_mask = IBV_WQ_ATTR_FLAGS, - .flags_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING, - .flags = vlan_offloads, - }; - - err = ibv_modify_wq(rxq_ctrl->ibv->wq, &mod); - if (err) { - ERROR("%p: failed to modified stripping mode: %s", - (void *)priv, strerror(err)); - return; - } - - /* Update related bits in RX queue. */ - rxq->vlan_strip = !!on; + if (dev->data->dev_started) + return mlx5_traffic_restart(dev); + return 0; } /** @@ -163,22 +114,47 @@ void mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) { struct priv *priv = dev->data->dev_private; + struct mlx5_rxq_data *rxq = (*priv->rxqs)[queue]; + struct mlx5_rxq_ctrl *rxq_ctrl = + container_of(rxq, struct mlx5_rxq_ctrl, rxq); + struct ibv_wq_attr mod; + uint16_t vlan_offloads = + (on ? IBV_WQ_FLAGS_CVLAN_STRIPPING : 0) | + 0; + int ret; /* Validate hw support */ if (!priv->hw_vlan_strip) { - ERROR("VLAN stripping is not supported"); + DRV_LOG(ERR, "port %u VLAN stripping is not supported", + dev->data->port_id); return; } - /* Validate queue number */ if (queue >= priv->rxqs_n) { - ERROR("VLAN stripping, invalid queue number %d", queue); + DRV_LOG(ERR, "port %u VLAN stripping, invalid queue number %d", + dev->data->port_id, queue); return; } - - priv_lock(priv); - priv_vlan_strip_queue_set(priv, queue, on); - priv_unlock(priv); + DRV_LOG(DEBUG, "port %u set VLAN offloads 0x%x for port %uqueue %d", + dev->data->port_id, vlan_offloads, rxq->port_id, queue); + if (!rxq_ctrl->ibv) { + /* Update related bits in RX queue. */ + rxq->vlan_strip = !!on; + return; + } + mod = (struct ibv_wq_attr){ + .attr_mask = IBV_WQ_ATTR_FLAGS, + .flags_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING, + .flags = vlan_offloads, + }; + ret = ibv_modify_wq(rxq_ctrl->ibv->wq, &mod); + if (ret) { + DRV_LOG(ERR, "port %u failed to modified stripping mode: %s", + dev->data->port_id, strerror(rte_errno)); + return; + } + /* Update related bits in RX queue. */ + rxq->vlan_strip = !!on; } /** @@ -188,6 +164,9 @@ mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on) * Pointer to Ethernet device structure. * @param mask * VLAN offload bit mask. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask) @@ -199,16 +178,13 @@ mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask) int hw_vlan_strip = !!dev->data->dev_conf.rxmode.hw_vlan_strip; if (!priv->hw_vlan_strip) { - ERROR("VLAN stripping is not supported"); + DRV_LOG(ERR, "port %u VLAN stripping is not supported", + dev->data->port_id); return 0; } - /* Run on every RX queue and set/reset VLAN stripping. */ - priv_lock(priv); for (i = 0; (i != priv->rxqs_n); i++) - priv_vlan_strip_queue_set(priv, i, hw_vlan_strip); - priv_unlock(priv); + mlx5_vlan_strip_queue_set(dev, i, hw_vlan_strip); } - return 0; } diff --git a/dpdk/drivers/net/mrvl/mrvl_ethdev.c b/dpdk/drivers/net/mrvl/mrvl_ethdev.c index 9a358194..2fc24785 100644 --- a/dpdk/drivers/net/mrvl/mrvl_ethdev.c +++ b/dpdk/drivers/net/mrvl/mrvl_ethdev.c @@ -674,7 +674,8 @@ mrvl_dev_stop(struct rte_eth_dev *dev) pp2_cls_qos_tbl_deinit(priv->qos_tbl); priv->qos_tbl = NULL; } - pp2_ppio_deinit(priv->ppio); + if (priv->ppio) + pp2_ppio_deinit(priv->ppio); priv->ppio = NULL; } @@ -1217,8 +1218,8 @@ mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num) { - struct buff_release_entry entries[MRVL_PP2_TXD_MAX]; - struct rte_mbuf *mbufs[MRVL_PP2_TXD_MAX]; + struct buff_release_entry entries[MRVL_PP2_RXD_MAX]; + struct rte_mbuf *mbufs[MRVL_PP2_RXD_MAX]; int i, ret; unsigned int core_id; struct pp2_hif *hif; @@ -1374,9 +1375,12 @@ mrvl_rx_queue_release(void *rxq) if (core_id == LCORE_ID_ANY) core_id = 0; + if (!q) + return; + hif = mrvl_get_hif(q->priv, core_id); - if (!q || !hif) + if (!hif) return; tc = q->priv->rxq_map[q->queue_id].tc; diff --git a/dpdk/drivers/net/nfp/nfp_net.c b/dpdk/drivers/net/nfp/nfp_net.c index fa8ff3cb..8ab28dde 100644 --- a/dpdk/drivers/net/nfp/nfp_net.c +++ b/dpdk/drivers/net/nfp/nfp_net.c @@ -301,7 +301,7 @@ nfp_net_tx_queue_release_mbufs(struct nfp_net_txq *txq) for (i = 0; i < txq->tx_count; i++) { if (txq->txbufs[i].mbuf) { - rte_pktmbuf_free(txq->txbufs[i].mbuf); + rte_pktmbuf_free_seg(txq->txbufs[i].mbuf); txq->txbufs[i].mbuf = NULL; } } @@ -1244,9 +1244,9 @@ nfp_net_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info) dev_info->reta_size = NFP_NET_CFG_RSS_ITBL_SZ; dev_info->hash_key_size = NFP_NET_CFG_RSS_KEY_SZ; - dev_info->speed_capa = ETH_SPEED_NUM_1G | ETH_LINK_SPEED_10G | - ETH_SPEED_NUM_25G | ETH_SPEED_NUM_40G | - ETH_SPEED_NUM_50G | ETH_LINK_SPEED_100G; + dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G | + ETH_LINK_SPEED_25G | ETH_LINK_SPEED_40G | + ETH_LINK_SPEED_50G | ETH_LINK_SPEED_100G; if (hw->cap & NFP_NET_CFG_CTRL_LSO) dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; @@ -1995,16 +1995,16 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) break; } + rxds = &rxq->rxds[rxq->rd_p]; + if ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0) + break; + /* * Memory barrier to ensure that we won't do other * reads before the DD bit. */ rte_rmb(); - rxds = &rxq->rxds[rxq->rd_p]; - if ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0) - break; - /* * We got a packet. Let's alloc a new mbuff for refilling the * free descriptor ring as soon as possible @@ -2065,6 +2065,8 @@ nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) mb->nb_segs = 1; mb->next = NULL; + mb->port = rxq->port_id; + /* Checking the RSS flag */ nfp_net_set_hash(rxq, rxds, mb); @@ -2290,11 +2292,15 @@ nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) txq->wr_p = 0; pkt_size -= dma_size; - if (!pkt_size) - /* End of packet */ - txds->offset_eop |= PCIE_DESC_TX_EOP; + + /* + * Making the EOP, packets with just one segment + * the priority + */ + if (likely(!pkt_size)) + txds->offset_eop = PCIE_DESC_TX_EOP; else - txds->offset_eop &= PCIE_DESC_TX_OFFSET_MASK; + txds->offset_eop = 0; pkt = pkt->next; /* Referencing next free TX descriptor */ @@ -2647,6 +2653,14 @@ nfp_net_init(struct rte_eth_dev *eth_dev) pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); + /* NFP can not handle DMA addresses requiring more than 40 bits */ + if (rte_eal_check_dma_mask(40) < 0) { + RTE_LOG(INFO, PMD, "device %s can not be used:", + pci_dev->device.name); + RTE_LOG(INFO, PMD, "\trestricted dma mask to 40 bits!\n"); + return -ENODEV; + }; + if ((pci_dev->id.device_id == PCI_DEVICE_ID_NFP4000_PF_NIC) || (pci_dev->id.device_id == PCI_DEVICE_ID_NFP6000_PF_NIC)) { port = get_pf_port_number(eth_dev->data->name); @@ -3043,14 +3057,16 @@ static int eth_nfp_pci_remove(struct rte_pci_device *pci_dev) static struct rte_pci_driver rte_nfp_net_pf_pmd = { .id_table = pci_id_nfp_pf_net_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | + RTE_PCI_DRV_IOVA_AS_VA, .probe = nfp_pf_pci_probe, .remove = eth_nfp_pci_remove, }; static struct rte_pci_driver rte_nfp_net_vf_pmd = { .id_table = pci_id_nfp_vf_net_map, - .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC, + .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC | + RTE_PCI_DRV_IOVA_AS_VA, .probe = eth_nfp_pci_probe, .remove = eth_nfp_pci_remove, }; diff --git a/dpdk/drivers/net/nfp/nfp_nfpu.c b/dpdk/drivers/net/nfp/nfp_nfpu.c index f11afef3..c003640c 100644 --- a/dpdk/drivers/net/nfp/nfp_nfpu.c +++ b/dpdk/drivers/net/nfp/nfp_nfpu.c @@ -18,6 +18,22 @@ #define NFP_CFG_EXP_BAR 7 #define NFP_CFG_EXP_BAR_CFG_BASE 0x30000 +#define NFP_LOCKFILE_PATH_FMT "%s/nfp%d" + +/* get nfp lock file path (/var/lock if root, $HOME otherwise) */ +static void +nspu_get_lockfile_path(char *buffer, int bufsz, nfpu_desc_t *desc) +{ + const char *dir = "/var/lock"; + const char *home_dir = getenv("HOME"); + + if (getuid() != 0 && home_dir != NULL) + dir = home_dir; + + /* use current prefix as file path */ + snprintf(buffer, bufsz, NFP_LOCKFILE_PATH_FMT, dir, + desc->nfp); +} /* There could be other NFP userspace tools using the NSP interface. * Make sure there is no other process using it and locking the access for @@ -30,9 +46,7 @@ nspv_aquire_process_lock(nfpu_desc_t *desc) struct flock lock; char lockname[30]; - memset(&lock, 0, sizeof(lock)); - - snprintf(lockname, sizeof(lockname), "/var/lock/nfp%d", desc->nfp); + nspu_get_lockfile_path(lockname, sizeof(lockname), desc); /* Using S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH */ desc->lock = open(lockname, O_RDWR | O_CREAT, 0666); @@ -101,8 +115,12 @@ nfpu_open(struct rte_pci_device *pci_dev, nfpu_desc_t *desc, int nfp) int nfpu_close(nfpu_desc_t *desc) { + char lockname[30]; + rte_free(desc->nspu); close(desc->lock); - unlink("/var/lock/nfp0"); + + nspu_get_lockfile_path(lockname, sizeof(lockname), desc); + unlink(lockname); return 0; } diff --git a/dpdk/drivers/net/null/rte_eth_null.c b/dpdk/drivers/net/null/rte_eth_null.c index 726a5c5e..7a5f1257 100644 --- a/dpdk/drivers/net/null/rte_eth_null.c +++ b/dpdk/drivers/net/null/rte_eth_null.c @@ -91,7 +91,7 @@ static struct rte_eth_link pmd_link = { .link_speed = ETH_SPEED_NUM_10G, .link_duplex = ETH_LINK_FULL_DUPLEX, .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_AUTONEG, + .link_autoneg = ETH_LINK_FIXED, }; static uint16_t diff --git a/dpdk/drivers/net/octeontx/octeontx_ethdev.c b/dpdk/drivers/net/octeontx/octeontx_ethdev.c index eca3a39f..13dfbbc0 100644 --- a/dpdk/drivers/net/octeontx/octeontx_ethdev.c +++ b/dpdk/drivers/net/octeontx/octeontx_ethdev.c @@ -127,7 +127,7 @@ octeontx_port_open(struct octeontx_nic *nic) int res; res = 0; - + memset(&bgx_port_conf, 0x0, sizeof(bgx_port_conf)); PMD_INIT_FUNC_TRACE(); res = octeontx_bgx_port_open(nic->port_id, &bgx_port_conf); @@ -377,6 +377,9 @@ octeontx_dev_close(struct rte_eth_dev *dev) rte_free(txq); } + + dev->tx_pkt_burst = NULL; + dev->rx_pkt_burst = NULL; } static int @@ -470,9 +473,6 @@ octeontx_dev_stop(struct rte_eth_dev *dev) ret); return; } - - dev->tx_pkt_burst = NULL; - dev->rx_pkt_burst = NULL; } static void @@ -537,7 +537,6 @@ octeontx_dev_link_update(struct rte_eth_dev *dev, struct rte_eth_link link; int res; - res = 0; PMD_INIT_FUNC_TRACE(); res = octeontx_port_link_status(nic); @@ -571,6 +570,7 @@ octeontx_dev_link_update(struct rte_eth_dev *dev, case OCTEONTX_LINK_SPEED_RESERVE1: case OCTEONTX_LINK_SPEED_RESERVE2: default: + link.link_speed = ETH_SPEED_NUM_NONE; octeontx_log_err("incorrect link speed %d", nic->speed); break; } @@ -1142,7 +1142,7 @@ octeontx_create(struct rte_vdev_device *dev, int port, uint8_t evdev, return data->port_id; err: - if (port) + if (nic) octeontx_port_close(nic); if (eth_dev != NULL) { diff --git a/dpdk/drivers/net/pcap/rte_eth_pcap.c b/dpdk/drivers/net/pcap/rte_eth_pcap.c index 3385d043..f71ba001 100644 --- a/dpdk/drivers/net/pcap/rte_eth_pcap.c +++ b/dpdk/drivers/net/pcap/rte_eth_pcap.c @@ -124,7 +124,7 @@ static struct rte_eth_link pmd_link = { .link_speed = ETH_SPEED_NUM_10G, .link_duplex = ETH_LINK_FULL_DUPLEX, .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_AUTONEG, + .link_autoneg = ETH_LINK_FIXED, }; static int @@ -687,19 +687,19 @@ static const struct eth_dev_ops ops = { static int open_rx_pcap(const char *key, const char *value, void *extra_args) { - unsigned int i; const char *pcap_filename = value; struct pmd_devargs *rx = extra_args; pcap_t *pcap = NULL; - for (i = 0; i < rx->num_of_queue; i++) { - if (open_single_rx_pcap(pcap_filename, &pcap) < 0) - return -1; + if (rx->num_of_queue >= RTE_PMD_PCAP_MAX_QUEUES) + return -1; + if (open_single_rx_pcap(pcap_filename, &pcap) < 0) + return -1; - rx->queue[i].pcap = pcap; - rx->queue[i].name = pcap_filename; - rx->queue[i].type = key; - } + rx->queue[rx->num_of_queue].pcap = pcap; + rx->queue[rx->num_of_queue].name = pcap_filename; + rx->queue[rx->num_of_queue].type = key; + rx->num_of_queue++; return 0; } @@ -711,19 +711,19 @@ open_rx_pcap(const char *key, const char *value, void *extra_args) static int open_tx_pcap(const char *key, const char *value, void *extra_args) { - unsigned int i; const char *pcap_filename = value; struct pmd_devargs *dumpers = extra_args; pcap_dumper_t *dumper; - for (i = 0; i < dumpers->num_of_queue; i++) { - if (open_single_tx_pcap(pcap_filename, &dumper) < 0) - return -1; + if (dumpers->num_of_queue >= RTE_PMD_PCAP_MAX_QUEUES) + return -1; + if (open_single_tx_pcap(pcap_filename, &dumper) < 0) + return -1; - dumpers->queue[i].dumper = dumper; - dumpers->queue[i].name = pcap_filename; - dumpers->queue[i].type = key; - } + dumpers->queue[dumpers->num_of_queue].dumper = dumper; + dumpers->queue[dumpers->num_of_queue].name = pcap_filename; + dumpers->queue[dumpers->num_of_queue].type = key; + dumpers->num_of_queue++; return 0; } @@ -754,18 +754,18 @@ open_rx_tx_iface(const char *key, const char *value, void *extra_args) static inline int open_rx_iface(const char *key, const char *value, void *extra_args) { - unsigned int i; const char *iface = value; struct pmd_devargs *rx = extra_args; pcap_t *pcap = NULL; - for (i = 0; i < rx->num_of_queue; i++) { - if (open_single_iface(iface, &pcap) < 0) - return -1; - rx->queue[i].pcap = pcap; - rx->queue[i].name = iface; - rx->queue[i].type = key; - } + if (rx->num_of_queue >= RTE_PMD_PCAP_MAX_QUEUES) + return -1; + if (open_single_iface(iface, &pcap) < 0) + return -1; + rx->queue[rx->num_of_queue].pcap = pcap; + rx->queue[rx->num_of_queue].name = iface; + rx->queue[rx->num_of_queue].type = key; + rx->num_of_queue++; return 0; } @@ -776,18 +776,18 @@ open_rx_iface(const char *key, const char *value, void *extra_args) static int open_tx_iface(const char *key, const char *value, void *extra_args) { - unsigned int i; const char *iface = value; struct pmd_devargs *tx = extra_args; pcap_t *pcap; - for (i = 0; i < tx->num_of_queue; i++) { - if (open_single_iface(iface, &pcap) < 0) - return -1; - tx->queue[i].pcap = pcap; - tx->queue[i].name = iface; - tx->queue[i].type = key; - } + if (tx->num_of_queue >= RTE_PMD_PCAP_MAX_QUEUES) + return -1; + if (open_single_iface(iface, &pcap) < 0) + return -1; + tx->queue[tx->num_of_queue].pcap = pcap; + tx->queue[tx->num_of_queue].name = iface; + tx->queue[tx->num_of_queue].type = key; + tx->num_of_queue++; return 0; } @@ -977,15 +977,8 @@ pmd_pcap_probe(struct rte_vdev_device *dev) * We check whether we want to open a RX stream from a real NIC or a * pcap file */ - pcaps.num_of_queue = rte_kvargs_count(kvlist, ETH_PCAP_RX_PCAP_ARG); - if (pcaps.num_of_queue) - is_rx_pcap = 1; - else - pcaps.num_of_queue = rte_kvargs_count(kvlist, - ETH_PCAP_RX_IFACE_ARG); - - if (pcaps.num_of_queue > RTE_PMD_PCAP_MAX_QUEUES) - pcaps.num_of_queue = RTE_PMD_PCAP_MAX_QUEUES; + is_rx_pcap = rte_kvargs_count(kvlist, ETH_PCAP_RX_PCAP_ARG) ? 1 : 0; + pcaps.num_of_queue = 0; if (is_rx_pcap) ret = rte_kvargs_process(kvlist, ETH_PCAP_RX_PCAP_ARG, @@ -1001,15 +994,8 @@ pmd_pcap_probe(struct rte_vdev_device *dev) * We check whether we want to open a TX stream to a real NIC or a * pcap file */ - dumpers.num_of_queue = rte_kvargs_count(kvlist, ETH_PCAP_TX_PCAP_ARG); - if (dumpers.num_of_queue) - is_tx_pcap = 1; - else - dumpers.num_of_queue = rte_kvargs_count(kvlist, - ETH_PCAP_TX_IFACE_ARG); - - if (dumpers.num_of_queue > RTE_PMD_PCAP_MAX_QUEUES) - dumpers.num_of_queue = RTE_PMD_PCAP_MAX_QUEUES; + is_tx_pcap = rte_kvargs_count(kvlist, ETH_PCAP_TX_PCAP_ARG) ? 1 : 0; + dumpers.num_of_queue = 0; if (is_tx_pcap) ret = rte_kvargs_process(kvlist, ETH_PCAP_TX_PCAP_ARG, diff --git a/dpdk/drivers/net/qede/base/bcm_osal.c b/dpdk/drivers/net/qede/base/bcm_osal.c index fe42f325..2b9a5237 100644 --- a/dpdk/drivers/net/qede/base/bcm_osal.c +++ b/dpdk/drivers/net/qede/base/bcm_osal.c @@ -133,7 +133,7 @@ void *osal_dma_alloc_coherent(struct ecore_dev *p_dev, snprintf(mz_name, sizeof(mz_name) - 1, "%lx", (unsigned long)rte_get_timer_cycles()); if (core_id == (unsigned int)LCORE_ID_ANY) - core_id = 0; + core_id = rte_get_master_lcore(); socket_id = rte_lcore_to_socket_id(core_id); mz = rte_memzone_reserve_aligned(mz_name, size, socket_id, 0, RTE_CACHE_LINE_SIZE); @@ -172,7 +172,7 @@ void *osal_dma_alloc_coherent_aligned(struct ecore_dev *p_dev, snprintf(mz_name, sizeof(mz_name) - 1, "%lx", (unsigned long)rte_get_timer_cycles()); if (core_id == (unsigned int)LCORE_ID_ANY) - core_id = 0; + core_id = rte_get_master_lcore(); socket_id = rte_lcore_to_socket_id(core_id); mz = rte_memzone_reserve_aligned(mz_name, size, socket_id, 0, align); if (!mz) { @@ -200,6 +200,11 @@ void osal_dma_free_mem(struct ecore_dev *p_dev, dma_addr_t phys) DP_VERBOSE(p_dev, ECORE_MSG_SP, "Free memzone %s\n", ecore_mz_mapping[j]->name); rte_memzone_free(ecore_mz_mapping[j]); + while (j < ecore_mz_count - 1) { + ecore_mz_mapping[j] = ecore_mz_mapping[j + 1]; + j++; + } + ecore_mz_count--; return; } } diff --git a/dpdk/drivers/net/qede/base/ecore.h b/dpdk/drivers/net/qede/base/ecore.h index ce5f3a90..a44ce515 100644 --- a/dpdk/drivers/net/qede/base/ecore.h +++ b/dpdk/drivers/net/qede/base/ecore.h @@ -534,6 +534,12 @@ enum ecore_mf_mode_bit { ECORE_MF_UFP_SPECIFIC, ECORE_MF_DISABLE_ARFS, + + /* Use vlan for steering */ + ECORE_MF_8021Q_TAGGING, + + /* Use stag for steering */ + ECORE_MF_8021AD_TAGGING, }; enum ecore_ufp_mode { diff --git a/dpdk/drivers/net/qede/base/ecore_dcbx.c b/dpdk/drivers/net/qede/base/ecore_dcbx.c index 21ddda92..157a605d 100644 --- a/dpdk/drivers/net/qede/base/ecore_dcbx.c +++ b/dpdk/drivers/net/qede/base/ecore_dcbx.c @@ -149,6 +149,10 @@ ecore_dcbx_set_params(struct ecore_dcbx_results *p_data, } p_data->arr[type].update = UPDATE_DCB_DSCP; + /* Do not add valn tag 0 when DCB is enabled and port is in UFP mode */ + if (OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, &p_hwfn->p_dev->mf_bits)) + p_data->arr[type].dont_add_vlan0 = true; + /* QM reconf data */ if (p_hwfn->hw_info.personality == personality) p_hwfn->hw_info.offload_tc = tc; @@ -935,6 +939,7 @@ static void ecore_dcbx_update_protocol_data(struct protocol_dcb_data *p_data, p_data->dcb_tc = p_src->arr[type].tc; p_data->dscp_enable_flag = p_src->arr[type].dscp_enable; p_data->dscp_val = p_src->arr[type].dscp_val; + p_data->dcb_dont_add_vlan0 = p_src->arr[type].dont_add_vlan0; } /* Set pf update ramrod command params */ diff --git a/dpdk/drivers/net/qede/base/ecore_dcbx_api.h b/dpdk/drivers/net/qede/base/ecore_dcbx_api.h index 9ff4df4c..4df99ae2 100644 --- a/dpdk/drivers/net/qede/base/ecore_dcbx_api.h +++ b/dpdk/drivers/net/qede/base/ecore_dcbx_api.h @@ -29,6 +29,7 @@ struct ecore_dcbx_app_data { u8 tc; /* Traffic Class */ bool dscp_enable; /* DSCP enabled */ u8 dscp_val; /* DSCP value */ + bool dont_add_vlan0; /* Do not insert a vlan tag with id 0 */ }; #ifndef __EXTRACT__LINUX__ diff --git a/dpdk/drivers/net/qede/base/ecore_dev.c b/dpdk/drivers/net/qede/base/ecore_dev.c index da1830ce..092606be 100644 --- a/dpdk/drivers/net/qede/base/ecore_dev.c +++ b/dpdk/drivers/net/qede/base/ecore_dev.c @@ -2423,9 +2423,8 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, } } - /* Log and clean previous pglue_b errors if such exist */ + /* Log and clear previous pglue_b errors if such exist */ ecore_pglueb_rbc_attn_handler(p_hwfn, p_hwfn->p_main_ptt); - ecore_pglueb_clear_err(p_hwfn, p_hwfn->p_main_ptt); /* Enable the PF's internal FID_enable in the PXP */ rc = ecore_pglueb_set_pfid_enable(p_hwfn, p_hwfn->p_main_ptt, @@ -2433,6 +2432,13 @@ enum _ecore_status_t ecore_hw_init(struct ecore_dev *p_dev, if (rc != ECORE_SUCCESS) goto load_err; + /* Clear the pglue_b was_error indication. + * In E4 it must be done after the BME and the internal + * FID_enable for the PF are set, since VDMs may cause the + * indication to be set again. + */ + ecore_pglueb_clear_err(p_hwfn, p_hwfn->p_main_ptt); + switch (load_code) { case FW_MSG_CODE_DRV_LOAD_ENGINE: rc = ecore_hw_init_common(p_hwfn, p_hwfn->p_main_ptt, @@ -3496,9 +3502,14 @@ ecore_hw_get_nvm_info(struct ecore_hwfn *p_hwfn, break; case NVM_CFG1_GLOB_MF_MODE_UFP: p_hwfn->p_dev->mf_bits = 1 << ECORE_MF_OVLAN_CLSS | - 1 << ECORE_MF_UFP_SPECIFIC; + 1 << ECORE_MF_UFP_SPECIFIC | + 1 << ECORE_MF_8021Q_TAGGING; + break; + case NVM_CFG1_GLOB_MF_MODE_BD: + p_hwfn->p_dev->mf_bits = 1 << ECORE_MF_OVLAN_CLSS | + 1 << ECORE_MF_LLH_PROTO_CLSS | + 1 << ECORE_MF_8021AD_TAGGING; break; - case NVM_CFG1_GLOB_MF_MODE_NPAR1_0: p_hwfn->p_dev->mf_bits = 1 << ECORE_MF_LLH_MAC_CLSS | 1 << ECORE_MF_LLH_PROTO_CLSS | @@ -3527,6 +3538,7 @@ ecore_hw_get_nvm_info(struct ecore_hwfn *p_hwfn, */ switch (mf_mode) { case NVM_CFG1_GLOB_MF_MODE_MF_ALLOWED: + case NVM_CFG1_GLOB_MF_MODE_BD: p_hwfn->p_dev->mf_mode = ECORE_MF_OVLAN; break; case NVM_CFG1_GLOB_MF_MODE_NPAR1_0: diff --git a/dpdk/drivers/net/qede/base/ecore_hsi_common.h b/dpdk/drivers/net/qede/base/ecore_hsi_common.h index d8abd604..31ae2a0f 100644 --- a/dpdk/drivers/net/qede/base/ecore_hsi_common.h +++ b/dpdk/drivers/net/qede/base/ecore_hsi_common.h @@ -1515,7 +1515,10 @@ struct protocol_dcb_data { u8 dcb_priority /* dcbPri flag value */; u8 dcb_tc /* dcb TC value */; u8 dscp_val /* dscp value to write if dscp_enable_flag is set */; - u8 reserved0; +/* When DCB is enabled - if this flag is set, dont add VLAN 0 tag to untagged + * frames + */ + u8 dcb_dont_add_vlan0; }; /* diff --git a/dpdk/drivers/net/qede/base/ecore_int.c b/dpdk/drivers/net/qede/base/ecore_int.c index e6cef85b..61e36a43 100644 --- a/dpdk/drivers/net/qede/base/ecore_int.c +++ b/dpdk/drivers/net/qede/base/ecore_int.c @@ -231,15 +231,19 @@ static const char *grc_timeout_attn_master_to_str(u8 master) static enum _ecore_status_t ecore_grc_attn_cb(struct ecore_hwfn *p_hwfn) { + enum _ecore_status_t rc = ECORE_SUCCESS; u32 tmp, tmp2; /* We've already cleared the timeout interrupt register, so we learn - * of interrupts via the validity register + * of interrupts via the validity register. + * Any attention which is not for a timeout event is treated as fatal. */ tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, GRC_REG_TIMEOUT_ATTN_ACCESS_VALID); - if (!(tmp & ECORE_GRC_ATTENTION_VALID_BIT)) + if (!(tmp & ECORE_GRC_ATTENTION_VALID_BIT)) { + rc = ECORE_INVAL; goto out; + } /* Read the GRC timeout information */ tmp = ecore_rd(p_hwfn, p_hwfn->p_dpc_ptt, @@ -263,11 +267,11 @@ static enum _ecore_status_t ecore_grc_attn_cb(struct ecore_hwfn *p_hwfn) (tmp2 & ECORE_GRC_ATTENTION_VF_MASK) >> ECORE_GRC_ATTENTION_VF_SHIFT); -out: - /* Regardles of anything else, clean the validity bit */ + /* Clean the validity bit */ ecore_wr(p_hwfn, p_hwfn->p_dpc_ptt, GRC_REG_TIMEOUT_ATTN_ACCESS_VALID, 0); - return ECORE_SUCCESS; +out: + return rc; } #define ECORE_PGLUE_ATTENTION_VALID (1 << 29) diff --git a/dpdk/drivers/net/qede/base/ecore_l2.c b/dpdk/drivers/net/qede/base/ecore_l2.c index e3afc8a3..1f16697e 100644 --- a/dpdk/drivers/net/qede/base/ecore_l2.c +++ b/dpdk/drivers/net/qede/base/ecore_l2.c @@ -687,7 +687,7 @@ ecore_sp_update_mcast_bin(struct vport_update_ramrod_data *p_ramrod, p_ramrod->common.update_approx_mcast_flg = 1; for (i = 0; i < ETH_MULTICAST_MAC_BINS_IN_REGS; i++) { - u32 *p_bins = (u32 *)p_params->bins; + u32 *p_bins = p_params->bins; p_ramrod->approx_mcast.bins[i] = OSAL_CPU_TO_LE32(p_bins[i]); } @@ -1556,8 +1556,8 @@ ecore_sp_eth_filter_mcast(struct ecore_hwfn *p_hwfn, enum spq_mode comp_mode, struct ecore_spq_comp_cb *p_comp_data) { - unsigned long bins[ETH_MULTICAST_MAC_BINS_IN_REGS]; struct vport_update_ramrod_data *p_ramrod = OSAL_NULL; + u32 bins[ETH_MULTICAST_MAC_BINS_IN_REGS]; struct ecore_spq_entry *p_ent = OSAL_NULL; struct ecore_sp_init_data init_data; u8 abs_vport_id = 0; @@ -1596,8 +1596,7 @@ ecore_sp_eth_filter_mcast(struct ecore_hwfn *p_hwfn, /* explicitly clear out the entire vector */ OSAL_MEMSET(&p_ramrod->approx_mcast.bins, 0, sizeof(p_ramrod->approx_mcast.bins)); - OSAL_MEMSET(bins, 0, sizeof(unsigned long) * - ETH_MULTICAST_MAC_BINS_IN_REGS); + OSAL_MEMSET(bins, 0, sizeof(u32) * ETH_MULTICAST_MAC_BINS_IN_REGS); /* filter ADD op is explicit set op and it removes * any existing filters for the vport. */ @@ -1606,16 +1605,15 @@ ecore_sp_eth_filter_mcast(struct ecore_hwfn *p_hwfn, u32 bit; bit = ecore_mcast_bin_from_mac(p_filter_cmd->mac[i]); - OSAL_SET_BIT(bit, bins); + bins[bit / 32] |= 1 << (bit % 32); } /* Convert to correct endianity */ for (i = 0; i < ETH_MULTICAST_MAC_BINS_IN_REGS; i++) { struct vport_update_ramrod_mcast *p_ramrod_bins; - u32 *p_bins = (u32 *)bins; p_ramrod_bins = &p_ramrod->approx_mcast; - p_ramrod_bins->bins[i] = OSAL_CPU_TO_LE32(p_bins[i]); + p_ramrod_bins->bins[i] = OSAL_CPU_TO_LE32(bins[i]); } } diff --git a/dpdk/drivers/net/qede/base/ecore_l2_api.h b/dpdk/drivers/net/qede/base/ecore_l2_api.h index ed9837bf..8cbe8dde 100644 --- a/dpdk/drivers/net/qede/base/ecore_l2_api.h +++ b/dpdk/drivers/net/qede/base/ecore_l2_api.h @@ -332,7 +332,7 @@ struct ecore_sp_vport_update_params { u8 anti_spoofing_en; u8 update_accept_any_vlan_flg; u8 accept_any_vlan; - unsigned long bins[8]; + u32 bins[8]; struct ecore_rss_params *rss_params; struct ecore_filter_accept_flags accept_flags; struct ecore_sge_tpa_params *sge_tpa_params; diff --git a/dpdk/drivers/net/qede/base/ecore_sp_commands.c b/dpdk/drivers/net/qede/base/ecore_sp_commands.c index 7598e7a6..83705b8e 100644 --- a/dpdk/drivers/net/qede/base/ecore_sp_commands.c +++ b/dpdk/drivers/net/qede/base/ecore_sp_commands.c @@ -295,6 +295,7 @@ ecore_tunn_set_pf_start_params(struct ecore_hwfn *p_hwfn, } #define ETH_P_8021Q 0x8100 +#define ETH_P_8021AD 0x88A8 /* 802.1ad Service VLAN */ enum _ecore_status_t ecore_sp_pf_start(struct ecore_hwfn *p_hwfn, struct ecore_ptt *p_ptt, @@ -308,7 +309,7 @@ enum _ecore_status_t ecore_sp_pf_start(struct ecore_hwfn *p_hwfn, struct ecore_sp_init_data init_data; enum _ecore_status_t rc = ECORE_NOTIMPL; u8 page_cnt; - int i; + u8 i; /* update initial eq producer */ ecore_eq_prod_update(p_hwfn, @@ -343,18 +344,27 @@ enum _ecore_status_t ecore_sp_pf_start(struct ecore_hwfn *p_hwfn, p_ramrod->outer_tag_config.outer_tag.tci = OSAL_CPU_TO_LE16(p_hwfn->hw_info.ovlan); + if (OSAL_TEST_BIT(ECORE_MF_8021Q_TAGGING, &p_hwfn->p_dev->mf_bits)) { + p_ramrod->outer_tag_config.outer_tag.tpid = ETH_P_8021Q; + } else if (OSAL_TEST_BIT(ECORE_MF_8021AD_TAGGING, + &p_hwfn->p_dev->mf_bits)) { + p_ramrod->outer_tag_config.outer_tag.tpid = ETH_P_8021AD; + p_ramrod->outer_tag_config.enable_stag_pri_change = 1; + } + p_ramrod->outer_tag_config.pri_map_valid = 1; + for (i = 0; i < ECORE_MAX_PFC_PRIORITIES; i++) + p_ramrod->outer_tag_config.inner_to_outer_pri_map[i] = i; + + /* enable_stag_pri_change should be set if port is in BD mode or, + * UFP with Host Control mode or, UFP with DCB over base interface. + */ if (OSAL_TEST_BIT(ECORE_MF_UFP_SPECIFIC, &p_hwfn->p_dev->mf_bits)) { - p_ramrod->outer_tag_config.outer_tag.tpid = - OSAL_CPU_TO_LE16(ETH_P_8021Q); - if (p_hwfn->ufp_info.pri_type == ECORE_UFP_PRI_OS) + if ((p_hwfn->ufp_info.pri_type == ECORE_UFP_PRI_OS) || + (p_hwfn->p_dcbx_info->results.dcbx_enabled)) p_ramrod->outer_tag_config.enable_stag_pri_change = 1; else p_ramrod->outer_tag_config.enable_stag_pri_change = 0; - p_ramrod->outer_tag_config.pri_map_valid = 1; - for (i = 0; i < 8; i++) - p_ramrod->outer_tag_config.inner_to_outer_pri_map[i] = - (u8)i; } /* Place EQ address in RAMROD */ @@ -451,7 +461,8 @@ enum _ecore_status_t ecore_sp_pf_update_ufp(struct ecore_hwfn *p_hwfn) return rc; p_ent->ramrod.pf_update.update_enable_stag_pri_change = true; - if (p_hwfn->ufp_info.pri_type == ECORE_UFP_PRI_OS) + if ((p_hwfn->ufp_info.pri_type == ECORE_UFP_PRI_OS) || + (p_hwfn->p_dcbx_info->results.dcbx_enabled)) p_ent->ramrod.pf_update.enable_stag_pri_change = 1; else p_ent->ramrod.pf_update.enable_stag_pri_change = 0; diff --git a/dpdk/drivers/net/qede/base/ecore_sriov.c b/dpdk/drivers/net/qede/base/ecore_sriov.c index b1e26d6f..1c885e1c 100644 --- a/dpdk/drivers/net/qede/base/ecore_sriov.c +++ b/dpdk/drivers/net/qede/base/ecore_sriov.c @@ -61,6 +61,8 @@ const char *ecore_channel_tlvs_string[] = { "CHANNEL_TLV_COALESCE_UPDATE", "CHANNEL_TLV_QID", "CHANNEL_TLV_COALESCE_READ", + "CHANNEL_TLV_BULLETIN_UPDATE_MAC", + "CHANNEL_TLV_UPDATE_MTU", "CHANNEL_TLV_MAX" }; @@ -2854,6 +2856,45 @@ out: length, status); } +static enum _ecore_status_t +ecore_iov_vf_pf_update_mtu(struct ecore_hwfn *p_hwfn, + struct ecore_ptt *p_ptt, + struct ecore_vf_info *p_vf) +{ + struct ecore_iov_vf_mbx *mbx = &p_vf->vf_mbx; + struct ecore_sp_vport_update_params params; + enum _ecore_status_t rc = ECORE_SUCCESS; + struct vfpf_update_mtu_tlv *p_req; + u8 status = PFVF_STATUS_SUCCESS; + + /* Valiate PF can send such a request */ + if (!p_vf->vport_instance) { + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "No VPORT instance available for VF[%d], failing MTU update\n", + p_vf->abs_vf_id); + status = PFVF_STATUS_FAILURE; + goto send_status; + } + + p_req = &mbx->req_virt->update_mtu; + + OSAL_MEMSET(¶ms, 0, sizeof(params)); + params.opaque_fid = p_vf->opaque_fid; + params.vport_id = p_vf->vport_id; + params.mtu = p_req->mtu; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, ECORE_SPQ_MODE_EBLOCK, + OSAL_NULL); + + if (rc) + status = PFVF_STATUS_FAILURE; +send_status: + ecore_iov_prepare_resp(p_hwfn, p_ptt, p_vf, + CHANNEL_TLV_UPDATE_MTU, + sizeof(struct pfvf_def_resp_tlv), + status); + return rc; +} + void *ecore_iov_search_list_tlvs(struct ecore_hwfn *p_hwfn, void *p_tlvs_list, u16 req_type) { @@ -2975,8 +3016,7 @@ ecore_iov_vp_update_mcast_bin_param(struct ecore_hwfn *p_hwfn, p_data->update_approx_mcast_flg = 1; OSAL_MEMCPY(p_data->bins, p_mcast_tlv->bins, - sizeof(unsigned long) * - ETH_MULTICAST_MAC_BINS_IN_REGS); + sizeof(u32) * ETH_MULTICAST_MAC_BINS_IN_REGS); *tlvs_mask |= 1 << ECORE_IOV_VP_UPDATE_MCAST; } @@ -4137,6 +4177,9 @@ void ecore_iov_process_mbx_req(struct ecore_hwfn *p_hwfn, case CHANNEL_TLV_COALESCE_READ: ecore_iov_vf_pf_get_coalesce(p_hwfn, p_ptt, p_vf); break; + case CHANNEL_TLV_UPDATE_MTU: + ecore_iov_vf_pf_update_mtu(p_hwfn, p_ptt, p_vf); + break; } } else if (ecore_iov_tlv_supported(mbx->first_tlv.tl.type)) { /* If we've received a message from a VF we consider malicious diff --git a/dpdk/drivers/net/qede/base/ecore_vf.c b/dpdk/drivers/net/qede/base/ecore_vf.c index e0f2dd5a..334db6b9 100644 --- a/dpdk/drivers/net/qede/base/ecore_vf.c +++ b/dpdk/drivers/net/qede/base/ecore_vf.c @@ -1275,8 +1275,7 @@ ecore_vf_pf_vport_update(struct ecore_hwfn *p_hwfn, resp_size += sizeof(struct pfvf_def_resp_tlv); OSAL_MEMCPY(p_mcast_tlv->bins, p_params->bins, - sizeof(unsigned long) * - ETH_MULTICAST_MAC_BINS_IN_REGS); + sizeof(u32) * ETH_MULTICAST_MAC_BINS_IN_REGS); } update_rx = p_params->accept_flags.update_rx_mode_config; @@ -1473,7 +1472,7 @@ void ecore_vf_pf_filter_mcast(struct ecore_hwfn *p_hwfn, u32 bit; bit = ecore_mcast_bin_from_mac(p_filter_cmd->mac[i]); - OSAL_SET_BIT(bit, sp_params.bins); + sp_params.bins[bit / 32] |= 1 << (bit % 32); } } @@ -1629,6 +1628,39 @@ exit: return rc; } +enum _ecore_status_t +ecore_vf_pf_update_mtu(struct ecore_hwfn *p_hwfn, u16 mtu) +{ + struct ecore_vf_iov *p_iov = p_hwfn->vf_iov_info; + struct vfpf_update_mtu_tlv *p_req; + struct pfvf_def_resp_tlv *p_resp; + enum _ecore_status_t rc; + + if (!mtu) + return ECORE_INVAL; + + /* clear mailbox and prep header tlv */ + p_req = ecore_vf_pf_prep(p_hwfn, CHANNEL_TLV_UPDATE_MTU, + sizeof(*p_req)); + p_req->mtu = mtu; + DP_VERBOSE(p_hwfn, ECORE_MSG_IOV, + "Requesting MTU update to %d\n", mtu); + + /* add list termination tlv */ + ecore_add_tlv(&p_iov->offset, + CHANNEL_TLV_LIST_END, + sizeof(struct channel_list_end_tlv)); + + p_resp = &p_iov->pf2vf_reply->default_resp; + rc = ecore_send_msg2pf(p_hwfn, &p_resp->hdr.status, sizeof(*p_resp)); + if (p_resp->hdr.status == PFVF_STATUS_NOT_SUPPORTED) + rc = ECORE_INVAL; + + ecore_vf_pf_req_end(p_hwfn, rc); + + return rc; +} + u16 ecore_vf_get_igu_sb_id(struct ecore_hwfn *p_hwfn, u16 sb_id) { diff --git a/dpdk/drivers/net/qede/base/ecore_vf.h b/dpdk/drivers/net/qede/base/ecore_vf.h index de2758cb..e5555bb1 100644 --- a/dpdk/drivers/net/qede/base/ecore_vf.h +++ b/dpdk/drivers/net/qede/base/ecore_vf.h @@ -319,5 +319,14 @@ void ecore_vf_set_vf_start_tunn_update_param(struct ecore_tunnel_info *p_tun); u32 ecore_vf_hw_bar_size(struct ecore_hwfn *p_hwfn, enum BAR_ID bar_id); + +/** + * @brief - ecore_vf_pf_update_mtu Update MTU for VF. + * + * @param p_hwfn + * @param - mtu + */ +enum _ecore_status_t +ecore_vf_pf_update_mtu(struct ecore_hwfn *p_hwfn, u16 mtu); #endif #endif /* __ECORE_VF_H__ */ diff --git a/dpdk/drivers/net/qede/base/ecore_vfpf_if.h b/dpdk/drivers/net/qede/base/ecore_vfpf_if.h index ecb00649..08b1f248 100644 --- a/dpdk/drivers/net/qede/base/ecore_vfpf_if.h +++ b/dpdk/drivers/net/qede/base/ecore_vfpf_if.h @@ -396,7 +396,13 @@ struct vfpf_vport_update_mcast_bin_tlv { struct channel_tlv tl; u8 padding[4]; - u64 bins[8]; + /* This was a mistake; There are only 256 approx bins, + * and in HSI they're divided into 32-bit values. + * As old VFs used to set-bit to the values on its side, + * the upper half of the array is never expected to contain any data. + */ + u64 bins[4]; + u64 obsolete_bins[4]; }; struct vfpf_vport_update_accept_param_tlv { @@ -525,6 +531,18 @@ struct pfvf_read_coal_resp_tlv { u8 padding[6]; }; +struct vfpf_bulletin_update_mac_tlv { + struct vfpf_first_tlv first_tlv; + u8 mac[ETH_ALEN]; + u8 padding[2]; +}; + +struct vfpf_update_mtu_tlv { + struct vfpf_first_tlv first_tlv; + u16 mtu; + u8 padding[6]; +}; + union vfpf_tlvs { struct vfpf_first_tlv first_tlv; struct vfpf_acquire_tlv acquire; @@ -539,6 +557,8 @@ union vfpf_tlvs { struct vfpf_update_tunn_param_tlv tunn_param_update; struct vfpf_update_coalesce update_coalesce; struct vfpf_read_coal_req_tlv read_coal_req; + struct vfpf_bulletin_update_mac_tlv bulletin_update_mac; + struct vfpf_update_mtu_tlv update_mtu; struct tlv_buffer_size tlv_buf_size; }; @@ -669,6 +689,8 @@ enum { CHANNEL_TLV_COALESCE_UPDATE, CHANNEL_TLV_QID, CHANNEL_TLV_COALESCE_READ, + CHANNEL_TLV_BULLETIN_UPDATE_MAC, + CHANNEL_TLV_UPDATE_MTU, CHANNEL_TLV_MAX, /* Required for iterating over vport-update tlvs. diff --git a/dpdk/drivers/net/qede/qede_ethdev.c b/dpdk/drivers/net/qede/qede_ethdev.c index 73764e9a..4a5e4857 100644 --- a/dpdk/drivers/net/qede/qede_ethdev.c +++ b/dpdk/drivers/net/qede/qede_ethdev.c @@ -335,6 +335,24 @@ static void qede_interrupt_action(struct ecore_hwfn *p_hwfn) ecore_int_sp_dpc((osal_int_ptr_t)(p_hwfn)); } +static void +qede_interrupt_handler_intx(void *param) +{ + struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)param; + struct qede_dev *qdev = eth_dev->data->dev_private; + struct ecore_dev *edev = &qdev->edev; + u64 status; + + /* Check if our device actually raised an interrupt */ + status = ecore_int_igu_read_sisr_reg(ECORE_LEADING_HWFN(edev)); + if (status & 0x1) { + qede_interrupt_action(ECORE_LEADING_HWFN(edev)); + + if (rte_intr_enable(eth_dev->intr_handle)) + DP_ERR(edev, "rte_intr_enable failed\n"); + } +} + static void qede_interrupt_handler(void *param) { @@ -442,38 +460,6 @@ static void qede_reset_queue_stats(struct qede_dev *qdev, bool xstats) } } -static int -qede_start_vport(struct qede_dev *qdev, uint16_t mtu) -{ - struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); - struct ecore_sp_vport_start_params params; - struct ecore_hwfn *p_hwfn; - int rc; - int i; - - memset(¶ms, 0, sizeof(params)); - params.vport_id = 0; - params.mtu = mtu; - /* @DPDK - Disable FW placement */ - params.zero_placement_offset = 1; - for_each_hwfn(edev, i) { - p_hwfn = &edev->hwfns[i]; - params.concrete_fid = p_hwfn->hw_info.concrete_fid; - params.opaque_fid = p_hwfn->hw_info.opaque_fid; - rc = ecore_sp_vport_start(p_hwfn, ¶ms); - if (rc != ECORE_SUCCESS) { - DP_ERR(edev, "Start V-PORT failed %d\n", rc); - return rc; - } - } - ecore_reset_vport_stats(edev); - if (IS_PF(edev)) - qede_reset_queue_stats(qdev, true); - DP_INFO(edev, "VPORT started with MTU = %u\n", mtu); - - return 0; -} - static int qede_stop_vport(struct ecore_dev *edev) { @@ -493,6 +479,42 @@ qede_stop_vport(struct ecore_dev *edev) } } + DP_INFO(edev, "vport stopped\n"); + + return 0; +} + +static int +qede_start_vport(struct qede_dev *qdev, uint16_t mtu) +{ + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_sp_vport_start_params params; + struct ecore_hwfn *p_hwfn; + int rc; + int i; + + if (qdev->vport_started) + qede_stop_vport(edev); + + memset(¶ms, 0, sizeof(params)); + params.vport_id = 0; + params.mtu = mtu; + /* @DPDK - Disable FW placement */ + params.zero_placement_offset = 1; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.concrete_fid = p_hwfn->hw_info.concrete_fid; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_start(p_hwfn, ¶ms); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Start V-PORT failed %d\n", rc); + return rc; + } + } + ecore_reset_vport_stats(edev); + qdev->vport_started = true; + DP_INFO(edev, "VPORT started with MTU = %u\n", mtu); + return 0; } @@ -512,12 +534,9 @@ int qede_activate_vport(struct rte_eth_dev *eth_dev, bool flg) params.update_vport_active_tx_flg = 1; params.vport_active_rx_flg = flg; params.vport_active_tx_flg = flg; - if (!qdev->enable_tx_switching) { - if (IS_VF(edev)) { - params.update_tx_switching_flg = 1; - params.tx_switching_flg = !flg; - DP_INFO(edev, "VF tx-switching is disabled\n"); - } + if (~qdev->enable_tx_switching & flg) { + params.update_tx_switching_flg = 1; + params.tx_switching_flg = !flg; } for_each_hwfn(edev, i) { p_hwfn = &edev->hwfns[i]; @@ -593,37 +612,6 @@ int qede_enable_tpa(struct rte_eth_dev *eth_dev, bool flg) return 0; } -/* Update MTU via vport-update without doing port restart. - * The vport must be deactivated before calling this API. - */ -int qede_update_mtu(struct rte_eth_dev *eth_dev, uint16_t mtu) -{ - struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); - struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); - struct ecore_sp_vport_update_params params; - struct ecore_hwfn *p_hwfn; - int rc; - int i; - - memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); - params.vport_id = 0; - params.mtu = mtu; - params.vport_id = 0; - for_each_hwfn(edev, i) { - p_hwfn = &edev->hwfns[i]; - params.opaque_fid = p_hwfn->hw_info.opaque_fid; - rc = ecore_sp_vport_update(p_hwfn, ¶ms, - ECORE_SPQ_MODE_EBLOCK, NULL); - if (rc != ECORE_SUCCESS) { - DP_ERR(edev, "Failed to update MTU\n"); - return -1; - } - } - DP_INFO(edev, "MTU updated to %u\n", mtu); - - return 0; -} - static void qede_set_ucast_cmn_params(struct ecore_filter_ucast *ucast) { memset(ucast, 0, sizeof(struct ecore_filter_ucast)); @@ -735,10 +723,10 @@ qede_ucast_filter(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *ucast, ETHER_ADDR_LEN) == 0) && ucast->vni == tmp->vni && ucast->vlan == tmp->vlan) { - DP_ERR(edev, "Unicast MAC is already added" - " with vlan = %u, vni = %u\n", - ucast->vlan, ucast->vni); - return -EEXIST; + DP_INFO(edev, "Unicast MAC is already added" + " with vlan = %u, vni = %u\n", + ucast->vlan, ucast->vni); + return 0; } } u = rte_malloc(NULL, sizeof(struct qede_ucast_entry), @@ -772,47 +760,69 @@ qede_ucast_filter(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *ucast, } static int -qede_mcast_filter(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *mcast, - bool add) +qede_add_mcast_filters(struct rte_eth_dev *eth_dev, struct ether_addr *mc_addrs, + uint32_t mc_addrs_num) { struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); - struct ether_addr *mac_addr; - struct qede_mcast_entry *tmp = NULL; - struct qede_mcast_entry *m; + struct ecore_filter_mcast mcast; + struct qede_mcast_entry *m = NULL; + uint8_t i; + int rc; - mac_addr = (struct ether_addr *)mcast->mac; - if (add) { - SLIST_FOREACH(tmp, &qdev->mc_list_head, list) { - if (memcmp(mac_addr, &tmp->mac, ETHER_ADDR_LEN) == 0) { - DP_ERR(edev, - "Multicast MAC is already added\n"); - return -EEXIST; - } - } + for (i = 0; i < mc_addrs_num; i++) { m = rte_malloc(NULL, sizeof(struct qede_mcast_entry), - RTE_CACHE_LINE_SIZE); + RTE_CACHE_LINE_SIZE); if (!m) { - DP_ERR(edev, - "Did not allocate memory for mcast\n"); + DP_ERR(edev, "Did not allocate memory for mcast\n"); return -ENOMEM; } - ether_addr_copy(mac_addr, &m->mac); + ether_addr_copy(&mc_addrs[i], &m->mac); SLIST_INSERT_HEAD(&qdev->mc_list_head, m, list); - qdev->num_mc_addr++; - } else { - SLIST_FOREACH(tmp, &qdev->mc_list_head, list) { - if (memcmp(mac_addr, &tmp->mac, ETHER_ADDR_LEN) == 0) - break; - } - if (tmp == NULL) { - DP_INFO(edev, "Multicast mac is not found\n"); - return -EINVAL; - } - SLIST_REMOVE(&qdev->mc_list_head, tmp, - qede_mcast_entry, list); - qdev->num_mc_addr--; } + memset(&mcast, 0, sizeof(mcast)); + mcast.num_mc_addrs = mc_addrs_num; + mcast.opcode = ECORE_FILTER_ADD; + for (i = 0; i < mc_addrs_num; i++) + ether_addr_copy(&mc_addrs[i], (struct ether_addr *) + &mcast.mac[i]); + rc = ecore_filter_mcast_cmd(edev, &mcast, ECORE_SPQ_MODE_CB, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to add multicast filter (rc = %d\n)", rc); + return -1; + } + + return 0; +} + +static int qede_del_mcast_filters(struct rte_eth_dev *eth_dev) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct qede_mcast_entry *tmp = NULL; + struct ecore_filter_mcast mcast; + int j; + int rc; + + memset(&mcast, 0, sizeof(mcast)); + mcast.num_mc_addrs = qdev->num_mc_addr; + mcast.opcode = ECORE_FILTER_REMOVE; + j = 0; + SLIST_FOREACH(tmp, &qdev->mc_list_head, list) { + ether_addr_copy(&tmp->mac, (struct ether_addr *)&mcast.mac[j]); + j++; + } + rc = ecore_filter_mcast_cmd(edev, &mcast, ECORE_SPQ_MODE_CB, NULL); + if (rc != ECORE_SUCCESS) { + DP_ERR(edev, "Failed to delete multicast filter\n"); + return -1; + } + /* Init the list */ + while (!SLIST_EMPTY(&qdev->mc_list_head)) { + tmp = SLIST_FIRST(&qdev->mc_list_head); + SLIST_REMOVE_HEAD(&qdev->mc_list_head, list); + } + SLIST_INIT(&qdev->mc_list_head); return 0; } @@ -823,59 +833,25 @@ qede_mac_int_ops(struct rte_eth_dev *eth_dev, struct ecore_filter_ucast *ucast, { struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); - enum _ecore_status_t rc; - struct ecore_filter_mcast mcast; - struct qede_mcast_entry *tmp; - uint16_t j = 0; + enum _ecore_status_t rc = ECORE_INVAL; - /* Multicast */ - if (is_multicast_ether_addr((struct ether_addr *)ucast->mac)) { - if (add) { - if (qdev->num_mc_addr >= ECORE_MAX_MC_ADDRS) { - DP_ERR(edev, - "Mcast filter table limit exceeded, " - "Please enable mcast promisc mode\n"); - return -ECORE_INVAL; - } - } - rc = qede_mcast_filter(eth_dev, ucast, add); - if (rc == 0) { - DP_INFO(edev, "num_mc_addrs = %u\n", qdev->num_mc_addr); - memset(&mcast, 0, sizeof(mcast)); - mcast.num_mc_addrs = qdev->num_mc_addr; - mcast.opcode = ECORE_FILTER_ADD; - SLIST_FOREACH(tmp, &qdev->mc_list_head, list) { - ether_addr_copy(&tmp->mac, - (struct ether_addr *)&mcast.mac[j]); - j++; - } - rc = ecore_filter_mcast_cmd(edev, &mcast, - ECORE_SPQ_MODE_CB, NULL); - } - if (rc != ECORE_SUCCESS) { - DP_ERR(edev, "Failed to add multicast filter" - " rc = %d, op = %d\n", rc, add); - } - } else { /* Unicast */ - if (add) { - if (qdev->num_uc_addr >= - qdev->dev_info.num_mac_filters) { - DP_ERR(edev, - "Ucast filter table limit exceeded," - " Please enable promisc mode\n"); - return -ECORE_INVAL; - } - } - rc = qede_ucast_filter(eth_dev, ucast, add); - if (rc == 0) - rc = ecore_filter_ucast_cmd(edev, ucast, - ECORE_SPQ_MODE_CB, NULL); - if (rc != ECORE_SUCCESS) { - DP_ERR(edev, "MAC filter failed, rc = %d, op = %d\n", - rc, add); - } + if (add && (qdev->num_uc_addr >= qdev->dev_info.num_mac_filters)) { + DP_ERR(edev, "Ucast filter table limit exceeded," + " Please enable promisc mode\n"); + return ECORE_INVAL; } + rc = qede_ucast_filter(eth_dev, ucast, add); + if (rc == 0) + rc = ecore_filter_ucast_cmd(edev, ucast, + ECORE_SPQ_MODE_CB, NULL); + /* Indicate error only for add filter operation. + * Delete filter operations are not severe. + */ + if ((rc != ECORE_SUCCESS) && add) + DP_ERR(edev, "MAC filter failed, rc = %d, op = %d\n", + rc, add); + return rc; } @@ -886,7 +862,11 @@ qede_mac_addr_add(struct rte_eth_dev *eth_dev, struct ether_addr *mac_addr, struct ecore_filter_ucast ucast; int re; + if (!is_valid_assigned_ether_addr(mac_addr)) + return -EINVAL; + qede_set_ucast_cmn_params(&ucast); + ucast.opcode = ECORE_FILTER_ADD; ucast.type = ECORE_FILTER_MAC; ether_addr_copy(mac_addr, (struct ether_addr *)&ucast.mac); re = (int)qede_mac_int_ops(eth_dev, &ucast, 1); @@ -908,6 +888,9 @@ qede_mac_addr_remove(struct rte_eth_dev *eth_dev, uint32_t index) return; } + if (!is_valid_assigned_ether_addr(ð_dev->data->mac_addrs[index])) + return; + qede_set_ucast_cmn_params(&ucast); ucast.opcode = ECORE_FILTER_REMOVE; ucast.type = ECORE_FILTER_MAC; @@ -916,7 +899,7 @@ qede_mac_addr_remove(struct rte_eth_dev *eth_dev, uint32_t index) ether_addr_copy(ð_dev->data->mac_addrs[index], (struct ether_addr *)&ucast.mac); - ecore_filter_ucast_cmd(edev, &ucast, ECORE_SPQ_MODE_CB, NULL); + qede_mac_int_ops(eth_dev, &ucast, false); } static void @@ -933,6 +916,7 @@ qede_mac_addr_set(struct rte_eth_dev *eth_dev, struct ether_addr *mac_addr) return; } + qede_mac_addr_remove(eth_dev, 0); qede_mac_addr_add(eth_dev, mac_addr, 0, 0); } @@ -1011,9 +995,9 @@ static int qede_vlan_filter_set(struct rte_eth_dev *eth_dev, SLIST_FOREACH(tmp, &qdev->vlan_list_head, list) { if (tmp->vid == vlan_id) { - DP_ERR(edev, "VLAN %u already configured\n", - vlan_id); - return -EEXIST; + DP_INFO(edev, "VLAN %u already configured\n", + vlan_id); + return 0; } } @@ -1116,6 +1100,8 @@ static int qede_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask) DP_INFO(edev, "No offloads are supported with VLAN Q-in-Q" " and classification is based on outer tag only\n"); + qdev->vlan_offload_mask = mask; + DP_INFO(edev, "vlan offload mask %d vlan-strip %d vlan-filter %d\n", mask, rxmode->hw_vlan_strip, rxmode->hw_vlan_filter); @@ -1192,10 +1178,9 @@ static int qede_dev_start(struct rte_eth_dev *eth_dev) PMD_INIT_FUNC_TRACE(edev); /* Update MTU only if it has changed */ - if (qdev->mtu != qdev->new_mtu) { - if (qede_update_mtu(eth_dev, qdev->new_mtu)) + if (eth_dev->data->mtu != qdev->mtu) { + if (qede_update_mtu(eth_dev, qdev->mtu)) goto err; - qdev->mtu = qdev->new_mtu; } /* Configure TPA parameters */ @@ -1211,6 +1196,9 @@ static int qede_dev_start(struct rte_eth_dev *eth_dev) if (qede_start_queues(eth_dev)) goto err; + if (IS_PF(edev)) + qede_reset_queue_stats(qdev, true); + /* Newer SR-IOV PF driver expects RX/TX queues to be started before * enabling RSS. Hence RSS configuration is deferred upto this point. * Also, we would like to retain similar behavior in PF case, so we @@ -1224,9 +1212,6 @@ static int qede_dev_start(struct rte_eth_dev *eth_dev) if (qede_activate_vport(eth_dev, true)) goto err; - /* Bring-up the link */ - qede_dev_set_link_state(eth_dev, true); - /* Update link status */ qede_link_update(eth_dev, 0); @@ -1261,16 +1246,13 @@ static void qede_dev_stop(struct rte_eth_dev *eth_dev) /* Disable traffic */ ecore_hw_stop_fastpath(edev); /* TBD - loop */ - /* Bring the link down */ - qede_dev_set_link_state(eth_dev, false); - DP_INFO(edev, "Device is stopped\n"); } -#define QEDE_TX_SWITCHING "vf_txswitch" +#define QEDE_VF_TX_SWITCHING "vf_tx_switching" const char *valid_args[] = { - QEDE_TX_SWITCHING, + QEDE_VF_TX_SWITCHING, NULL, }; @@ -1280,9 +1262,7 @@ static int qede_args_check(const char *key, const char *val, void *opaque) int ret = 0; struct rte_eth_dev *eth_dev = opaque; struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); -#ifdef RTE_LIBRTE_QEDE_DEBUG_INFO struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); -#endif errno = 0; tmp = strtoul(val, NULL, 0); @@ -1291,8 +1271,10 @@ static int qede_args_check(const char *key, const char *val, void *opaque) return errno; } - if (strcmp(QEDE_TX_SWITCHING, key) == 0) + if (strcmp(QEDE_VF_TX_SWITCHING, key) == 0 && IS_VF(edev)) { qdev->enable_tx_switching = !!tmp; + DP_INFO(edev, "Disabling VF tx-switching\n"); + } return ret; } @@ -1367,7 +1349,8 @@ static int qede_dev_configure(struct rte_eth_dev *eth_dev) /* Parse devargs and fix up rxmode */ if (qede_args(eth_dev)) - return -ENOTSUP; + DP_NOTICE(edev, false, + "Invalid devargs supplied, requested change will not take effect\n"); /* Sanity checks and throw warnings */ if (rxmode->enable_scatter) @@ -1390,20 +1373,11 @@ static int qede_dev_configure(struct rte_eth_dev *eth_dev) if (qede_check_fdir_support(eth_dev)) return -ENOTSUP; - /* Deallocate resources if held previously. It is needed only if the - * queue count has been changed from previous configuration. If its - * going to change then it means RX/TX queue setup will be called - * again and the fastpath pointers will be reinitialized there. - */ - if (qdev->num_tx_queues != eth_dev->data->nb_tx_queues || - qdev->num_rx_queues != eth_dev->data->nb_rx_queues) { - qede_dealloc_fp_resc(eth_dev); - /* Proceed with updated queue count */ - qdev->num_tx_queues = eth_dev->data->nb_tx_queues; - qdev->num_rx_queues = eth_dev->data->nb_rx_queues; - if (qede_alloc_fp_resc(qdev)) - return -ENOMEM; - } + qede_dealloc_fp_resc(eth_dev); + qdev->num_tx_queues = eth_dev->data->nb_tx_queues; + qdev->num_rx_queues = eth_dev->data->nb_rx_queues; + if (qede_alloc_fp_resc(qdev)) + return -ENOMEM; /* If jumbo enabled adjust MTU */ if (eth_dev->data->dev_conf.rxmode.jumbo_frame) @@ -1411,24 +1385,13 @@ static int qede_dev_configure(struct rte_eth_dev *eth_dev) eth_dev->data->dev_conf.rxmode.max_rx_pkt_len - ETHER_HDR_LEN - ETHER_CRC_LEN; - /* VF's MTU has to be set using vport-start where as - * PF's MTU can be updated via vport-update. - */ - if (IS_VF(edev)) { - if (qede_start_vport(qdev, eth_dev->data->mtu)) - return -1; - } else { - if (qede_update_mtu(eth_dev, eth_dev->data->mtu)) - return -1; - } - + if (qede_start_vport(qdev, eth_dev->data->mtu)) + return -1; qdev->mtu = eth_dev->data->mtu; - qdev->new_mtu = qdev->mtu; /* Enable VLAN offloads by default */ ret = qede_vlan_offload_set(eth_dev, ETH_VLAN_STRIP_MASK | - ETH_VLAN_FILTER_MASK | - ETH_VLAN_EXTEND_MASK); + ETH_VLAN_FILTER_MASK); if (ret) return ret; @@ -1526,7 +1489,7 @@ qede_link_update(struct rte_eth_dev *eth_dev, __rte_unused int wait_to_complete) { struct qede_dev *qdev = eth_dev->data->dev_private; struct ecore_dev *edev = &qdev->edev; - uint16_t link_duplex; + uint16_t link_duplex, old_link_status; struct qed_link_output link; struct rte_eth_link *curr = ð_dev->data->dev_link; @@ -1551,6 +1514,7 @@ qede_link_update(struct rte_eth_dev *eth_dev, __rte_unused int wait_to_complete) curr->link_duplex = link_duplex; /* Link Status */ + old_link_status = curr->link_status; curr->link_status = (link.link_up) ? ETH_LINK_UP : ETH_LINK_DOWN; /* AN */ @@ -1562,7 +1526,7 @@ qede_link_update(struct rte_eth_dev *eth_dev, __rte_unused int wait_to_complete) curr->link_autoneg, curr->link_status); /* return 0 means link status changed, -1 means not changed */ - return ((curr->link_status == link.link_up) ? -1 : 0); + return ((curr->link_status == old_link_status) ? -1 : 0); } static void qede_promiscuous_enable(struct rte_eth_dev *eth_dev) @@ -1636,17 +1600,32 @@ static void qede_dev_close(struct rte_eth_dev *eth_dev) qede_dev_stop(eth_dev); qede_stop_vport(edev); + qdev->vport_started = false; qede_fdir_dealloc_resc(eth_dev); qede_dealloc_fp_resc(eth_dev); eth_dev->data->nb_rx_queues = 0; eth_dev->data->nb_tx_queues = 0; + /* Bring the link down */ + qede_dev_set_link_state(eth_dev, false); qdev->ops->common->slowpath_stop(edev); qdev->ops->common->remove(edev); rte_intr_disable(&pci_dev->intr_handle); - rte_intr_callback_unregister(&pci_dev->intr_handle, - qede_interrupt_handler, (void *)eth_dev); + + switch (pci_dev->intr_handle.type) { + case RTE_INTR_HANDLE_UIO_INTX: + case RTE_INTR_HANDLE_VFIO_LEGACY: + rte_intr_callback_unregister(&pci_dev->intr_handle, + qede_interrupt_handler_intx, + (void *)eth_dev); + break; + default: + rte_intr_callback_unregister(&pci_dev->intr_handle, + qede_interrupt_handler, + (void *)eth_dev); + } + if (ECORE_IS_CMT(edev)) rte_eal_alarm_cancel(qede_poll_sp_sb_cb, (void *)eth_dev); } @@ -1935,6 +1914,99 @@ static void qede_allmulticast_disable(struct rte_eth_dev *eth_dev) QED_FILTER_RX_MODE_TYPE_REGULAR); } +static int +qede_set_mc_addr_list(struct rte_eth_dev *eth_dev, struct ether_addr *mc_addrs, + uint32_t mc_addrs_num) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + uint8_t i; + + if (mc_addrs_num > ECORE_MAX_MC_ADDRS) { + DP_ERR(edev, "Reached max multicast filters limit," + "Please enable multicast promisc mode\n"); + return -ENOSPC; + } + + for (i = 0; i < mc_addrs_num; i++) { + if (!is_multicast_ether_addr(&mc_addrs[i])) { + DP_ERR(edev, "Not a valid multicast MAC\n"); + return -EINVAL; + } + } + + /* Flush all existing entries */ + if (qede_del_mcast_filters(eth_dev)) + return -1; + + /* Set new mcast list */ + return qede_add_mcast_filters(eth_dev, mc_addrs, mc_addrs_num); +} + +/* Update MTU via vport-update without doing port restart. + * The vport must be deactivated before calling this API. + */ +int qede_update_mtu(struct rte_eth_dev *eth_dev, uint16_t mtu) +{ + struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + struct ecore_hwfn *p_hwfn; + int rc; + int i; + + if (IS_PF(edev)) { + struct ecore_sp_vport_update_params params; + + memset(¶ms, 0, sizeof(struct ecore_sp_vport_update_params)); + params.vport_id = 0; + params.mtu = mtu; + params.vport_id = 0; + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + params.opaque_fid = p_hwfn->hw_info.opaque_fid; + rc = ecore_sp_vport_update(p_hwfn, ¶ms, + ECORE_SPQ_MODE_EBLOCK, NULL); + if (rc != ECORE_SUCCESS) + goto err; + } + } else { + for_each_hwfn(edev, i) { + p_hwfn = &edev->hwfns[i]; + rc = ecore_vf_pf_update_mtu(p_hwfn, mtu); + if (rc == ECORE_INVAL) { + DP_INFO(edev, "VF MTU Update TLV not supported\n"); + /* Recreate vport */ + rc = qede_start_vport(qdev, mtu); + if (rc != ECORE_SUCCESS) + goto err; + + /* Restore config lost due to vport stop */ + if (eth_dev->data->promiscuous) + qede_promiscuous_enable(eth_dev); + else + qede_promiscuous_disable(eth_dev); + + if (eth_dev->data->all_multicast) + qede_allmulticast_enable(eth_dev); + else + qede_allmulticast_disable(eth_dev); + + qede_vlan_offload_set(eth_dev, + qdev->vlan_offload_mask); + } else if (rc != ECORE_SUCCESS) { + goto err; + } + } + } + DP_INFO(edev, "%s MTU updated to %u\n", IS_PF(edev) ? "PF" : "VF", mtu); + + return 0; + +err: + DP_ERR(edev, "Failed to update MTU\n"); + return -1; +} + static int qede_flow_ctrl_set(struct rte_eth_dev *eth_dev, struct rte_eth_fc_conf *fc_conf) { @@ -2087,7 +2159,7 @@ int qede_rss_hash_update(struct rte_eth_dev *eth_dev, vport_update_params.vport_id = 0; /* pass the L2 handles instead of qids */ for (i = 0 ; i < ECORE_RSS_IND_TABLE_SIZE ; i++) { - idx = qdev->rss_ind_table[i]; + idx = i % QEDE_RSS_COUNT(qdev); rss_params.rss_ind_table[i] = qdev->fp_array[idx].rxq->handle; } vport_update_params.rss_params = &rss_params; @@ -2311,8 +2383,6 @@ static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) int i; PMD_INIT_FUNC_TRACE(edev); - if (IS_VF(edev)) - return -ENOTSUP; qede_dev_info_get(dev, &dev_info); max_rx_pkt_len = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN; frame_size = max_rx_pkt_len + QEDE_ETH_OVERHEAD; @@ -2339,7 +2409,8 @@ static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) restart = true; } rte_delay_ms(1000); - qdev->new_mtu = mtu; + qdev->mtu = mtu; + /* Fix up RX buf size for all queues of the port */ for_each_rss(i) { fp = &qdev->fp_array[i]; @@ -2353,17 +2424,19 @@ static int qede_set_mtu(struct rte_eth_dev *dev, uint16_t mtu) rx_buf_size = frame_size; rx_buf_size = QEDE_CEIL_TO_CACHE_LINE_SIZE(rx_buf_size); fp->rxq->rx_buf_size = rx_buf_size; - DP_INFO(edev, "buf_size adjusted to %u\n", rx_buf_size); + DP_INFO(edev, "RX buffer size %u\n", rx_buf_size); } } if (max_rx_pkt_len > ETHER_MAX_LEN) dev->data->dev_conf.rxmode.jumbo_frame = 1; else dev->data->dev_conf.rxmode.jumbo_frame = 0; + if (!dev->data->dev_started && restart) { qede_dev_start(dev); dev->data->dev_started = 1; } + /* update max frame size */ dev->data->dev_conf.rxmode.max_rx_pkt_len = max_rx_pkt_len; /* Reassign back */ @@ -2715,6 +2788,7 @@ static const struct eth_dev_ops qede_eth_dev_ops = { .promiscuous_disable = qede_promiscuous_disable, .allmulticast_enable = qede_allmulticast_enable, .allmulticast_disable = qede_allmulticast_disable, + .set_mc_addr_list = qede_set_mc_addr_list, .dev_stop = qede_dev_stop, .dev_close = qede_dev_close, .stats_get = qede_get_stats, @@ -2755,6 +2829,7 @@ static const struct eth_dev_ops qede_eth_vf_dev_ops = { .promiscuous_disable = qede_promiscuous_disable, .allmulticast_enable = qede_allmulticast_enable, .allmulticast_disable = qede_allmulticast_disable, + .set_mc_addr_list = qede_set_mc_addr_list, .dev_stop = qede_dev_stop, .dev_close = qede_dev_close, .stats_get = qede_get_stats, @@ -2772,6 +2847,9 @@ static const struct eth_dev_ops qede_eth_vf_dev_ops = { .mtu_set = qede_set_mtu, .udp_tunnel_port_add = qede_udp_dst_port_add, .udp_tunnel_port_del = qede_udp_dst_port_del, + .mac_addr_add = qede_mac_addr_add, + .mac_addr_remove = qede_mac_addr_remove, + .mac_addr_set = qede_mac_addr_set, }; static void qede_update_pf_params(struct ecore_dev *edev) @@ -2800,6 +2878,7 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) /* Fix up ecore debug level */ uint32_t dp_module = ~0 & ~ECORE_MSG_HW; uint8_t dp_level = ECORE_LEVEL_VERBOSE; + uint32_t int_mode; int rc; /* Extract key data structures */ @@ -2844,8 +2923,22 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) return -ENODEV; } qede_update_pf_params(edev); - rte_intr_callback_register(&pci_dev->intr_handle, - qede_interrupt_handler, (void *)eth_dev); + + switch (pci_dev->intr_handle.type) { + case RTE_INTR_HANDLE_UIO_INTX: + case RTE_INTR_HANDLE_VFIO_LEGACY: + int_mode = ECORE_INT_MODE_INTA; + rte_intr_callback_register(&pci_dev->intr_handle, + qede_interrupt_handler_intx, + (void *)eth_dev); + break; + default: + int_mode = ECORE_INT_MODE_MSIX; + rte_intr_callback_register(&pci_dev->intr_handle, + qede_interrupt_handler, + (void *)eth_dev); + } + if (rte_intr_enable(&pci_dev->intr_handle)) { DP_ERR(edev, "rte_intr_enable() failed\n"); return -ENODEV; @@ -2853,7 +2946,8 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) /* Start the Slowpath-process */ memset(¶ms, 0, sizeof(struct qed_slowpath_params)); - params.int_mode = ECORE_INT_MODE_MSIX; + + params.int_mode = int_mode; params.drv_major = QEDE_PMD_VERSION_MAJOR; params.drv_minor = QEDE_PMD_VERSION_MINOR; params.drv_rev = QEDE_PMD_VERSION_REVISION; @@ -2936,7 +3030,7 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) ECORE_LEADING_HWFN(edev), vf_mac, &is_mac_forced); - if (is_mac_exist && is_mac_forced) { + if (is_mac_exist) { DP_INFO(edev, "VF macaddr received from PF\n"); ether_addr_copy((struct ether_addr *)&vf_mac, ð_dev->data->mac_addrs[0]); @@ -2957,16 +3051,24 @@ static int qede_common_dev_init(struct rte_eth_dev *eth_dev, bool is_vf) do_once = false; } + /* Bring-up the link */ + qede_dev_set_link_state(eth_dev, true); + adapter->num_tx_queues = 0; adapter->num_rx_queues = 0; SLIST_INIT(&adapter->fdir_info.fdir_list_head); SLIST_INIT(&adapter->vlan_list_head); SLIST_INIT(&adapter->uc_list_head); + SLIST_INIT(&adapter->mc_list_head); adapter->mtu = ETHER_MTU; - adapter->new_mtu = ETHER_MTU; - if (!is_vf) - if (qede_start_vport(adapter, adapter->mtu)) - return -1; + adapter->vport_started = false; + + /* VF tunnel offloads is enabled by default in PF driver */ + adapter->vxlan.enable = true; + adapter->vxlan.num_filters = 0; + adapter->vxlan.filter_type = ETH_TUNNEL_FILTER_IMAC | + ETH_TUNNEL_FILTER_IVLAN; + adapter->vxlan.udp_port = QEDE_VXLAN_DEF_PORT; DP_INFO(edev, "MAC address : %02x:%02x:%02x:%02x:%02x:%02x\n", adapter->primary_mac.addr_bytes[0], diff --git a/dpdk/drivers/net/qede/qede_ethdev.h b/dpdk/drivers/net/qede/qede_ethdev.h index 8f21b338..cc1a409f 100644 --- a/dpdk/drivers/net/qede/qede_ethdev.h +++ b/dpdk/drivers/net/qede/qede_ethdev.h @@ -34,6 +34,7 @@ #include "base/nvm_cfg.h" #include "base/ecore_sp_commands.h" #include "base/ecore_l2.h" +#include "base/ecore_vf.h" #include "qede_logs.h" #include "qede_if.h" @@ -184,7 +185,6 @@ struct qede_dev { struct ecore_sb_info *sb_array; struct qede_fastpath *fp_array; uint16_t mtu; - uint16_t new_mtu; bool enable_tx_switching; bool rss_enable; struct rte_eth_rss_conf rss_conf; @@ -207,6 +207,8 @@ struct qede_dev { struct qede_fdir_info fdir_info; bool vlan_strip_flg; char drv_ver[QEDE_PMD_DRV_VER_STR_SIZE]; + bool vport_started; + int vlan_offload_mask; void *ethdev; }; diff --git a/dpdk/drivers/net/qede/qede_fdir.c b/dpdk/drivers/net/qede/qede_fdir.c index da6364ee..05152566 100644 --- a/dpdk/drivers/net/qede/qede_fdir.c +++ b/dpdk/drivers/net/qede/qede_fdir.c @@ -141,8 +141,8 @@ qede_config_cmn_fdir_filter(struct rte_eth_dev *eth_dev, if (add) { SLIST_FOREACH(tmp, &qdev->fdir_info.fdir_list_head, list) { if (memcmp(tmp->mz->addr, pkt, pkt_len) == 0) { - DP_ERR(edev, "flowdir filter exist\n"); - rc = -EEXIST; + DP_INFO(edev, "flowdir filter exist\n"); + rc = 0; goto err2; } } @@ -465,5 +465,8 @@ int qede_ntuple_filter_conf(struct rte_eth_dev *eth_dev, udpv4_flow->src_port = ntuple->src_port; udpv4_flow->dst_port = ntuple->dst_port; } + + fdir_entry.action.rx_queue = ntuple->queue; + return qede_config_cmn_fdir_filter(eth_dev, &fdir_entry, add); } diff --git a/dpdk/drivers/net/qede/qede_main.c b/dpdk/drivers/net/qede/qede_main.c index ae187321..95b4cd91 100644 --- a/dpdk/drivers/net/qede/qede_main.c +++ b/dpdk/drivers/net/qede/qede_main.c @@ -279,7 +279,7 @@ static int qed_slowpath_start(struct ecore_dev *edev, /* Start the slowpath */ memset(&hw_init_params, 0, sizeof(hw_init_params)); hw_init_params.b_hw_start = true; - hw_init_params.int_mode = ECORE_INT_MODE_MSIX; + hw_init_params.int_mode = params->int_mode; hw_init_params.allow_npar_tx_switch = true; hw_init_params.bin_fw_data = data; @@ -633,8 +633,11 @@ void qed_link_update(struct ecore_hwfn *hwfn) { struct ecore_dev *edev = hwfn->p_dev; struct qede_dev *qdev = (struct qede_dev *)edev; + struct rte_eth_dev *dev = (struct rte_eth_dev *)qdev->ethdev; - qede_link_update((struct rte_eth_dev *)qdev->ethdev, 0); + if (!qede_link_update(dev, 0)) + _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, + NULL, NULL); } static int qed_drain(struct ecore_dev *edev) diff --git a/dpdk/drivers/net/qede/qede_rxtx.c b/dpdk/drivers/net/qede/qede_rxtx.c index 31132ceb..ffe196a6 100644 --- a/dpdk/drivers/net/qede/qede_rxtx.c +++ b/dpdk/drivers/net/qede/qede_rxtx.c @@ -192,9 +192,15 @@ static void qede_rx_queue_release_mbufs(struct qede_rx_queue *rxq) void qede_rx_queue_release(void *rx_queue) { struct qede_rx_queue *rxq = rx_queue; + struct qede_dev *qdev = rxq->qdev; + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + + PMD_INIT_FUNC_TRACE(edev); if (rxq) { qede_rx_queue_release_mbufs(rxq); + qdev->ops->common->chain_free(edev, &rxq->rx_bd_ring); + qdev->ops->common->chain_free(edev, &rxq->rx_comp_ring); rte_free(rxq->sw_rx_ring); rte_free(rxq); } @@ -350,9 +356,14 @@ static void qede_tx_queue_release_mbufs(struct qede_tx_queue *txq) void qede_tx_queue_release(void *tx_queue) { struct qede_tx_queue *txq = tx_queue; + struct qede_dev *qdev = txq->qdev; + struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); + + PMD_INIT_FUNC_TRACE(edev); if (txq) { qede_tx_queue_release_mbufs(txq); + qdev->ops->common->chain_free(edev, &txq->tx_pbl); rte_free(txq->sw_tx_ring); rte_free(txq); } @@ -417,6 +428,8 @@ int qede_alloc_fp_resc(struct qede_dev *qdev) for (sb_idx = 0; sb_idx < QEDE_RXTX_MAX(qdev); sb_idx++) { fp = &qdev->fp_array[sb_idx]; + if (!fp) + continue; fp->sb_info = rte_calloc("sb", 1, sizeof(struct ecore_sb_info), RTE_CACHE_LINE_SIZE); if (!fp->sb_info) { @@ -439,8 +452,6 @@ void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev) struct qede_dev *qdev = QEDE_INIT_QDEV(eth_dev); struct ecore_dev *edev = QEDE_INIT_EDEV(qdev); struct qede_fastpath *fp; - struct qede_rx_queue *rxq; - struct qede_tx_queue *txq; uint16_t sb_idx; uint8_t i; @@ -448,6 +459,8 @@ void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev) for (sb_idx = 0; sb_idx < QEDE_RXTX_MAX(qdev); sb_idx++) { fp = &qdev->fp_array[sb_idx]; + if (!fp) + continue; DP_INFO(edev, "Free sb_info index 0x%x\n", fp->sb_info->igu_sb_id); if (fp->sb_info) { @@ -463,21 +476,13 @@ void qede_dealloc_fp_resc(struct rte_eth_dev *eth_dev) for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { if (eth_dev->data->rx_queues[i]) { qede_rx_queue_release(eth_dev->data->rx_queues[i]); - rxq = eth_dev->data->rx_queues[i]; - qdev->ops->common->chain_free(edev, - &rxq->rx_bd_ring); - qdev->ops->common->chain_free(edev, - &rxq->rx_comp_ring); eth_dev->data->rx_queues[i] = NULL; } } for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { if (eth_dev->data->tx_queues[i]) { - txq = eth_dev->data->tx_queues[i]; qede_tx_queue_release(eth_dev->data->tx_queues[i]); - qdev->ops->common->chain_free(edev, - &txq->tx_pbl); eth_dev->data->tx_queues[i] = NULL; } } @@ -1462,6 +1467,8 @@ qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) */ rx_mb->data_off = offset + RTE_PKTMBUF_HEADROOM; packet_type |= qede_rx_cqe_to_pkt_type_outer(rx_mb); + } else { + packet_type |= qede_rx_cqe_to_pkt_type(parse_flag); } /* Common handling for non-tunnel packets and for inner @@ -1483,7 +1490,6 @@ qede_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) ol_flags |= PKT_RX_IP_CKSUM_BAD; } else { ol_flags |= PKT_RX_IP_CKSUM_GOOD; - packet_type |= qede_rx_cqe_to_pkt_type(parse_flag); } if (CQE_HAS_VLAN(parse_flag) || @@ -1627,6 +1633,7 @@ qede_encode_sg_bd(struct qede_tx_queue *p_txq, struct rte_mbuf *m_seg, QEDE_BD_SET_ADDR_LEN(tx_bd, mapping, m_seg->data_len); PMD_TX_LOG(DEBUG, txq, "BD len %04x", m_seg->data_len); } + start_seg++; m_seg = m_seg->next; } diff --git a/dpdk/drivers/net/ring/rte_eth_ring.c b/dpdk/drivers/net/ring/rte_eth_ring.c index 8583a673..dbd350e1 100644 --- a/dpdk/drivers/net/ring/rte_eth_ring.c +++ b/dpdk/drivers/net/ring/rte_eth_ring.c @@ -89,7 +89,7 @@ static struct rte_eth_link pmd_link = { .link_speed = ETH_SPEED_NUM_10G, .link_duplex = ETH_LINK_FULL_DUPLEX, .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_AUTONEG + .link_autoneg = ETH_LINK_FIXED, }; static uint16_t diff --git a/dpdk/drivers/net/sfc/base/efx_port.c b/dpdk/drivers/net/sfc/base/efx_port.c index 518c2a22..ec8a1575 100644 --- a/dpdk/drivers/net/sfc/base/efx_port.c +++ b/dpdk/drivers/net/sfc/base/efx_port.c @@ -145,7 +145,7 @@ efx_port_loopback_set( EFSYS_ASSERT(link_mode < EFX_LINK_NMODES); if (EFX_TEST_QWORD_BIT(encp->enc_loopback_types[link_mode], - loopback_type) == 0) { + (int)loopback_type) == 0) { rc = ENOTSUP; goto fail1; } diff --git a/dpdk/drivers/net/sfc/efsys.h b/dpdk/drivers/net/sfc/efsys.h index f428b624..5b52c222 100644 --- a/dpdk/drivers/net/sfc/efsys.h +++ b/dpdk/drivers/net/sfc/efsys.h @@ -141,6 +141,8 @@ prefetch_read_once(const volatile void *addr) #define __out_ecount_opt(_n) #define __out_bcount(_n) #define __out_bcount_opt(_n) +#define __out_bcount_part(_n, _l) +#define __out_bcount_part_opt(_n, _l) #define __deref_out diff --git a/dpdk/drivers/net/sfc/sfc.c b/dpdk/drivers/net/sfc/sfc.c index 49d7e937..2196c3f1 100644 --- a/dpdk/drivers/net/sfc/sfc.c +++ b/dpdk/drivers/net/sfc/sfc.c @@ -548,6 +548,8 @@ sfc_set_rss_defaults(struct sfc_adapter *sa) fail_hash_support_get: fail_scale_support_get: + efx_rx_fini(sa->nic); + fail_rx_init: efx_ev_fini(sa->nic); diff --git a/dpdk/drivers/net/sfc/sfc_ef10_rx.c b/dpdk/drivers/net/sfc/sfc_ef10_rx.c index 4c76f747..bd2cbd09 100644 --- a/dpdk/drivers/net/sfc/sfc_ef10_rx.c +++ b/dpdk/drivers/net/sfc/sfc_ef10_rx.c @@ -296,6 +296,8 @@ sfc_ef10_rx_ev_to_offloads(struct sfc_ef10_rxq *rxq, const efx_qword_t rx_ev, /* Override Layer 2 packet type */ l2_ptype = RTE_PTYPE_L2_ETHER_ARP; break; + case ESE_DZ_L3_CLASS_UNKNOWN: + break; default: /* Unexpected Layer 3 class */ SFC_ASSERT(false); diff --git a/dpdk/drivers/net/sfc/sfc_ethdev.c b/dpdk/drivers/net/sfc/sfc_ethdev.c index fabcc329..1b160b5e 100644 --- a/dpdk/drivers/net/sfc/sfc_ethdev.c +++ b/dpdk/drivers/net/sfc/sfc_ethdev.c @@ -1030,7 +1030,7 @@ sfc_set_mc_addr_list(struct rte_eth_dev *dev, struct ether_addr *mc_addr_set, if (rc != 0) sfc_err(sa, "cannot set multicast address list (rc = %u)", rc); - SFC_ASSERT(rc > 0); + SFC_ASSERT(rc >= 0); return -rc; } diff --git a/dpdk/drivers/net/sfc/sfc_ev.c b/dpdk/drivers/net/sfc/sfc_ev.c index 5fbebbf1..ba22f7ef 100644 --- a/dpdk/drivers/net/sfc/sfc_ev.c +++ b/dpdk/drivers/net/sfc/sfc_ev.c @@ -859,7 +859,7 @@ static int sfc_kvarg_perf_profile_handler(__rte_unused const char *key, const char *value_str, void *opaque) { - uint64_t *value = opaque; + uint32_t *value = opaque; if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_THROUGHPUT) == 0) *value = EFX_EVQ_FLAGS_TYPE_THROUGHPUT; diff --git a/dpdk/drivers/net/sfc/sfc_flow.c b/dpdk/drivers/net/sfc/sfc_flow.c index e770b98e..90ef5bf2 100644 --- a/dpdk/drivers/net/sfc/sfc_flow.c +++ b/dpdk/drivers/net/sfc/sfc_flow.c @@ -107,7 +107,6 @@ sfc_flow_parse_init(const struct rte_flow_item *item, const uint8_t *spec; const uint8_t *mask; const uint8_t *last; - uint8_t match; uint8_t supp; unsigned int i; @@ -168,12 +167,11 @@ sfc_flow_parse_init(const struct rte_flow_item *item, return -rte_errno; } - /* Check that mask and spec not asks for more match than supp_mask */ + /* Check that mask does not ask for more match than supp_mask */ for (i = 0; i < size; i++) { - match = spec[i] | mask[i]; supp = ((const uint8_t *)supp_mask)[i]; - if ((match | supp) != supp) { + if (~supp & mask[i]) { rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ITEM, item, "Item's field is not supported"); @@ -332,7 +330,8 @@ sfc_flow_parse_vlan(const struct rte_flow_item *item, * the outer tag and the next matches the inner tag. */ if (mask->tci == supp_mask.tci) { - vid = rte_bswap16(spec->tci); + /* Apply mask to keep VID only */ + vid = rte_bswap16(spec->tci & mask->tci); if (!(efx_spec->efs_match_flags & EFX_FILTER_MATCH_OUTER_VID)) { diff --git a/dpdk/drivers/net/sfc/sfc_rx.c b/dpdk/drivers/net/sfc/sfc_rx.c index 7816393b..8fc93d68 100644 --- a/dpdk/drivers/net/sfc/sfc_rx.c +++ b/dpdk/drivers/net/sfc/sfc_rx.c @@ -814,7 +814,7 @@ sfc_rx_mbuf_data_alignment(struct rte_mempool *mb_pool) order = MIN(order, rte_bsf32(data_off)); - return 1u << (order - 1); + return 1u << order; } static uint16_t @@ -1095,6 +1095,41 @@ sfc_efx_to_rte_hash_type(efx_rx_hash_type_t efx_hash_types) #endif #if EFSYS_OPT_RX_SCALE +static int +sfc_rx_process_adv_conf_rss(struct sfc_adapter *sa, + struct rte_eth_rss_conf *conf) +{ + efx_rx_hash_type_t efx_hash_types = sa->rss_hash_types; + + if (sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) { + if ((conf->rss_hf != 0 && conf->rss_hf != SFC_RSS_OFFLOADS) || + conf->rss_key != NULL) + return EINVAL; + } + + if (conf->rss_hf != 0) { + if ((conf->rss_hf & ~SFC_RSS_OFFLOADS) != 0) { + sfc_err(sa, "unsupported hash functions requested"); + return EINVAL; + } + + efx_hash_types = sfc_rte_to_efx_hash_type(conf->rss_hf); + } + + if (conf->rss_key != NULL) { + if (conf->rss_key_len != sizeof(sa->rss_key)) { + sfc_err(sa, "RSS key size is wrong (should be %lu)", + sizeof(sa->rss_key)); + return EINVAL; + } + rte_memcpy(sa->rss_key, conf->rss_key, sizeof(sa->rss_key)); + } + + sa->rss_hash_types = efx_hash_types; + + return 0; +} + static int sfc_rx_rss_config(struct sfc_adapter *sa) { @@ -1347,16 +1382,23 @@ sfc_rx_configure(struct sfc_adapter *sa) MIN(sa->rxq_count, EFX_MAXRSS) : 0; if (sa->rss_channels > 0) { + struct rte_eth_rss_conf *adv_conf_rss; unsigned int sw_index; for (sw_index = 0; sw_index < EFX_RSS_TBL_SIZE; ++sw_index) sa->rss_tbl[sw_index] = sw_index % sa->rss_channels; + + adv_conf_rss = &dev_conf->rx_adv_conf.rss_conf; + rc = sfc_rx_process_adv_conf_rss(sa, adv_conf_rss); + if (rc != 0) + goto fail_rx_process_adv_conf_rss; } #endif done: return 0; +fail_rx_process_adv_conf_rss: fail_rx_qinit_info: fail_rxqs_realloc: fail_rxqs_alloc: diff --git a/dpdk/drivers/net/softnic/rte_eth_softnic.c b/dpdk/drivers/net/softnic/rte_eth_softnic.c index c8f918df..3c695e4d 100644 --- a/dpdk/drivers/net/softnic/rte_eth_softnic.c +++ b/dpdk/drivers/net/softnic/rte_eth_softnic.c @@ -551,7 +551,7 @@ pmd_ethdev_register(struct rte_vdev_device *vdev, soft_dev->data->dev_private = dev_private; soft_dev->data->dev_link.link_speed = hard_speed; soft_dev->data->dev_link.link_duplex = ETH_LINK_FULL_DUPLEX; - soft_dev->data->dev_link.link_autoneg = ETH_LINK_AUTONEG; + soft_dev->data->dev_link.link_autoneg = ETH_LINK_FIXED; soft_dev->data->dev_link.link_status = ETH_LINK_DOWN; soft_dev->data->mac_addrs = ð_addr; soft_dev->data->promiscuous = 1; diff --git a/dpdk/drivers/net/szedata2/rte_eth_szedata2.c b/dpdk/drivers/net/szedata2/rte_eth_szedata2.c index 45aebed3..88f5fb8a 100644 --- a/dpdk/drivers/net/szedata2/rte_eth_szedata2.c +++ b/dpdk/drivers/net/szedata2/rte_eth_szedata2.c @@ -1054,22 +1054,29 @@ eth_stats_get(struct rte_eth_dev *dev, uint64_t tx_err_total = 0; uint64_t rx_total_bytes = 0; uint64_t tx_total_bytes = 0; - const struct pmd_internals *internals = dev->data->dev_private; - for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_rx; i++) { - stats->q_ipackets[i] = internals->rx_queue[i].rx_pkts; - stats->q_ibytes[i] = internals->rx_queue[i].rx_bytes; - rx_total += stats->q_ipackets[i]; - rx_total_bytes += stats->q_ibytes[i]; + for (i = 0; i < nb_rx; i++) { + struct szedata2_rx_queue *rxq = dev->data->rx_queues[i]; + + if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { + stats->q_ipackets[i] = rxq->rx_pkts; + stats->q_ibytes[i] = rxq->rx_bytes; + } + rx_total += rxq->rx_pkts; + rx_total_bytes += rxq->rx_bytes; } - for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_tx; i++) { - stats->q_opackets[i] = internals->tx_queue[i].tx_pkts; - stats->q_obytes[i] = internals->tx_queue[i].tx_bytes; - stats->q_errors[i] = internals->tx_queue[i].err_pkts; - tx_total += stats->q_opackets[i]; - tx_total_bytes += stats->q_obytes[i]; - tx_err_total += stats->q_errors[i]; + for (i = 0; i < nb_tx; i++) { + struct szedata2_tx_queue *txq = dev->data->tx_queues[i]; + + if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) { + stats->q_opackets[i] = txq->tx_pkts; + stats->q_obytes[i] = txq->tx_bytes; + stats->q_errors[i] = txq->err_pkts; + } + tx_total += txq->tx_pkts; + tx_total_bytes += txq->tx_bytes; + tx_err_total += txq->err_pkts; } stats->ipackets = rx_total; @@ -1417,9 +1424,9 @@ get_szedata2_index(const struct rte_pci_addr *pcislot_addr, uint32_t *index) FILE *fd; char pcislot_path[PATH_MAX]; uint32_t domain; - uint32_t bus; - uint32_t devid; - uint32_t function; + uint8_t bus; + uint8_t devid; + uint8_t function; dir = opendir("/sys/class/combo"); if (dir == NULL) @@ -1444,7 +1451,7 @@ get_szedata2_index(const struct rte_pci_addr *pcislot_addr, uint32_t *index) if (fd == NULL) continue; - ret = fscanf(fd, "%4" PRIx16 ":%2" PRIx8 ":%2" PRIx8 ".%" PRIx8, + ret = fscanf(fd, "%8" SCNx32 ":%2" SCNx8 ":%2" SCNx8 ".%" SCNx8, &domain, &bus, &devid, &function); fclose(fd); if (ret != 4) diff --git a/dpdk/drivers/net/tap/rte_eth_tap.c b/dpdk/drivers/net/tap/rte_eth_tap.c index b8187f9b..c38c02ef 100644 --- a/dpdk/drivers/net/tap/rte_eth_tap.c +++ b/dpdk/drivers/net/tap/rte_eth_tap.c @@ -95,7 +95,7 @@ static struct rte_eth_link pmd_link = { .link_speed = ETH_SPEED_NUM_10G, .link_duplex = ETH_LINK_FULL_DUPLEX, .link_status = ETH_LINK_DOWN, - .link_autoneg = ETH_LINK_AUTONEG + .link_autoneg = ETH_LINK_FIXED, }; static void @@ -544,7 +544,9 @@ apply: case SIOCSIFMTU: break; default: - RTE_ASSERT(!"unsupported request type: must not happen"); + RTE_LOG(WARNING, PMD, "%s: ioctl() called with wrong arg\n", + pmd->name); + return -EINVAL; } if (ioctl(pmd->ioctl_sock, request, ifr) < 0) goto error; diff --git a/dpdk/drivers/net/tap/tap_flow.c b/dpdk/drivers/net/tap/tap_flow.c index ffc0b85b..f10bbf60 100644 --- a/dpdk/drivers/net/tap/tap_flow.c +++ b/dpdk/drivers/net/tap/tap_flow.c @@ -464,7 +464,7 @@ tap_flow_create_eth(const struct rte_flow_item *item, void *data) if (!flow) return 0; msg = &flow->msg; - if (!is_zero_ether_addr(&spec->dst)) { + if (!is_zero_ether_addr(&mask->dst)) { nlattr_add(&msg->nh, TCA_FLOWER_KEY_ETH_DST, ETHER_ADDR_LEN, &spec->dst.addr_bytes); nlattr_add(&msg->nh, @@ -570,13 +570,13 @@ tap_flow_create_ipv4(const struct rte_flow_item *item, void *data) info->eth_type = htons(ETH_P_IP); if (!spec) return 0; - if (spec->hdr.dst_addr) { + if (mask->hdr.dst_addr) { nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST, spec->hdr.dst_addr); nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_DST_MASK, mask->hdr.dst_addr); } - if (spec->hdr.src_addr) { + if (mask->hdr.src_addr) { nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC, spec->hdr.src_addr); nlattr_add32(&msg->nh, TCA_FLOWER_KEY_IPV4_SRC_MASK, @@ -626,13 +626,13 @@ tap_flow_create_ipv6(const struct rte_flow_item *item, void *data) info->eth_type = htons(ETH_P_IPV6); if (!spec) return 0; - if (memcmp(spec->hdr.dst_addr, empty_addr, 16)) { + if (memcmp(mask->hdr.dst_addr, empty_addr, 16)) { nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST, sizeof(spec->hdr.dst_addr), &spec->hdr.dst_addr); nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_DST_MASK, sizeof(mask->hdr.dst_addr), &mask->hdr.dst_addr); } - if (memcmp(spec->hdr.src_addr, empty_addr, 16)) { + if (memcmp(mask->hdr.src_addr, empty_addr, 16)) { nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC, sizeof(spec->hdr.src_addr), &spec->hdr.src_addr); nlattr_add(&msg->nh, TCA_FLOWER_KEY_IPV6_SRC_MASK, @@ -680,10 +680,10 @@ tap_flow_create_udp(const struct rte_flow_item *item, void *data) nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_UDP); if (!spec) return 0; - if (spec->hdr.dst_port & mask->hdr.dst_port) + if (mask->hdr.dst_port) nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_DST, spec->hdr.dst_port); - if (spec->hdr.src_port & mask->hdr.src_port) + if (mask->hdr.src_port) nlattr_add16(&msg->nh, TCA_FLOWER_KEY_UDP_SRC, spec->hdr.src_port); return 0; @@ -726,10 +726,10 @@ tap_flow_create_tcp(const struct rte_flow_item *item, void *data) nlattr_add8(&msg->nh, TCA_FLOWER_KEY_IP_PROTO, IPPROTO_TCP); if (!spec) return 0; - if (spec->hdr.dst_port & mask->hdr.dst_port) + if (mask->hdr.dst_port) nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_DST, spec->hdr.dst_port); - if (spec->hdr.src_port & mask->hdr.src_port) + if (mask->hdr.src_port) nlattr_add16(&msg->nh, TCA_FLOWER_KEY_TCP_SRC, spec->hdr.src_port); return 0; diff --git a/dpdk/drivers/net/thunderx/nicvf_ethdev.c b/dpdk/drivers/net/thunderx/nicvf_ethdev.c index c62371cb..3843084b 100644 --- a/dpdk/drivers/net/thunderx/nicvf_ethdev.c +++ b/dpdk/drivers/net/thunderx/nicvf_ethdev.c @@ -897,7 +897,7 @@ nicvf_dev_tx_queue_release(void *sq) static void nicvf_set_tx_function(struct rte_eth_dev *dev) { - struct nicvf_txq *txq; + struct nicvf_txq *txq = NULL; size_t i; bool multiseg = false; @@ -918,6 +918,9 @@ nicvf_set_tx_function(struct rte_eth_dev *dev) dev->tx_pkt_burst = nicvf_xmit_pkts; } + if (!txq) + return; + if (txq->pool_free == nicvf_single_pool_free_xmited_buffers) PMD_DRV_LOG(DEBUG, "Using single-mempool tx free method"); else diff --git a/dpdk/drivers/net/thunderx/nicvf_rxtx.c b/dpdk/drivers/net/thunderx/nicvf_rxtx.c index 9d69cf4e..06cbc463 100644 --- a/dpdk/drivers/net/thunderx/nicvf_rxtx.c +++ b/dpdk/drivers/net/thunderx/nicvf_rxtx.c @@ -190,12 +190,14 @@ nicvf_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) free_desc -= TX_DESC_PER_PKT; } - sq->tail = tail; - sq->xmit_bufs += i; - rte_wmb(); + if (likely(i)) { + sq->tail = tail; + sq->xmit_bufs += i; + rte_wmb(); - /* Inform HW to xmit the packets */ - nicvf_addr_write(sq->sq_door, i * TX_DESC_PER_PKT); + /* Inform HW to xmit the packets */ + nicvf_addr_write(sq->sq_door, i * TX_DESC_PER_PKT); + } return i; } @@ -246,12 +248,14 @@ nicvf_xmit_pkts_multiseg(void *tx_queue, struct rte_mbuf **tx_pkts, } } - sq->tail = tail; - sq->xmit_bufs += used_bufs; - rte_wmb(); + if (likely(used_desc)) { + sq->tail = tail; + sq->xmit_bufs += used_bufs; + rte_wmb(); - /* Inform HW to xmit the packets */ - nicvf_addr_write(sq->sq_door, used_desc); + /* Inform HW to xmit the packets */ + nicvf_addr_write(sq->sq_door, used_desc); + } return i; } diff --git a/dpdk/drivers/net/vhost/rte_eth_vhost.c b/dpdk/drivers/net/vhost/rte_eth_vhost.c index 2536ee4a..6091b322 100644 --- a/dpdk/drivers/net/vhost/rte_eth_vhost.c +++ b/dpdk/drivers/net/vhost/rte_eth_vhost.c @@ -117,6 +117,7 @@ struct pmd_internal { char *dev_name; char *iface_name; uint16_t max_queues; + int vid; rte_atomic32_t started; }; @@ -527,6 +528,9 @@ update_queuing_status(struct rte_eth_dev *dev) unsigned int i; int allow_queuing = 1; + if (!dev->data->rx_queues || !dev->data->tx_queues) + return; + if (rte_atomic32_read(&internal->started) == 0 || rte_atomic32_read(&internal->dev_attached) == 0) allow_queuing = 0; @@ -551,13 +555,36 @@ update_queuing_status(struct rte_eth_dev *dev) } } +static void +queue_setup(struct rte_eth_dev *eth_dev, struct pmd_internal *internal) +{ + struct vhost_queue *vq; + int i; + + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + vq = eth_dev->data->rx_queues[i]; + if (!vq) + continue; + vq->vid = internal->vid; + vq->internal = internal; + vq->port = eth_dev->data->port_id; + } + for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { + vq = eth_dev->data->tx_queues[i]; + if (!vq) + continue; + vq->vid = internal->vid; + vq->internal = internal; + vq->port = eth_dev->data->port_id; + } +} + static int new_device(int vid) { struct rte_eth_dev *eth_dev; struct internal_list *list; struct pmd_internal *internal; - struct vhost_queue *vq; unsigned i; char ifname[PATH_MAX]; #ifdef RTE_LIBRTE_VHOST_NUMA @@ -580,22 +607,11 @@ new_device(int vid) eth_dev->data->numa_node = newnode; #endif - for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { - vq = eth_dev->data->rx_queues[i]; - if (vq == NULL) - continue; - vq->vid = vid; - vq->internal = internal; - vq->port = eth_dev->data->port_id; - } - for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { - vq = eth_dev->data->tx_queues[i]; - if (vq == NULL) - continue; - vq->vid = vid; - vq->internal = internal; - vq->port = eth_dev->data->port_id; - } + internal->vid = vid; + if (rte_atomic32_read(&internal->started) == 1) + queue_setup(eth_dev, internal); + else + RTE_LOG(INFO, PMD, "RX/TX queues not exist yet\n"); for (i = 0; i < rte_vhost_get_vring_num(vid); i++) rte_vhost_enable_guest_notification(vid, i, 0); @@ -640,17 +656,19 @@ destroy_device(int vid) eth_dev->data->dev_link.link_status = ETH_LINK_DOWN; - for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { - vq = eth_dev->data->rx_queues[i]; - if (vq == NULL) - continue; - vq->vid = -1; - } - for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { - vq = eth_dev->data->tx_queues[i]; - if (vq == NULL) - continue; - vq->vid = -1; + if (eth_dev->data->rx_queues && eth_dev->data->tx_queues) { + for (i = 0; i < eth_dev->data->nb_rx_queues; i++) { + vq = eth_dev->data->rx_queues[i]; + if (!vq) + continue; + vq->vid = -1; + } + for (i = 0; i < eth_dev->data->nb_tx_queues; i++) { + vq = eth_dev->data->tx_queues[i]; + if (!vq) + continue; + vq->vid = -1; + } } state = vring_states[eth_dev->data->port_id]; @@ -773,12 +791,13 @@ rte_eth_vhost_get_vid_from_port_id(uint16_t port_id) } static int -eth_dev_start(struct rte_eth_dev *dev) +eth_dev_start(struct rte_eth_dev *eth_dev) { - struct pmd_internal *internal = dev->data->dev_private; + struct pmd_internal *internal = eth_dev->data->dev_private; + queue_setup(eth_dev, internal); rte_atomic32_set(&internal->started, 1); - update_queuing_status(dev); + update_queuing_status(eth_dev); return 0; } @@ -816,10 +835,13 @@ eth_dev_close(struct rte_eth_dev *dev) pthread_mutex_unlock(&internal_list_lock); rte_free(list); - for (i = 0; i < dev->data->nb_rx_queues; i++) - rte_free(dev->data->rx_queues[i]); - for (i = 0; i < dev->data->nb_tx_queues; i++) - rte_free(dev->data->tx_queues[i]); + if (dev->data->rx_queues) + for (i = 0; i < dev->data->nb_rx_queues; i++) + rte_free(dev->data->rx_queues[i]); + + if (dev->data->tx_queues) + for (i = 0; i < dev->data->nb_tx_queues; i++) + rte_free(dev->data->tx_queues[i]); rte_free(dev->data->mac_addrs); free(internal->dev_name); @@ -1086,6 +1108,7 @@ eth_dev_vhost_create(struct rte_vdev_device *dev, char *iface_name, data->nb_rx_queues = queues; data->nb_tx_queues = queues; internal->max_queues = queues; + internal->vid = -1; data->dev_link = pmd_link; data->mac_addrs = eth_addr; data->dev_flags = RTE_ETH_DEV_INTR_LSC; diff --git a/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c b/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c index 93d96499..7669c64d 100644 --- a/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c +++ b/dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c @@ -670,6 +670,8 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) Vmxnet3_TxQueueDesc *tqd = &hw->tqd_start[i]; vmxnet3_tx_queue_t *txq = dev->data->tx_queues[i]; + txq->shared = &hw->tqd_start[i]; + tqd->ctrl.txNumDeferred = 0; tqd->ctrl.txThreshold = 1; tqd->conf.txRingBasePA = txq->cmd_ring.basePA; @@ -690,6 +692,8 @@ vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) Vmxnet3_RxQueueDesc *rqd = &hw->rqd_start[i]; vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; + rxq->shared = &hw->rqd_start[i]; + rqd->conf.rxRingBasePA[0] = rxq->cmd_ring[0].basePA; rqd->conf.rxRingBasePA[1] = rxq->cmd_ring[1].basePA; rqd->conf.compRingBasePA = rxq->comp_ring.basePA; diff --git a/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c b/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c index aa396ab2..940068ff 100644 --- a/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c +++ b/dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c @@ -937,7 +937,7 @@ vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev, txq->queue_id = queue_idx; txq->port_id = dev->data->port_id; - txq->shared = &hw->tqd_start[queue_idx]; + txq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */ txq->hw = hw; txq->qid = queue_idx; txq->stopped = TRUE; @@ -1040,7 +1040,7 @@ vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev, rxq->mp = mp; rxq->queue_id = queue_idx; rxq->port_id = dev->data->port_id; - rxq->shared = &hw->rqd_start[queue_idx]; + rxq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */ rxq->hw = hw; rxq->qid1 = queue_idx; rxq->qid2 = queue_idx + hw->num_rx_queues; diff --git a/dpdk/examples/exception_path/main.c b/dpdk/examples/exception_path/main.c index f8f5bbdf..ba081e33 100644 --- a/dpdk/examples/exception_path/main.c +++ b/dpdk/examples/exception_path/main.c @@ -70,13 +70,21 @@ #include #include +#ifndef APP_MAX_LCORE +#if (RTE_MAX_LCORE > 64) +#define APP_MAX_LCORE 64 +#else +#define APP_MAX_LCORE RTE_MAX_LCORE +#endif +#endif + /* Macros for printing using RTE_LOG */ #define RTE_LOGTYPE_APP RTE_LOGTYPE_USER1 #define FATAL_ERROR(fmt, args...) rte_exit(EXIT_FAILURE, fmt "\n", ##args) #define PRINT_INFO(fmt, args...) RTE_LOG(INFO, APP, fmt "\n", ##args) /* Max ports than can be used (each port is associated with two lcores) */ -#define MAX_PORTS (RTE_MAX_LCORE / 2) +#define MAX_PORTS (APP_MAX_LCORE / 2) /* Max size of a single packet */ #define MAX_PACKET_SZ (2048) @@ -133,7 +141,7 @@ static uint64_t input_cores_mask = 0; static uint64_t output_cores_mask = 0; /* Array storing port_id that is associated with each lcore */ -static uint16_t port_ids[RTE_MAX_LCORE]; +static uint16_t port_ids[APP_MAX_LCORE]; /* Structure type for recording lcore-specific stats */ struct stats { @@ -143,7 +151,7 @@ struct stats { }; /* Array of lcore-specific stats */ -static struct stats lcore_stats[RTE_MAX_LCORE]; +static struct stats lcore_stats[APP_MAX_LCORE]; /* Print out statistics on packets handled */ static void @@ -156,6 +164,9 @@ print_stats(void) " Lcore Port RX TX Dropped on TX\n" "------- ------ ------------ ------------ ---------------\n"); RTE_LCORE_FOREACH(i) { + /* limit ourselves to application supported cores only */ + if (i >= APP_MAX_LCORE) + break; printf("%6u %7u %13"PRIu64" %13"PRIu64" %16"PRIu64"\n", i, (unsigned)port_ids[i], lcore_stats[i].rx, lcore_stats[i].tx, @@ -362,7 +373,9 @@ setup_port_lcore_affinities(void) uint16_t rx_port = 0; /* Setup port_ids[] array, and check masks were ok */ - RTE_LCORE_FOREACH(i) { + for (i = 0; i < APP_MAX_LCORE; i++) { + if (!rte_lcore_is_enabled(i)) + continue; if (input_cores_mask & (1ULL << i)) { /* Skip ports that are not enabled */ while ((ports_mask & (1 << rx_port)) == 0) { diff --git a/dpdk/examples/flow_filtering/main.c b/dpdk/examples/flow_filtering/main.c index 7d739b4a..fe512dae 100644 --- a/dpdk/examples/flow_filtering/main.c +++ b/dpdk/examples/flow_filtering/main.c @@ -149,6 +149,22 @@ init_port(void) /**< CRC stripped by hardware */ .hw_strip_crc = 1, }, + /* + * Initialize fdir_conf of rte_eth_conf. + * Fdir is used in flow filtering for I40e, + * so rte_flow rules involve some fdir + * configurations. In long term it's better + * that drivers don't require any fdir + * configuration for rte_flow, but we need to + * get this workaround so that sample app can + * run on I40e. + */ + .fdir_conf = { + .mode = RTE_FDIR_MODE_PERFECT, + .pballoc = RTE_FDIR_PBALLOC_64K, + .status = RTE_FDIR_REPORT_STATUS, + .drop_queue = 127, + }, }; printf(":: initializing port: %d\n", port_id); diff --git a/dpdk/examples/ip_pipeline/config/diagram-generator.py b/dpdk/examples/ip_pipeline/config/diagram-generator.py old mode 100644 new mode 100755 diff --git a/dpdk/examples/ip_pipeline/config/pipeline-to-core-mapping.py b/dpdk/examples/ip_pipeline/config/pipeline-to-core-mapping.py old mode 100644 new mode 100755 diff --git a/dpdk/examples/ip_pipeline/config_parse_tm.c b/dpdk/examples/ip_pipeline/config_parse_tm.c index e75eed71..1c945c9c 100644 --- a/dpdk/examples/ip_pipeline/config_parse_tm.c +++ b/dpdk/examples/ip_pipeline/config_parse_tm.c @@ -352,7 +352,7 @@ tm_cfgfile_load_sched_subport( char name[CFG_NAME_LEN + 1]; profile = atoi(entries[j].value); - strncpy(name, + memcpy(name, entries[j].name, sizeof(name)); n_tokens = rte_strsplit( diff --git a/dpdk/examples/ipsec-secgw/ipsec-secgw.c b/dpdk/examples/ipsec-secgw/ipsec-secgw.c index b5ec70a1..46af3f05 100644 --- a/dpdk/examples/ipsec-secgw/ipsec-secgw.c +++ b/dpdk/examples/ipsec-secgw/ipsec-secgw.c @@ -302,6 +302,7 @@ prepare_tx_pkt(struct rte_mbuf *pkt, uint16_t port) pkt->l3_len = sizeof(struct ip); pkt->l2_len = ETHER_HDR_LEN; + ip->ip_sum = 0; ethhdr->ether_type = rte_cpu_to_be_16(ETHER_TYPE_IPv4); } else { pkt->ol_flags |= PKT_TX_IPV6; @@ -475,11 +476,13 @@ outbound_sp(struct sp_ctx *sp, struct traffic_type *ip, sa_idx = ip->res[i] & PROTECT_MASK; if (ip->res[i] & DISCARD) rte_pktmbuf_free(m); + else if (ip->res[i] & BYPASS) + ip->pkts[j++] = m; else if (sa_idx < IPSEC_SA_MAX_ENTRIES) { ipsec->res[ipsec->num] = sa_idx; ipsec->pkts[ipsec->num++] = m; - } else /* BYPASS */ - ip->pkts[j++] = m; + } else /* invalid SA idx */ + rte_pktmbuf_free(m); } ip->num = j; } diff --git a/dpdk/examples/ipsec-secgw/parser.c b/dpdk/examples/ipsec-secgw/parser.c index 9d0ea462..e0881b74 100644 --- a/dpdk/examples/ipsec-secgw/parser.c +++ b/dpdk/examples/ipsec-secgw/parser.c @@ -544,7 +544,7 @@ parse_cfg_file(const char *cfg_filename) goto error_exit; } - strncpy(str + strlen(str), oneline, + memcpy(str + strlen(str), oneline, strlen(oneline)); continue; diff --git a/dpdk/examples/l2fwd-crypto/main.c b/dpdk/examples/l2fwd-crypto/main.c index d4e1682c..77af7e6a 100644 --- a/dpdk/examples/l2fwd-crypto/main.c +++ b/dpdk/examples/l2fwd-crypto/main.c @@ -439,7 +439,7 @@ l2fwd_simple_crypto_enqueue(struct rte_mbuf *m, /* Zero pad data to be crypto'd so it is block aligned */ data_len = rte_pktmbuf_data_len(m) - ipdata_offset; - if (cparams->do_hash && cparams->hash_verify) + if ((cparams->do_hash || cparams->do_aead) && cparams->hash_verify) data_len -= cparams->digest_length; if (cparams->do_cipher) { @@ -1506,8 +1506,8 @@ l2fwd_crypto_default_options(struct l2fwd_crypto_options *options) options->aead_iv_random_size = -1; options->aead_iv.length = 0; - options->auth_xform.aead.algo = RTE_CRYPTO_AEAD_AES_GCM; - options->auth_xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT; + options->aead_xform.aead.algo = RTE_CRYPTO_AEAD_AES_GCM; + options->aead_xform.aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT; options->aad_param = 0; options->aad_random_size = -1; @@ -2081,10 +2081,11 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, options->block_size = cap->sym.aead.block_size; - check_iv_param(&cap->sym.aead.iv_size, + if (check_iv_param(&cap->sym.aead.iv_size, options->aead_iv_param, options->aead_iv_random_size, - &options->aead_iv.length); + &options->aead_iv.length) < 0) + continue; /* * Check if length of provided AEAD key is supported @@ -2098,7 +2099,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, cap->sym.aead.key_size.increment) != 0) { printf("Unsupported aead key length\n"); - return -1; + continue; } /* * Check if length of the aead key to be randomly generated @@ -2111,7 +2112,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, cap->sym.aead.key_size.increment) != 0) { printf("Unsupported aead key length\n"); - return -1; + continue; } options->aead_xform.aead.key.length = options->aead_key_random_size; @@ -2136,7 +2137,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, cap->sym.aead.aad_size.increment) != 0) { printf("Unsupported AAD length\n"); - return -1; + continue; } /* * Check if length of AAD to be randomly generated @@ -2149,7 +2150,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, cap->sym.aead.aad_size.increment) != 0) { printf("Unsupported AAD length\n"); - return -1; + continue; } options->aad.length = options->aad_random_size; /* No size provided, use minimum size. */ @@ -2167,7 +2168,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, cap->sym.aead.digest_size.increment) != 0) { printf("Unsupported digest length\n"); - return -1; + continue; } options->aead_xform.aead.digest_length = options->digest_size; @@ -2189,10 +2190,11 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, options->block_size = cap->sym.cipher.block_size; - check_iv_param(&cap->sym.cipher.iv_size, + if (check_iv_param(&cap->sym.cipher.iv_size, options->cipher_iv_param, options->cipher_iv_random_size, - &options->cipher_iv.length); + &options->cipher_iv.length) < 0) + continue; /* * Check if length of provided cipher key is supported @@ -2206,7 +2208,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, cap->sym.cipher.key_size.increment) != 0) { printf("Unsupported cipher key length\n"); - return -1; + continue; } /* * Check if length of the cipher key to be randomly generated @@ -2219,7 +2221,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, cap->sym.cipher.key_size.increment) != 0) { printf("Unsupported cipher key length\n"); - return -1; + continue; } options->cipher_xform.cipher.key.length = options->ckey_random_size; @@ -2245,10 +2247,11 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, if (cap == NULL) continue; - check_iv_param(&cap->sym.auth.iv_size, + if (check_iv_param(&cap->sym.auth.iv_size, options->auth_iv_param, options->auth_iv_random_size, - &options->auth_iv.length); + &options->auth_iv.length) < 0) + continue; /* * Check if length of provided auth key is supported * by the algorithm chosen. @@ -2261,7 +2264,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, cap->sym.auth.key_size.increment) != 0) { printf("Unsupported auth key length\n"); - return -1; + continue; } /* * Check if length of the auth key to be randomly generated @@ -2274,7 +2277,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, cap->sym.auth.key_size.increment) != 0) { printf("Unsupported auth key length\n"); - return -1; + continue; } options->auth_xform.auth.key.length = options->akey_random_size; @@ -2296,7 +2299,7 @@ initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports, cap->sym.auth.digest_size.increment) != 0) { printf("Unsupported digest length\n"); - return -1; + continue; } options->auth_xform.auth.digest_length = options->digest_size; diff --git a/dpdk/examples/l3fwd/l3fwd_em.c b/dpdk/examples/l3fwd/l3fwd_em.c index 2b7c173b..cc7a8354 100644 --- a/dpdk/examples/l3fwd/l3fwd_em.c +++ b/dpdk/examples/l3fwd/l3fwd_em.c @@ -47,7 +47,6 @@ #include #include #include -#include #include #include #include diff --git a/dpdk/examples/l3fwd/l3fwd_lpm.c b/dpdk/examples/l3fwd/l3fwd_lpm.c index 2d0e1724..c185e5e1 100644 --- a/dpdk/examples/l3fwd/l3fwd_lpm.c +++ b/dpdk/examples/l3fwd/l3fwd_lpm.c @@ -46,7 +46,6 @@ #include #include #include -#include #include #include #include diff --git a/dpdk/examples/performance-thread/common/lthread.c b/dpdk/examples/performance-thread/common/lthread.c index 7d76c8c4..0b60a42a 100644 --- a/dpdk/examples/performance-thread/common/lthread.c +++ b/dpdk/examples/performance-thread/common/lthread.c @@ -320,13 +320,14 @@ struct lthread *lthread_current(void) /* * Tasklet to cancel a thread */ -static void +static void * _cancel(void *arg) { struct lthread *lt = (struct lthread *) arg; lt->state |= BIT(ST_LT_CANCELLED); lthread_detach(); + return NULL; } diff --git a/dpdk/examples/performance-thread/common/lthread_api.h b/dpdk/examples/performance-thread/common/lthread_api.h index ff245a08..a74680bf 100644 --- a/dpdk/examples/performance-thread/common/lthread_api.h +++ b/dpdk/examples/performance-thread/common/lthread_api.h @@ -143,7 +143,7 @@ struct lthread_mutex; struct lthread_condattr; struct lthread_mutexattr; -typedef void (*lthread_func_t) (void *); +typedef void *(*lthread_func_t) (void *); /* * Define the size of stack for an lthread diff --git a/dpdk/examples/performance-thread/l3fwd-thread/main.c b/dpdk/examples/performance-thread/l3fwd-thread/main.c index fa65234f..2ac1aabc 100644 --- a/dpdk/examples/performance-thread/l3fwd-thread/main.c +++ b/dpdk/examples/performance-thread/l3fwd-thread/main.c @@ -2021,17 +2021,18 @@ cpu_load_collector(__rte_unused void *arg) { * * This loop is used to start empty scheduler on lcore. */ -static void +static void * lthread_null(__rte_unused void *args) { int lcore_id = rte_lcore_id(); RTE_LOG(INFO, L3FWD, "Starting scheduler on lcore %d.\n", lcore_id); lthread_exit(NULL); + return NULL; } /* main processing loop */ -static void +static void * lthread_tx_per_ring(void *dummy) { int nb_rx; @@ -2076,6 +2077,7 @@ lthread_tx_per_ring(void *dummy) lthread_cond_wait(ready, 0); } + return NULL; } /* @@ -2084,7 +2086,7 @@ lthread_tx_per_ring(void *dummy) * This lthread is used to spawn one new lthread per ring from producers. * */ -static void +static void * lthread_tx(void *args) { struct lthread *lt; @@ -2129,9 +2131,10 @@ lthread_tx(void *args) } } + return NULL; } -static void +static void * lthread_rx(void *dummy) { int ret; @@ -2155,7 +2158,7 @@ lthread_rx(void *dummy) if (rx_conf->n_rx_queue == 0) { RTE_LOG(INFO, L3FWD, "lcore %u has nothing to do\n", rte_lcore_id()); - return; + return NULL; } RTE_LOG(INFO, L3FWD, "Entering main Rx loop on lcore %u\n", rte_lcore_id()); @@ -2227,6 +2230,7 @@ lthread_rx(void *dummy) lthread_yield(); } } + return NULL; } /* @@ -2235,8 +2239,9 @@ lthread_rx(void *dummy) * This lthread loop spawns all rx and tx lthreads on master lcore */ -static void -lthread_spawner(__rte_unused void *arg) { +static void * +lthread_spawner(__rte_unused void *arg) +{ struct lthread *lt[MAX_THREAD]; int i; int n_thread = 0; @@ -2277,6 +2282,7 @@ lthread_spawner(__rte_unused void *arg) { for (i = 0; i < n_thread; i++) lthread_join(lt[i], NULL); + return NULL; } /* diff --git a/dpdk/examples/performance-thread/l3fwd-thread/test.sh b/dpdk/examples/performance-thread/l3fwd-thread/test.sh old mode 100644 new mode 100755 diff --git a/dpdk/examples/performance-thread/pthread_shim/main.c b/dpdk/examples/performance-thread/pthread_shim/main.c index febae39b..5811cff8 100644 --- a/dpdk/examples/performance-thread/pthread_shim/main.c +++ b/dpdk/examples/performance-thread/pthread_shim/main.c @@ -149,8 +149,7 @@ void *helloworld_pthread(void *arg) */ __thread pthread_t tid[HELLOW_WORLD_MAX_LTHREADS]; -static void initial_lthread(void *args); -static void initial_lthread(void *args __attribute__((unused))) +static void *initial_lthread(void *args __attribute__((unused))) { int lcore = (int) rte_lcore_id(); /* @@ -225,6 +224,7 @@ static void initial_lthread(void *args __attribute__((unused))) /* shutdown the lthread scheduler */ lthread_scheduler_shutdown(rte_lcore_id()); lthread_detach(); + return NULL; } @@ -235,8 +235,6 @@ static void initial_lthread(void *args __attribute__((unused))) * in the core mask */ static int -lthread_scheduler(void *args); -static int lthread_scheduler(void *args __attribute__((unused))) { /* create initial thread */ diff --git a/dpdk/examples/performance-thread/pthread_shim/pthread_shim.c b/dpdk/examples/performance-thread/pthread_shim/pthread_shim.c index bc7cf2b0..24cc3896 100644 --- a/dpdk/examples/performance-thread/pthread_shim/pthread_shim.c +++ b/dpdk/examples/performance-thread/pthread_shim/pthread_shim.c @@ -394,7 +394,7 @@ int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex) int pthread_create(pthread_t *__restrict tid, const pthread_attr_t *__restrict attr, - void *(func) (void *), + lthread_func_t func, void *__restrict arg) { if (override) { @@ -419,7 +419,7 @@ pthread_create(pthread_t *__restrict tid, } } return lthread_create((struct lthread **)tid, lcore, - (void (*)(void *))func, arg); + func, arg); } return _sys_pthread_funcs.f_pthread_create(tid, attr, func, arg); } diff --git a/dpdk/examples/quota_watermark/qw/main.c b/dpdk/examples/quota_watermark/qw/main.c index fe174526..a9a89b98 100644 --- a/dpdk/examples/quota_watermark/qw/main.c +++ b/dpdk/examples/quota_watermark/qw/main.c @@ -210,7 +210,7 @@ receive_stage(__attribute__((unused)) void *args) } } -static void +static int pipeline_stage(__attribute__((unused)) void *args) { int i, ret; @@ -272,9 +272,11 @@ pipeline_stage(__attribute__((unused)) void *args) } } } + + return 0; } -static void +static int send_stage(__attribute__((unused)) void *args) { uint16_t nb_dq_pkts; @@ -316,6 +318,8 @@ send_stage(__attribute__((unused)) void *args) /* TODO: Check if nb_dq_pkts == nb_tx_pkts? */ } } + + return 0; } int @@ -375,15 +379,13 @@ main(int argc, char **argv) if (is_bit_set(port_id, portmask)) init_ring(lcore_id, port_id); - /* typecast is a workaround for GCC 4.3 bug */ - rte_eal_remote_launch((int (*)(void *))pipeline_stage, + rte_eal_remote_launch(pipeline_stage, NULL, lcore_id); } } /* Start send_stage() on the last slave core */ - /* typecast is a workaround for GCC 4.3 bug */ - rte_eal_remote_launch((int (*)(void *))send_stage, NULL, last_lcore_id); + rte_eal_remote_launch(send_stage, NULL, last_lcore_id); /* Start receive_stage() on the master core */ receive_stage(NULL); diff --git a/dpdk/examples/vhost_scsi/scsi.c b/dpdk/examples/vhost_scsi/scsi.c index fd430ec2..47c5c83c 100644 --- a/dpdk/examples/vhost_scsi/scsi.c +++ b/dpdk/examples/vhost_scsi/scsi.c @@ -210,7 +210,7 @@ vhost_bdev_scsi_inquiry_command(struct vhost_block_dev *bdev, break; case SPC_VPD_UNIT_SERIAL_NUMBER: hlen = 4; - strncpy((char *)vpage->params, bdev->name, 32); + memcpy((char *)vpage->params, bdev->name, 32); vpage->alloc_len = rte_cpu_to_be_16(32); break; case SPC_VPD_DEVICE_IDENTIFICATION: @@ -247,7 +247,7 @@ vhost_bdev_scsi_inquiry_command(struct vhost_block_dev *bdev, strncpy((char *)desig->desig, "INTEL", 8); vhost_strcpy_pad((char *)&desig->desig[8], bdev->product_name, 16, ' '); - strncpy((char *)&desig->desig[24], bdev->name, 32); + memcpy((char *)&desig->desig[24], bdev->name, 32); len += sizeof(struct scsi_desig_desc) + 8 + 16 + 32; buf += sizeof(struct scsi_desig_desc) + desig->len; @@ -312,7 +312,7 @@ vhost_bdev_scsi_inquiry_command(struct vhost_block_dev *bdev, bdev->product_name); /* PRODUCT REVISION LEVEL */ - strncpy((char *)inqdata->product_rev, "0001", 4); + memcpy((char *)inqdata->product_rev, "0001", 4); /* Standard inquiry data ends here. Only populate * remaining fields if alloc_len indicates enough diff --git a/dpdk/lib/librte_bitratestats/rte_bitrate.c b/dpdk/lib/librte_bitratestats/rte_bitrate.c index f373697a..d39edbf6 100644 --- a/dpdk/lib/librte_bitratestats/rte_bitrate.c +++ b/dpdk/lib/librte_bitratestats/rte_bitrate.c @@ -76,6 +76,9 @@ rte_stats_bitrate_reg(struct rte_stats_bitrates *bitrate_data) }; int return_value; + if (bitrate_data == NULL) + return -EINVAL; + return_value = rte_metrics_reg_names(&names[0], ARRAY_SIZE(names)); if (return_value >= 0) bitrate_data->id_stats_set = return_value; @@ -94,6 +97,9 @@ rte_stats_bitrate_calc(struct rte_stats_bitrates *bitrate_data, const int64_t alpha_percent = 20; uint64_t values[6]; + if (bitrate_data == NULL) + return -EINVAL; + ret_code = rte_eth_stats_get(port_id, ð_stats); if (ret_code != 0) return ret_code; diff --git a/dpdk/lib/librte_cryptodev/rte_cryptodev.c b/dpdk/lib/librte_cryptodev/rte_cryptodev.c index 9fe0d9da..b8e14b86 100644 --- a/dpdk/lib/librte_cryptodev/rte_cryptodev.c +++ b/dpdk/lib/librte_cryptodev/rte_cryptodev.c @@ -290,19 +290,40 @@ rte_cryptodev_sym_capability_get(uint8_t dev_id, } -#define param_range_check(x, y) \ - (((x < y.min) || (x > y.max)) || \ - (y.increment != 0 && (x % y.increment) != 0)) +static int +param_range_check(uint16_t size, const struct rte_crypto_param_range *range) +{ + unsigned int next_size; + + /* Check lower/upper bounds */ + if (size < range->min) + return -1; + + if (size > range->max) + return -1; + + /* If range is actually only one value, size is correct */ + if (range->increment == 0) + return 0; + + /* Check if value is one of the supported sizes */ + for (next_size = range->min; next_size <= range->max; + next_size += range->increment) + if (size == next_size) + return 0; + + return -1; +} int rte_cryptodev_sym_capability_check_cipher( const struct rte_cryptodev_symmetric_capability *capability, uint16_t key_size, uint16_t iv_size) { - if (param_range_check(key_size, capability->cipher.key_size)) + if (param_range_check(key_size, &capability->cipher.key_size) != 0) return -1; - if (param_range_check(iv_size, capability->cipher.iv_size)) + if (param_range_check(iv_size, &capability->cipher.iv_size) != 0) return -1; return 0; @@ -313,13 +334,13 @@ rte_cryptodev_sym_capability_check_auth( const struct rte_cryptodev_symmetric_capability *capability, uint16_t key_size, uint16_t digest_size, uint16_t iv_size) { - if (param_range_check(key_size, capability->auth.key_size)) + if (param_range_check(key_size, &capability->auth.key_size) != 0) return -1; - if (param_range_check(digest_size, capability->auth.digest_size)) + if (param_range_check(digest_size, &capability->auth.digest_size) != 0) return -1; - if (param_range_check(iv_size, capability->auth.iv_size)) + if (param_range_check(iv_size, &capability->auth.iv_size) != 0) return -1; return 0; @@ -331,16 +352,16 @@ rte_cryptodev_sym_capability_check_aead( uint16_t key_size, uint16_t digest_size, uint16_t aad_size, uint16_t iv_size) { - if (param_range_check(key_size, capability->aead.key_size)) + if (param_range_check(key_size, &capability->aead.key_size) != 0) return -1; - if (param_range_check(digest_size, capability->aead.digest_size)) + if (param_range_check(digest_size, &capability->aead.digest_size) != 0) return -1; - if (param_range_check(aad_size, capability->aead.aad_size)) + if (param_range_check(aad_size, &capability->aead.aad_size) != 0) return -1; - if (param_range_check(iv_size, capability->aead.iv_size)) + if (param_range_check(iv_size, &capability->aead.iv_size) != 0) return -1; return 0; diff --git a/dpdk/lib/librte_eal/common/eal_common_memory.c b/dpdk/lib/librte_eal/common/eal_common_memory.c index fc6c44da..a0922f18 100644 --- a/dpdk/lib/librte_eal/common/eal_common_memory.c +++ b/dpdk/lib/librte_eal/common/eal_common_memory.c @@ -109,6 +109,49 @@ rte_dump_physmem_layout(FILE *f) } } +/* 63 bits is good enough for a sanity check */ +#define MAX_DMA_MASK_BITS 63 + +/* check memseg iovas are within the required range based on dma mask */ +int +rte_eal_check_dma_mask(uint8_t maskbits) +{ + + const struct rte_mem_config *mcfg; + uint64_t mask; + int i; + + /* sanity check */ + if (maskbits > MAX_DMA_MASK_BITS) { + RTE_LOG(INFO, EAL, "wrong dma mask size %u (Max: %u)\n", + maskbits, MAX_DMA_MASK_BITS); + return -1; + } + + /* create dma mask */ + mask = ~((1ULL << maskbits) - 1); + + /* get pointer to global configuration */ + mcfg = rte_eal_get_configuration()->mem_config; + + for (i = 0; i < RTE_MAX_MEMSEG; i++) { + if (mcfg->memseg[i].addr == NULL) + break; + + if (mcfg->memseg[i].iova & mask) { + RTE_LOG(INFO, EAL, + "memseg[%d] iova %"PRIx64" out of range:\n", + i, mcfg->memseg[i].iova); + + RTE_LOG(INFO, EAL, "\tusing dma mask %"PRIx64"\n", + mask); + return -1; + } + } + + return 0; +} + /* return the number of memory channels */ unsigned rte_memory_get_nchannel(void) { diff --git a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h index 6993dd29..3a80311b 100644 --- a/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h +++ b/dpdk/lib/librte_eal/common/include/arch/ppc_64/rte_atomic.h @@ -55,7 +55,7 @@ extern "C" { * Guarantees that the LOAD and STORE operations generated before the * barrier occur before the LOAD and STORE operations generated after. */ -#define rte_mb() {asm volatile("sync" : : : "memory"); } +#define rte_mb() asm volatile("sync" : : : "memory") /** * Write memory barrier. diff --git a/dpdk/lib/librte_eal/common/include/rte_bitmap.h b/dpdk/lib/librte_eal/common/include/rte_bitmap.h index b9067e67..13bfd9cb 100644 --- a/dpdk/lib/librte_eal/common/include/rte_bitmap.h +++ b/dpdk/lib/librte_eal/common/include/rte_bitmap.h @@ -227,12 +227,12 @@ rte_bitmap_get_memory_footprint(uint32_t n_bits) { /** * Bitmap initialization * - * @param mem_size - * Minimum expected size of bitmap. + * @param n_bits + * Number of pre-allocated bits in array2. * @param mem * Base address of array1 and array2. - * @param n_bits - * Number of pre-allocated bits in array2. Must be non-zero and multiple of 512. + * @param mem_size + * Minimum expected size of bitmap. * @return * Handle to bitmap instance. */ diff --git a/dpdk/lib/librte_eal/common/include/rte_memory.h b/dpdk/lib/librte_eal/common/include/rte_memory.h index 80a8fc02..b2a01689 100644 --- a/dpdk/lib/librte_eal/common/include/rte_memory.h +++ b/dpdk/lib/librte_eal/common/include/rte_memory.h @@ -209,6 +209,9 @@ unsigned rte_memory_get_nchannel(void); */ unsigned rte_memory_get_nrank(void); +/* check memsegs iovas are within a range based on dma mask */ +int rte_eal_check_dma_mask(uint8_t maskbits); + /** * Drivers based on uio will not load unless physical * addresses are obtainable. It is only possible to get diff --git a/dpdk/lib/librte_eal/common/include/rte_version.h b/dpdk/lib/librte_eal/common/include/rte_version.h index 4616a080..4d0a9f7c 100644 --- a/dpdk/lib/librte_eal/common/include/rte_version.h +++ b/dpdk/lib/librte_eal/common/include/rte_version.h @@ -66,7 +66,7 @@ extern "C" { /** * Patch level number i.e. the z in yy.mm.z */ -#define RTE_VER_MINOR 2 +#define RTE_VER_MINOR 4 /** * Extra string to be appended to version number diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_interrupts.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_interrupts.c index 1c20693d..e1179b85 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_interrupts.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_interrupts.c @@ -449,7 +449,7 @@ rte_intr_callback_register(const struct rte_intr_handle *intr_handle, TAILQ_FOREACH(src, &intr_sources, next) { if (src->intr_handle.fd == intr_handle->fd) { /* we had no interrupts for this */ - if TAILQ_EMPTY(&src->callbacks) + if (TAILQ_EMPTY(&src->callbacks)) wake_thread = 1; TAILQ_INSERT_TAIL(&(src->callbacks), callback, next); diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_memory.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_memory.c index 16a181c3..bac969a1 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_memory.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_memory.c @@ -88,6 +88,23 @@ static uint64_t baseaddr_offset; +#ifdef RTE_ARCH_64 +/* + * Linux kernel uses a really high address as starting address for serving + * mmaps calls. If there exists addressing limitations and IOVA mode is VA, + * this starting address is likely too high for those devices. However, it + * is possible to use a lower address in the process virtual address space + * as with 64 bits there is a lot of available space. + * + * Current known limitations are 39 or 40 bits. Setting the starting address + * at 4GB implies there are 508GB or 1020GB for mapping the available + * hugepages. This is likely enough for most systems, although a device with + * addressing limitations should call rte_dev_check_dma_mask for ensuring all + * memory is within supported range. + */ +static uint64_t baseaddr = 0x100000000; +#endif + static bool phys_addrs_available = true; #define RANDOMIZE_VA_SPACE_FILE "/proc/sys/kernel/randomize_va_space" @@ -95,7 +112,7 @@ static bool phys_addrs_available = true; static void test_phys_addrs_available(void) { - uint64_t tmp; + uint64_t tmp = 0; phys_addr_t physaddr; if (!rte_eal_has_hugepages()) { @@ -250,6 +267,23 @@ aslr_enabled(void) } } +static void * +get_addr_hint(void) +{ + if (internal_config.base_virtaddr != 0) { + return (void *) (uintptr_t) + (internal_config.base_virtaddr + + baseaddr_offset); + } else { +#ifdef RTE_ARCH_64 + return (void *) (uintptr_t) (baseaddr + + baseaddr_offset); +#else + return NULL; +#endif + } +} + /* * Try to mmap *size bytes in /dev/zero. If it is successful, return the * pointer to the mmap'd area and keep *size unmodified. Else, retry @@ -260,16 +294,10 @@ aslr_enabled(void) static void * get_virtual_area(size_t *size, size_t hugepage_sz) { - void *addr; + void *addr, *addr_hint; int fd; long aligned_addr; - if (internal_config.base_virtaddr != 0) { - addr = (void*) (uintptr_t) (internal_config.base_virtaddr + - baseaddr_offset); - } - else addr = NULL; - RTE_LOG(DEBUG, EAL, "Ask a virtual area of 0x%zx bytes\n", *size); fd = open("/dev/zero", O_RDONLY); @@ -278,7 +306,9 @@ get_virtual_area(size_t *size, size_t hugepage_sz) return NULL; } do { - addr = mmap(addr, + addr_hint = get_addr_hint(); + + addr = mmap(addr_hint, (*size) + hugepage_sz, PROT_READ, #ifdef RTE_ARCH_PPC_64 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, @@ -286,8 +316,15 @@ get_virtual_area(size_t *size, size_t hugepage_sz) MAP_PRIVATE, #endif fd, 0); - if (addr == MAP_FAILED) + if (addr == MAP_FAILED) { + /* map failed. Let's try with less memory */ *size -= hugepage_sz; + } else if (addr_hint && addr != addr_hint) { + /* hint was not used. Try with another offset */ + munmap(addr, (*size) + hugepage_sz); + addr = MAP_FAILED; + baseaddr_offset += 0x100000000; + } } while (addr == MAP_FAILED && *size > 0); if (addr == MAP_FAILED) { @@ -491,6 +528,9 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi, hugepg_tbl[i].orig_va = virtaddr; } else { + /* rewrite physical addresses in IOVA as VA mode */ + if (rte_eal_iova_mode() == RTE_IOVA_VA) + hugepg_tbl[i].physaddr = (uintptr_t)virtaddr; hugepg_tbl[i].final_va = virtaddr; } @@ -1109,7 +1149,8 @@ rte_eal_hugepage_init(void) continue; } - if (phys_addrs_available) { + if (phys_addrs_available && + rte_eal_iova_mode() != RTE_IOVA_VA) { /* find physical addresses for each hugepage */ if (find_physaddrs(&tmp_hp[hp_offset], hpi) < 0) { RTE_LOG(DEBUG, EAL, "Failed to find phys addr " diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_thread.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_thread.c index e9a579e4..c3947d73 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_thread.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_thread.c @@ -205,7 +205,7 @@ int rte_sys_gettid(void) int rte_thread_setname(pthread_t id, const char *name) { - int ret = -1; + int ret = ENOSYS; #if defined(__GLIBC__) && defined(__GLIBC_PREREQ) #if __GLIBC_PREREQ(2, 12) ret = pthread_setname_np(id, name); @@ -213,5 +213,5 @@ int rte_thread_setname(pthread_t id, const char *name) #endif RTE_SET_USED(id); RTE_SET_USED(name); - return ret; + return -ret; } diff --git a/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c b/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c index fb1a6226..a75ef5a1 100644 --- a/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c +++ b/dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c @@ -710,10 +710,7 @@ vfio_type1_dma_map(int vfio_container_fd) dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map); dma_map.vaddr = ms[i].addr_64; dma_map.size = ms[i].len; - if (rte_eal_iova_mode() == RTE_IOVA_VA) - dma_map.iova = dma_map.vaddr; - else - dma_map.iova = ms[i].iova; + dma_map.iova = ms[i].iova; dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; ret = ioctl(vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dma_map); @@ -813,10 +810,7 @@ vfio_spapr_dma_map(int vfio_container_fd) dma_map.argsz = sizeof(struct vfio_iommu_type1_dma_map); dma_map.vaddr = ms[i].addr_64; dma_map.size = ms[i].len; - if (rte_eal_iova_mode() == RTE_IOVA_VA) - dma_map.iova = dma_map.vaddr; - else - dma_map.iova = ms[i].iova; + dma_map.iova = ms[i].iova; dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; diff --git a/dpdk/lib/librte_eal/linuxapp/kni/compat.h b/dpdk/lib/librte_eal/linuxapp/kni/compat.h index 3f8c0bc8..6a6968d9 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/compat.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/compat.h @@ -101,6 +101,11 @@ #undef NET_NAME_UNKNOWN #endif +#if (defined(RHEL_RELEASE_CODE) && \ + (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7, 5))) +#define ndo_change_mtu ndo_change_mtu_rh74 +#endif + #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) #define HAVE_SIGNAL_FUNCTIONS_OWN_HEADER #endif diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ethtool.c b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ethtool.c index 95e262b7..aed14bcc 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ethtool.c +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ethtool.c @@ -826,9 +826,10 @@ static void igb_get_drvinfo(struct net_device *netdev, strncpy(drvinfo->driver, igb_driver_name, sizeof(drvinfo->driver) - 1); strncpy(drvinfo->version, igb_driver_version, sizeof(drvinfo->version) - 1); - strncpy(drvinfo->fw_version, adapter->fw_version, - sizeof(drvinfo->fw_version) - 1); - strncpy(drvinfo->bus_info, pci_name(adapter->pdev), sizeof(drvinfo->bus_info) -1); + strlcpy(drvinfo->fw_version, adapter->fw_version, + sizeof(drvinfo->fw_version)); + strlcpy(drvinfo->bus_info, pci_name(adapter->pdev), + sizeof(drvinfo->bus_info)); drvinfo->n_stats = IGB_STATS_LEN; drvinfo->testinfo_len = IGB_TEST_LEN; drvinfo->regdump_len = igb_get_regs_len(netdev); diff --git a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h index 443a3f28..6b738911 100644 --- a/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h +++ b/dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h @@ -3915,7 +3915,8 @@ skb_set_hash(struct sk_buff *skb, __u32 hash, __always_unused int type) #define HAVE_NDO_BRIDGE_GETLINK_NLFLAGS #endif /* >= 4.1.0 */ -#if ( LINUX_VERSION_CODE >= KERNEL_VERSION(4,2,0) ) +#if (( LINUX_VERSION_CODE >= KERNEL_VERSION(4,2,0) ) \ + || ( RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7,4) )) /* ndo_bridge_getlink adds new filter_mask and vlan_fill parameters */ #define HAVE_NDO_BRIDGE_GETLINK_FILTER_MASK_VLAN_FILL #endif /* >= 4.2.0 */ @@ -3933,9 +3934,20 @@ skb_set_hash(struct sk_buff *skb, __u32 hash, __always_unused int type) #endif #if ((LINUX_VERSION_CODE >= KERNEL_VERSION(4, 9, 0)) || \ - (SLE_VERSION_CODE && SLE_VERSION_CODE >= SLE_VERSION(12, 3, 0))) + (SLE_VERSION_CODE && SLE_VERSION_CODE >= SLE_VERSION(12, 3, 0)) || \ + (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7, 4))) #define HAVE_VF_VLAN_PROTO -#endif /* >= 4.9.0, >= SLES12SP3 */ +#if (RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(7, 4)) +/* In RHEL/Centos 7.4, the "new" version of ndo_set_vf_vlan + * is in the struct net_device_ops_extended */ +#define ndo_set_vf_vlan extended.ndo_set_vf_vlan +#endif +#endif + +#if (defined(RHEL_RELEASE_CODE) && \ + (RHEL_RELEASE_VERSION(7, 5) <= RHEL_RELEASE_CODE)) +#define ndo_change_mtu ndo_change_mtu_rh74 +#endif #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0) #define HAVE_PCI_ENABLE_MSIX diff --git a/dpdk/lib/librte_eal/rte_eal_version.map b/dpdk/lib/librte_eal/rte_eal_version.map index f4f46c1b..aa6cf871 100644 --- a/dpdk/lib/librte_eal/rte_eal_version.map +++ b/dpdk/lib/librte_eal/rte_eal_version.map @@ -184,6 +184,7 @@ DPDK_17.11 { rte_eal_create_uio_dev; rte_bus_get_iommu_class; + rte_eal_check_dma_mask; rte_eal_has_pci; rte_eal_iova_mode; rte_eal_mbuf_default_mempool_ops; diff --git a/dpdk/lib/librte_ether/rte_ethdev.c b/dpdk/lib/librte_ether/rte_ethdev.c index 4d23bc1c..eea11d06 100644 --- a/dpdk/lib/librte_ether/rte_ethdev.c +++ b/dpdk/lib/librte_ether/rte_ethdev.c @@ -533,6 +533,12 @@ rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id) RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); dev = &rte_eth_devices[port_id]; + if (!dev->data->dev_started) { + RTE_PMD_DEBUG_TRACE( + "port %d must be started before start any queue\n", port_id); + return -EINVAL; + } + if (rx_queue_id >= dev->data->nb_rx_queues) { RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id); return -EINVAL; @@ -585,6 +591,12 @@ rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id) RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL); dev = &rte_eth_devices[port_id]; + if (!dev->data->dev_started) { + RTE_PMD_DEBUG_TRACE( + "port %d must be started before start any queue\n", port_id); + return -EINVAL; + } + if (tx_queue_id >= dev->data->nb_tx_queues) { RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id); return -EINVAL; diff --git a/dpdk/lib/librte_ether/rte_ethdev.h b/dpdk/lib/librte_ether/rte_ethdev.h index 2cc2eedf..47e37a6f 100644 --- a/dpdk/lib/librte_ether/rte_ethdev.h +++ b/dpdk/lib/librte_ether/rte_ethdev.h @@ -1877,7 +1877,6 @@ struct rte_eth_dev *rte_eth_dev_allocated(const char *name); * to that slot for the driver to use. * * @param name Unique identifier name for each Ethernet device - * @param type Device type of this Ethernet device * @return * - Slot in the rte_dev_devices array for a new device; */ @@ -2543,7 +2542,7 @@ void rte_eth_xstats_reset(uint16_t port_id); * @param stat_idx * The per-queue packet statistics functionality number that the transmit * queue is to be assigned. - * The value must be in the range [0, RTE_MAX_ETHPORT_QUEUE_STATS_MAPS - 1]. + * The value must be in the range [0, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1]. * @return * Zero if successful. Non-zero otherwise. */ @@ -2563,7 +2562,7 @@ int rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, * @param stat_idx * The per-queue packet statistics functionality number that the receive * queue is to be assigned. - * The value must be in the range [0, RTE_MAX_ETHPORT_QUEUE_STATS_MAPS - 1]. + * The value must be in the range [0, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1]. * @return * Zero if successful. Non-zero otherwise. */ @@ -3125,6 +3124,9 @@ static inline int rte_eth_tx_descriptor_status(uint16_t port_id, * invoke this function concurrently on the same tx queue without SW lock. * @see rte_eth_dev_info_get, struct rte_eth_txconf::txq_flags * + * @see rte_eth_tx_prepare to perform some prior checks or adjustments + * for offloads. + * * @param port_id * The port identifier of the Ethernet device. * @param queue_id diff --git a/dpdk/lib/librte_ether/rte_ethdev_pci.h b/dpdk/lib/librte_ether/rte_ethdev_pci.h index ad64a169..f69316d5 100644 --- a/dpdk/lib/librte_ether/rte_ethdev_pci.h +++ b/dpdk/lib/librte_ether/rte_ethdev_pci.h @@ -123,9 +123,6 @@ rte_eth_dev_pci_allocate(struct rte_pci_device *dev, size_t private_data_size) static inline void rte_eth_dev_pci_release(struct rte_eth_dev *eth_dev) { - /* free ether device */ - rte_eth_dev_release_port(eth_dev); - if (rte_eal_process_type() == RTE_PROC_PRIMARY) rte_free(eth_dev->data->dev_private); @@ -139,6 +136,9 @@ rte_eth_dev_pci_release(struct rte_eth_dev *eth_dev) eth_dev->device = NULL; eth_dev->intr_handle = NULL; + + /* free ether device */ + rte_eth_dev_release_port(eth_dev); } typedef int (*eth_dev_pci_callback_t)(struct rte_eth_dev *eth_dev); diff --git a/dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.c b/dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.c index 90106e6c..d5c3fd56 100644 --- a/dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.c +++ b/dpdk/lib/librte_eventdev/rte_event_eth_rx_adapter.c @@ -87,6 +87,8 @@ struct rte_event_eth_rx_adapter { int socket_id; /* Per adapter EAL service */ uint32_t service_id; + /* Adapter started flag */ + uint8_t rxa_started; } __rte_cache_aligned; /* Per eth device */ @@ -220,6 +222,8 @@ eth_poll_wrr_calc(struct rte_event_eth_rx_adapter *rx_adapter) nb_rx_queues = dev_info->dev->data->nb_rx_queues; if (dev_info->rx_queue == NULL) continue; + if (dev_info->internal_event_port) + continue; for (q = 0; q < nb_rx_queues; q++) { struct eth_rx_queue_info *queue_info = &dev_info->rx_queue[q]; @@ -414,14 +418,14 @@ flush_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter) static inline void fill_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter, - uint8_t dev_id, + uint16_t eth_dev_id, uint16_t rx_queue_id, struct rte_mbuf **mbufs, uint16_t num) { uint32_t i; struct eth_device_info *eth_device_info = - &rx_adapter->eth_devices[dev_id]; + &rx_adapter->eth_devices[eth_dev_id]; struct eth_rx_queue_info *eth_rx_queue_info = ð_device_info->rx_queue[rx_queue_id]; @@ -476,7 +480,7 @@ fill_event_buffer(struct rte_event_eth_rx_adapter *rx_adapter, * the hypervisor's switching layer where adjustments can be made to deal with * it. */ -static inline uint32_t +static inline void eth_rx_poll(struct rte_event_eth_rx_adapter *rx_adapter) { uint32_t num_queue; @@ -503,8 +507,10 @@ eth_rx_poll(struct rte_event_eth_rx_adapter *rx_adapter) */ if (buf->count >= BATCH_SIZE) flush_event_buffer(rx_adapter); - if (BATCH_SIZE > (ETH_EVENT_BUFFER_SIZE - buf->count)) - break; + if (BATCH_SIZE > (ETH_EVENT_BUFFER_SIZE - buf->count)) { + rx_adapter->wrr_pos = wrr_pos; + return; + } stats->rx_poll_count++; n = rte_eth_rx_burst(d, qid, mbufs, BATCH_SIZE); @@ -519,7 +525,7 @@ eth_rx_poll(struct rte_event_eth_rx_adapter *rx_adapter) if (nb_rx > max_nb_rx) { rx_adapter->wrr_pos = (wrr_pos + 1) % rx_adapter->wrr_len; - return nb_rx; + break; } } @@ -527,20 +533,22 @@ eth_rx_poll(struct rte_event_eth_rx_adapter *rx_adapter) wrr_pos = 0; } - return nb_rx; + if (buf->count >= BATCH_SIZE) + flush_event_buffer(rx_adapter); } static int event_eth_rx_adapter_service_func(void *args) { struct rte_event_eth_rx_adapter *rx_adapter = args; - struct rte_eth_event_enqueue_buffer *buf; - buf = &rx_adapter->event_enqueue_buffer; if (rte_spinlock_trylock(&rx_adapter->rx_lock) == 0) return 0; - if (eth_rx_poll(rx_adapter) == 0 && buf->count) - flush_event_buffer(rx_adapter); + if (!rx_adapter->rxa_started) { + return 0; + rte_spinlock_unlock(&rx_adapter->rx_lock); + } + eth_rx_poll(rx_adapter); rte_spinlock_unlock(&rx_adapter->rx_lock); return 0; } @@ -829,8 +837,12 @@ rx_adapter_ctrl(uint8_t id, int start) &rte_eth_devices[i]); } - if (use_service) + if (use_service) { + rte_spinlock_lock(&rx_adapter->rx_lock); + rx_adapter->rxa_started = start; rte_service_runstate_set(rx_adapter->service_id, start); + rte_spinlock_unlock(&rx_adapter->rx_lock); + } return 0; } @@ -1032,6 +1044,7 @@ rte_event_eth_rx_adapter_queue_add(uint8_t id, &rte_eth_devices[eth_dev_id], rx_queue_id, queue_conf); if (ret == 0) { + dev_info->internal_event_port = 1; update_queue_info(rx_adapter, &rx_adapter->eth_devices[eth_dev_id], rx_queue_id, @@ -1039,6 +1052,7 @@ rte_event_eth_rx_adapter_queue_add(uint8_t id, } } else { rte_spinlock_lock(&rx_adapter->rx_lock); + dev_info->internal_event_port = 0; ret = init_service(rx_adapter, id); if (ret == 0) ret = add_rx_queue(rx_adapter, eth_dev_id, rx_queue_id, diff --git a/dpdk/lib/librte_eventdev/rte_event_ring.c b/dpdk/lib/librte_eventdev/rte_event_ring.c index b14c2127..c0603cb9 100644 --- a/dpdk/lib/librte_eventdev/rte_event_ring.c +++ b/dpdk/lib/librte_eventdev/rte_event_ring.c @@ -110,11 +110,16 @@ rte_event_ring_create(const char *name, unsigned int count, int socket_id, mz = rte_memzone_reserve(mz_name, ring_size, socket_id, mz_flags); if (mz != NULL) { r = mz->addr; - /* - * no need to check return value here, we already checked the - * arguments above - */ - rte_event_ring_init(r, name, requested_count, flags); + /* Check return value in case rte_ring_init() fails on size */ + int err = rte_event_ring_init(r, name, requested_count, flags); + if (err) { + RTE_LOG(ERR, RING, "Ring init failed\n"); + if (rte_memzone_free(mz) != 0) + RTE_LOG(ERR, RING, "Cannot free memzone\n"); + rte_free(te); + rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + return NULL; + } te->data = (void *) r; r->r.memzone = mz; diff --git a/dpdk/lib/librte_hash/rte_cuckoo_hash.c b/dpdk/lib/librte_hash/rte_cuckoo_hash.c index 55fd7bdc..9be514dc 100644 --- a/dpdk/lib/librte_hash/rte_cuckoo_hash.c +++ b/dpdk/lib/librte_hash/rte_cuckoo_hash.c @@ -154,13 +154,13 @@ rte_hash_create(const struct rte_hash_parameters *params) * except for the first cache */ num_key_slots = params->entries + (RTE_MAX_LCORE - 1) * - LCORE_CACHE_SIZE + 1; + (LCORE_CACHE_SIZE - 1) + 1; else num_key_slots = params->entries + 1; snprintf(ring_name, sizeof(ring_name), "HT_%s", params->name); /* Create ring (Dummy slot index is not enqueued) */ - r = rte_ring_create(ring_name, rte_align32pow2(num_key_slots - 1), + r = rte_ring_create(ring_name, rte_align32pow2(num_key_slots), params->socket_id, 0); if (r == NULL) { RTE_LOG(ERR, HASH, "memory allocation failed\n"); @@ -302,14 +302,17 @@ rte_hash_create(const struct rte_hash_parameters *params) h->add_key = ADD_KEY_MULTIWRITER; h->multiwriter_lock = rte_malloc(NULL, sizeof(rte_spinlock_t), - LCORE_CACHE_SIZE); + RTE_CACHE_LINE_SIZE); + if (h->multiwriter_lock == NULL) + goto err_unlock; + rte_spinlock_init(h->multiwriter_lock); } } else h->add_key = ADD_KEY_SINGLEWRITER; /* Populate free slots ring. Entry zero is reserved for key misses. */ - for (i = 1; i < params->entries + 1; i++) + for (i = 1; i < num_key_slots; i++) rte_ring_sp_enqueue(r, (void *)((uintptr_t) i)); te->data = (void *) h; @@ -391,7 +394,7 @@ void rte_hash_reset(struct rte_hash *h) { void *ptr; - unsigned i; + uint32_t tot_ring_cnt, i; if (h == NULL) return; @@ -404,7 +407,13 @@ rte_hash_reset(struct rte_hash *h) rte_pause(); /* Repopulate the free slots ring. Entry zero is reserved for key misses */ - for (i = 1; i < h->entries + 1; i++) + if (h->hw_trans_mem_support) + tot_ring_cnt = h->entries + (RTE_MAX_LCORE - 1) * + (LCORE_CACHE_SIZE - 1); + else + tot_ring_cnt = h->entries; + + for (i = 1; i < tot_ring_cnt + 1; i++) rte_ring_sp_enqueue(h->free_slots, (void *)((uintptr_t) i)); if (h->hw_trans_mem_support) { @@ -573,7 +582,8 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, * Return index where key is stored, * subtracting the first dummy index */ - return prim_bkt->key_idx[i] - 1; + ret = prim_bkt->key_idx[i] - 1; + goto failure; } } } @@ -593,7 +603,8 @@ __rte_hash_add_key_with_hash(const struct rte_hash *h, const void *key, * Return index where key is stored, * subtracting the first dummy index */ - return sec_bkt->key_idx[i] - 1; + ret = sec_bkt->key_idx[i] - 1; + goto failure; } } } diff --git a/dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h b/dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h index 0c94244a..a2f1663e 100644 --- a/dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h +++ b/dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h @@ -95,6 +95,9 @@ rte_hash_cuckoo_move_insert_mw_tm(const struct rte_hash *h, while (try < RTE_HASH_TSX_MAX_RETRY) { status = rte_xbegin(); if (likely(status == RTE_XBEGIN_STARTED)) { + /* In case empty slot was gone before entering TSX */ + if (curr_bkt->key_idx[curr_slot] != EMPTY_SLOT) + rte_xabort(RTE_XABORT_CUCKOO_PATH_INVALIDED); while (likely(curr_node->prev != NULL)) { prev_node = curr_node->prev; prev_bkt = prev_node->bkt; diff --git a/dpdk/lib/librte_hash/rte_hash.h b/dpdk/lib/librte_hash/rte_hash.h index 622c1800..5539e338 100644 --- a/dpdk/lib/librte_hash/rte_hash.h +++ b/dpdk/lib/librte_hash/rte_hash.h @@ -283,8 +283,8 @@ rte_hash_del_key_with_hash(const struct rte_hash *h, const void *key, hash_sig_t * Output containing a pointer to the key * @return * - 0 if retrieved successfully - * - EINVAL if the parameters are invalid. - * - ENOENT if no valid key is found in the given position. + * - -EINVAL if the parameters are invalid. + * - -ENOENT if no valid key is found in the given position. */ int rte_hash_get_key_with_position(const struct rte_hash *h, const int32_t position, @@ -301,9 +301,11 @@ rte_hash_get_key_with_position(const struct rte_hash *h, const int32_t position, * @param data * Output with pointer to data returned from the hash table. * @return - * 0 if successful lookup - * - EINVAL if the parameters are invalid. - * - ENOENT if the key is not found. + * - A positive value that can be used by the caller as an offset into an + * array of user data. This value is unique for this key, and is the same + * value that was returned when the key was added. + * - -EINVAL if the parameters are invalid. + * - -ENOENT if the key is not found. */ int rte_hash_lookup_data(const struct rte_hash *h, const void *key, void **data); @@ -322,9 +324,11 @@ rte_hash_lookup_data(const struct rte_hash *h, const void *key, void **data); * @param data * Output with pointer to data returned from the hash table. * @return - * 0 if successful lookup - * - EINVAL if the parameters are invalid. - * - ENOENT if the key is not found. + * - A positive value that can be used by the caller as an offset into an + * array of user data. This value is unique for this key, and is the same + * value that was returned when the key was added. + * - -EINVAL if the parameters are invalid. + * - -ENOENT if the key is not found. */ int rte_hash_lookup_with_hash_data(const struct rte_hash *h, const void *key, diff --git a/dpdk/lib/librte_ip_frag/rte_ipv4_reassembly.c b/dpdk/lib/librte_ip_frag/rte_ipv4_reassembly.c index 040bd70a..3b84dcad 100644 --- a/dpdk/lib/librte_ip_frag/rte_ipv4_reassembly.c +++ b/dpdk/lib/librte_ip_frag/rte_ipv4_reassembly.c @@ -88,7 +88,9 @@ ipv4_frag_reassemble(struct ip_frag_pkt *fp) /* chain with the first fragment. */ rte_pktmbuf_adj(m, (uint16_t)(m->l2_len + m->l3_len)); rte_pktmbuf_chain(fp->frags[IP_FIRST_FRAG_IDX].mb, m); + fp->frags[curr_idx].mb = NULL; m = fp->frags[IP_FIRST_FRAG_IDX].mb; + fp->frags[IP_FIRST_FRAG_IDX].mb = NULL; /* update mbuf fields for reassembled packet. */ m->ol_flags |= PKT_TX_IP_CKSUM; diff --git a/dpdk/lib/librte_ip_frag/rte_ipv6_reassembly.c b/dpdk/lib/librte_ip_frag/rte_ipv6_reassembly.c index dde58cb7..26b9a883 100644 --- a/dpdk/lib/librte_ip_frag/rte_ipv6_reassembly.c +++ b/dpdk/lib/librte_ip_frag/rte_ipv6_reassembly.c @@ -111,7 +111,9 @@ ipv6_frag_reassemble(struct ip_frag_pkt *fp) /* chain with the first fragment. */ rte_pktmbuf_adj(m, (uint16_t)(m->l2_len + m->l3_len)); rte_pktmbuf_chain(fp->frags[IP_FIRST_FRAG_IDX].mb, m); + fp->frags[curr_idx].mb = NULL; m = fp->frags[IP_FIRST_FRAG_IDX].mb; + fp->frags[IP_FIRST_FRAG_IDX].mb = NULL; /* update mbuf fields for reassembled packet. */ m->ol_flags |= PKT_TX_IP_CKSUM; diff --git a/dpdk/lib/librte_kni/rte_kni.c b/dpdk/lib/librte_kni/rte_kni.c index 8eca8c03..4b45d1a0 100644 --- a/dpdk/lib/librte_kni/rte_kni.c +++ b/dpdk/lib/librte_kni/rte_kni.c @@ -681,6 +681,9 @@ rte_kni_get(const char *name) struct rte_kni_memzone_slot *it; struct rte_kni *kni; + if (name == NULL || name[0] == '\0') + return NULL; + /* Note: could be improved perf-wise if necessary */ for (i = 0; i < kni_memzone_pool.max_ifaces; i++) { it = &kni_memzone_pool.slots[i]; diff --git a/dpdk/lib/librte_mbuf/rte_mbuf.h b/dpdk/lib/librte_mbuf/rte_mbuf.h index 16a6048c..807a8d41 100644 --- a/dpdk/lib/librte_mbuf/rte_mbuf.h +++ b/dpdk/lib/librte_mbuf/rte_mbuf.h @@ -222,14 +222,17 @@ extern "C" { /** * Bits 45:48 used for the tunnel type. - * When doing Tx offload like TSO or checksum, the HW needs to configure the - * tunnel type into the HW descriptors. + * The tunnel type must be specified for TSO or checksum on the inner part + * of tunnel packets. + * These flags can be used with PKT_TX_TCP_SEG for TSO, or PKT_TX_xxx_CKSUM. + * The mbuf fields for inner and outer header lengths are required: + * outer_l2_len, outer_l3_len, l2_len, l3_len, l4_len and tso_segsz for TSO. */ #define PKT_TX_TUNNEL_VXLAN (0x1ULL << 45) #define PKT_TX_TUNNEL_GRE (0x2ULL << 45) #define PKT_TX_TUNNEL_IPIP (0x3ULL << 45) #define PKT_TX_TUNNEL_GENEVE (0x4ULL << 45) -/**< TX packet with MPLS-in-UDP RFC 7510 header. */ +/** TX packet with MPLS-in-UDP RFC 7510 header. */ #define PKT_TX_TUNNEL_MPLSINUDP (0x5ULL << 45) /* add new TX TUNNEL type here */ #define PKT_TX_TUNNEL_MASK (0xFULL << 45) @@ -245,12 +248,8 @@ extern "C" { * - set the PKT_TX_TCP_SEG flag in mbuf->ol_flags (this flag implies * PKT_TX_TCP_CKSUM) * - set the flag PKT_TX_IPV4 or PKT_TX_IPV6 - * - if it's IPv4, set the PKT_TX_IP_CKSUM flag and write the IP checksum - * to 0 in the packet + * - if it's IPv4, set the PKT_TX_IP_CKSUM flag * - fill the mbuf offload information: l2_len, l3_len, l4_len, tso_segsz - * - calculate the pseudo header checksum without taking ip_len in account, - * and set it in the TCP header. Refer to rte_ipv4_phdr_cksum() and - * rte_ipv6_phdr_cksum() that can be used as helpers. */ #define PKT_TX_TCP_SEG (1ULL << 50) @@ -263,9 +262,6 @@ extern "C" { * - fill l2_len and l3_len in mbuf * - set the flags PKT_TX_TCP_CKSUM, PKT_TX_SCTP_CKSUM or PKT_TX_UDP_CKSUM * - set the flag PKT_TX_IPV4 or PKT_TX_IPV6 - * - calculate the pseudo header checksum and set it in the L4 header (only - * for TCP or UDP). See rte_ipv4_phdr_cksum() and rte_ipv6_phdr_cksum(). - * For SCTP, set the crc field to 0. */ #define PKT_TX_L4_NO_CKSUM (0ULL << 52) /**< Disable L4 cksum of TX pkt. */ #define PKT_TX_TCP_CKSUM (1ULL << 52) /**< TCP cksum of TX pkt. computed by NIC. */ @@ -277,7 +273,6 @@ extern "C" { * Offload the IP checksum in the hardware. The flag PKT_TX_IPV4 should * also be set by the application, although a PMD will only check * PKT_TX_IP_CKSUM. - * - set the IP checksum field in the packet to 0 * - fill the mbuf offload information: l2_len, l3_len */ #define PKT_TX_IP_CKSUM (1ULL << 54) @@ -302,10 +297,8 @@ extern "C" { /** * Offload the IP checksum of an external header in the hardware. The - * flag PKT_TX_OUTER_IPV4 should also be set by the application, alto ugh - * a PMD will only check PKT_TX_IP_CKSUM. The IP checksum field in the - * packet must be set to 0. - * - set the outer IP checksum field in the packet to 0 + * flag PKT_TX_OUTER_IPV4 should also be set by the application, although + * a PMD will only check PKT_TX_OUTER_IP_CKSUM. * - fill the mbuf offload information: outer_l2_len, outer_l3_len */ #define PKT_TX_OUTER_IP_CKSUM (1ULL << 58) diff --git a/dpdk/lib/librte_mempool/rte_mempool.c b/dpdk/lib/librte_mempool/rte_mempool.c index 5bd74eac..ad174582 100644 --- a/dpdk/lib/librte_mempool/rte_mempool.c +++ b/dpdk/lib/librte_mempool/rte_mempool.c @@ -437,12 +437,18 @@ rte_mempool_populate_iova(struct rte_mempool *mp, char *vaddr, } /* not enough room to store one object */ - if (i == 0) - return -EINVAL; + if (i == 0) { + ret = -EINVAL; + goto fail; + } STAILQ_INSERT_TAIL(&mp->mem_list, memhdr, next); mp->nb_mem_chunks++; return i; + +fail: + rte_free(memhdr); + return ret; } int @@ -514,9 +520,6 @@ rte_mempool_populate_virt(struct rte_mempool *mp, char *addr, size_t off, phys_len; int ret, cnt = 0; - /* mempool must not be populated */ - if (mp->nb_mem_chunks != 0) - return -EEXIST; /* address and len must be page-aligned */ if (RTE_PTR_ALIGN_CEIL(addr, pg_sz) != addr) return -EINVAL; @@ -685,7 +688,7 @@ rte_mempool_populate_anon(struct rte_mempool *mp) char *addr; /* mempool is already populated, error */ - if (!STAILQ_EMPTY(&mp->mem_list)) { + if ((!STAILQ_EMPTY(&mp->mem_list)) || mp->nb_mem_chunks != 0) { rte_errno = EINVAL; return 0; } diff --git a/dpdk/lib/librte_metrics/rte_metrics.c b/dpdk/lib/librte_metrics/rte_metrics.c index d9404001..dc634467 100644 --- a/dpdk/lib/librte_metrics/rte_metrics.c +++ b/dpdk/lib/librte_metrics/rte_metrics.c @@ -124,6 +124,9 @@ rte_metrics_reg_names(const char * const *names, uint16_t cnt_names) /* Some sanity checks */ if (cnt_names < 1 || names == NULL) return -EINVAL; + for (idx_name = 0; idx_name < cnt_names; idx_name++) + if (names[idx_name] == NULL) + return -EINVAL; memzone = rte_memzone_lookup(RTE_METRICS_MEMZONE_NAME); if (memzone == NULL) @@ -190,6 +193,11 @@ rte_metrics_update_values(int port_id, stats = memzone->addr; rte_spinlock_lock(&stats->lock); + + if (key >= stats->cnt_stats) { + rte_spinlock_unlock(&stats->lock); + return -EINVAL; + } idx_metric = key; cnt_setsize = 1; while (idx_metric < stats->cnt_stats) { @@ -231,9 +239,8 @@ rte_metrics_get_names(struct rte_metric_name *names, int return_value; memzone = rte_memzone_lookup(RTE_METRICS_MEMZONE_NAME); - /* If not allocated, fail silently */ if (memzone == NULL) - return 0; + return -EIO; stats = memzone->addr; rte_spinlock_lock(&stats->lock); @@ -269,9 +276,9 @@ rte_metrics_get_values(int port_id, return -EINVAL; memzone = rte_memzone_lookup(RTE_METRICS_MEMZONE_NAME); - /* If not allocated, fail silently */ if (memzone == NULL) - return 0; + return -EIO; + stats = memzone->addr; rte_spinlock_lock(&stats->lock); diff --git a/dpdk/lib/librte_net/rte_ip.h b/dpdk/lib/librte_net/rte_ip.h index 73ec398f..23468cb9 100644 --- a/dpdk/lib/librte_net/rte_ip.h +++ b/dpdk/lib/librte_net/rte_ip.h @@ -406,6 +406,12 @@ struct ipv6_hdr { uint8_t dst_addr[16]; /**< IP address of destination host(s). */ } __attribute__((__packed__)); +/* IPv6 vtc_flow: IPv / TC / flow_label */ +#define IPV6_HDR_FL_SHIFT 0 +#define IPV6_HDR_TC_SHIFT 20 +#define IPV6_HDR_FL_MASK ((1u << IPV6_HDR_TC_SHIFT) - 1) +#define IPV6_HDR_TC_MASK (0xf << IPV6_HDR_TC_SHIFT) + /** * Process the pseudo-header checksum of an IPv6 header. * diff --git a/dpdk/lib/librte_pci/rte_pci_version.map b/dpdk/lib/librte_pci/rte_pci_version.map index 15d93d95..c0280277 100644 --- a/dpdk/lib/librte_pci/rte_pci_version.map +++ b/dpdk/lib/librte_pci/rte_pci_version.map @@ -3,12 +3,11 @@ DPDK_17.11 { eal_parse_pci_BDF; eal_parse_pci_DomBDF; - rte_pci_addr_cmp; - rte_pci_addr_parse; - rte_pci_device_name; pci_map_resource; pci_unmap_resource; rte_eal_compare_pci_addr; + rte_pci_addr_cmp; + rte_pci_addr_parse; rte_pci_device_name; local: *; diff --git a/dpdk/lib/librte_ring/rte_ring.h b/dpdk/lib/librte_ring/rte_ring.h index 7069d52e..fc433b05 100644 --- a/dpdk/lib/librte_ring/rte_ring.h +++ b/dpdk/lib/librte_ring/rte_ring.h @@ -582,7 +582,7 @@ __rte_ring_do_dequeue(struct rte_ring *r, void **obj_table, uint32_t cons_head, cons_next; uint32_t entries; - n = __rte_ring_move_cons_head(r, is_sc, n, behavior, + n = __rte_ring_move_cons_head(r, (int)is_sc, n, behavior, &cons_head, &cons_next, &entries); if (n == 0) goto end; diff --git a/dpdk/lib/librte_security/rte_security.c b/dpdk/lib/librte_security/rte_security.c index 685729f4..503f5b53 100644 --- a/dpdk/lib/librte_security/rte_security.c +++ b/dpdk/lib/librte_security/rte_security.c @@ -84,7 +84,6 @@ rte_security_session_destroy(struct rte_security_ctx *instance, struct rte_security_session *sess) { int ret; - struct rte_mempool *mp = rte_mempool_from_obj(sess); RTE_FUNC_PTR_OR_ERR_RET(*instance->ops->session_destroy, -ENOTSUP); @@ -93,7 +92,7 @@ rte_security_session_destroy(struct rte_security_ctx *instance, ret = instance->ops->session_destroy(instance->device, sess); if (!ret) - rte_mempool_put(mp, (void *)sess); + rte_mempool_put(rte_mempool_from_obj(sess), (void *)sess); return ret; } diff --git a/dpdk/lib/librte_vhost/fd_man.c b/dpdk/lib/librte_vhost/fd_man.c index 4c6fed41..173ec75c 100644 --- a/dpdk/lib/librte_vhost/fd_man.c +++ b/dpdk/lib/librte_vhost/fd_man.c @@ -200,6 +200,38 @@ fdset_del(struct fdset *pfdset, int fd) return dat; } +/** + * Unregister the fd from the fdset. + * + * If parameters are invalid, return directly -2. + * And check whether fd is busy, if yes, return -1. + * Otherwise, try to delete the fd from fdset and + * return true. + */ +int +fdset_try_del(struct fdset *pfdset, int fd) +{ + int i; + + if (pfdset == NULL || fd == -1) + return -2; + + pthread_mutex_lock(&pfdset->fd_mutex); + i = fdset_find_fd(pfdset, fd); + if (i != -1 && pfdset->fd[i].busy) { + pthread_mutex_unlock(&pfdset->fd_mutex); + return -1; + } + + if (i != -1) { + pfdset->fd[i].fd = -1; + pfdset->fd[i].rcb = pfdset->fd[i].wcb = NULL; + pfdset->fd[i].dat = NULL; + } + + pthread_mutex_unlock(&pfdset->fd_mutex); + return 0; +} /** * This functions runs in infinite blocking loop until there is no fd in diff --git a/dpdk/lib/librte_vhost/fd_man.h b/dpdk/lib/librte_vhost/fd_man.h index 90d34db1..c0088388 100644 --- a/dpdk/lib/librte_vhost/fd_man.h +++ b/dpdk/lib/librte_vhost/fd_man.h @@ -63,6 +63,7 @@ int fdset_add(struct fdset *pfdset, int fd, fd_cb rcb, fd_cb wcb, void *dat); void *fdset_del(struct fdset *pfdset, int fd); +int fdset_try_del(struct fdset *pfdset, int fd); void *fdset_event_dispatch(void *arg); diff --git a/dpdk/lib/librte_vhost/iotlb.c b/dpdk/lib/librte_vhost/iotlb.c index c11ebcaa..c6354fef 100644 --- a/dpdk/lib/librte_vhost/iotlb.c +++ b/dpdk/lib/librte_vhost/iotlb.c @@ -303,6 +303,13 @@ out: return vva; } +void +vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq) +{ + vhost_user_iotlb_cache_remove_all(vq); + vhost_user_iotlb_pending_remove_all(vq); +} + int vhost_user_iotlb_init(struct virtio_net *dev, int vq_index) { @@ -315,8 +322,7 @@ vhost_user_iotlb_init(struct virtio_net *dev, int vq_index) * The cache has already been initialized, * just drop all cached and pending entries. */ - vhost_user_iotlb_cache_remove_all(vq); - vhost_user_iotlb_pending_remove_all(vq); + vhost_user_iotlb_flush_all(vq); } #ifdef RTE_LIBRTE_VHOST_NUMA diff --git a/dpdk/lib/librte_vhost/iotlb.h b/dpdk/lib/librte_vhost/iotlb.h index e7083e37..60b9e4c5 100644 --- a/dpdk/lib/librte_vhost/iotlb.h +++ b/dpdk/lib/librte_vhost/iotlb.h @@ -73,7 +73,7 @@ void vhost_user_iotlb_pending_insert(struct vhost_virtqueue *vq, uint64_t iova, uint8_t perm); void vhost_user_iotlb_pending_remove(struct vhost_virtqueue *vq, uint64_t iova, uint64_t size, uint8_t perm); - +void vhost_user_iotlb_flush_all(struct vhost_virtqueue *vq); int vhost_user_iotlb_init(struct virtio_net *dev, int vq_index); #endif /* _VHOST_IOTLB_H_ */ diff --git a/dpdk/lib/librte_vhost/socket.c b/dpdk/lib/librte_vhost/socket.c index 811e6bf1..f816601b 100644 --- a/dpdk/lib/librte_vhost/socket.c +++ b/dpdk/lib/librte_vhost/socket.c @@ -181,6 +181,11 @@ send_fd_message(int sockfd, char *buf, int buflen, int *fds, int fd_num) msgh.msg_control = control; msgh.msg_controllen = sizeof(control); cmsg = CMSG_FIRSTHDR(&msgh); + if (cmsg == NULL) { + RTE_LOG(ERR, VHOST_CONFIG, "cmsg == NULL\n"); + errno = EINVAL; + return -1; + } cmsg->cmsg_len = CMSG_LEN(fdsize); cmsg->cmsg_level = SOL_SOCKET; cmsg->cmsg_type = SCM_RIGHTS; @@ -756,13 +761,25 @@ rte_vhost_driver_unregister(const char *path) vhost_user_remove_reconnect(vsocket); } +again: pthread_mutex_lock(&vsocket->conn_mutex); for (conn = TAILQ_FIRST(&vsocket->conn_list); conn != NULL; conn = next) { next = TAILQ_NEXT(conn, next); - fdset_del(&vhost_user.fdset, conn->connfd); + /* + * If r/wcb is executing, release the + * conn_mutex lock, and try again since + * the r/wcb may use the conn_mutex lock. + */ + if (fdset_try_del(&vhost_user.fdset, + conn->connfd) == -1) { + pthread_mutex_unlock( + &vsocket->conn_mutex); + goto again; + } + RTE_LOG(INFO, VHOST_CONFIG, "free connfd = %d for device '%s'\n", conn->connfd, path); diff --git a/dpdk/lib/librte_vhost/vhost.h b/dpdk/lib/librte_vhost/vhost.h index 16d6b891..11ce2711 100644 --- a/dpdk/lib/librte_vhost/vhost.h +++ b/dpdk/lib/librte_vhost/vhost.h @@ -59,6 +59,8 @@ #define BUF_VECTOR_MAX 256 +#define VHOST_LOG_CACHE_NR 32 + /** * Structure contains buffer address, length and descriptor index * from vring to do scatter RX. @@ -92,6 +94,14 @@ struct batch_copy_elem { uint64_t log_addr; }; +/* + * Structure that contains the info for batched dirty logging. + */ +struct log_cache_entry { + uint32_t offset; + unsigned long val; +}; + /** * Structure contains variables relevant to RX/TX virtqueues. */ @@ -133,6 +143,9 @@ struct vhost_virtqueue { struct batch_copy_elem *batch_copy_elems; uint16_t batch_copy_nb_elems; + struct log_cache_entry log_cache[VHOST_LOG_CACHE_NR]; + uint16_t log_cache_nb_elem; + rte_rwlock_t iotlb_lock; rte_rwlock_t iotlb_pending_lock; struct rte_mempool *iotlb_pool; @@ -266,7 +279,15 @@ struct virtio_net { static __rte_always_inline void vhost_set_bit(unsigned int nr, volatile uint8_t *addr) { - __sync_fetch_and_or_8(addr, (1U << nr)); +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) + /* + * __sync_ built-ins are deprecated, but __atomic_ ones + * are sub-optimized in older GCC versions. + */ + __sync_fetch_and_or_1(addr, (1U << nr)); +#else + __atomic_fetch_or(addr, (1U << nr), __ATOMIC_RELAXED); +#endif } static __rte_always_inline void @@ -297,6 +318,103 @@ vhost_log_write(struct virtio_net *dev, uint64_t addr, uint64_t len) } } +static __rte_always_inline void +vhost_log_cache_sync(struct virtio_net *dev, struct vhost_virtqueue *vq) +{ + unsigned long *log_base; + int i; + + if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || + !dev->log_base)) + return; + + log_base = (unsigned long *)(uintptr_t)dev->log_base; + + /* + * It is expected a write memory barrier has been issued + * before this function is called. + */ + + for (i = 0; i < vq->log_cache_nb_elem; i++) { + struct log_cache_entry *elem = vq->log_cache + i; + +#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION < 70100) + /* + * '__sync' builtins are deprecated, but '__atomic' ones + * are sub-optimized in older GCC versions. + */ + __sync_fetch_and_or(log_base + elem->offset, elem->val); +#else + __atomic_fetch_or(log_base + elem->offset, elem->val, + __ATOMIC_RELAXED); +#endif + } + + rte_smp_wmb(); + + vq->log_cache_nb_elem = 0; +} + +static __rte_always_inline void +vhost_log_cache_page(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t page) +{ + uint32_t bit_nr = page % (sizeof(unsigned long) << 3); + uint32_t offset = page / (sizeof(unsigned long) << 3); + int i; + + for (i = 0; i < vq->log_cache_nb_elem; i++) { + struct log_cache_entry *elem = vq->log_cache + i; + + if (elem->offset == offset) { + elem->val |= (1UL << bit_nr); + return; + } + } + + if (unlikely(i >= VHOST_LOG_CACHE_NR)) { + /* + * No more room for a new log cache entry, + * so write the dirty log map directly. + */ + rte_smp_wmb(); + vhost_log_page((uint8_t *)(uintptr_t)dev->log_base, page); + + return; + } + + vq->log_cache[i].offset = offset; + vq->log_cache[i].val = (1UL << bit_nr); + vq->log_cache_nb_elem++; +} + +static __rte_always_inline void +vhost_log_cache_write(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t addr, uint64_t len) +{ + uint64_t page; + + if (likely(((dev->features & (1ULL << VHOST_F_LOG_ALL)) == 0) || + !dev->log_base || !len)) + return; + + if (unlikely(dev->log_size <= ((addr + len - 1) / VHOST_LOG_PAGE / 8))) + return; + + page = addr / VHOST_LOG_PAGE; + while (page * VHOST_LOG_PAGE < addr + len) { + vhost_log_cache_page(dev, vq, page); + page += 1; + } +} + +static __rte_always_inline void +vhost_log_cache_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq, + uint64_t offset, uint64_t len) +{ + vhost_log_cache_write(dev, vq, vq->log_guest_addr + offset, len); +} + static __rte_always_inline void vhost_log_used_vring(struct virtio_net *dev, struct vhost_virtqueue *vq, uint64_t offset, uint64_t len) diff --git a/dpdk/lib/librte_vhost/vhost_user.c b/dpdk/lib/librte_vhost/vhost_user.c index 50e654db..0eb5e0d6 100644 --- a/dpdk/lib/librte_vhost/vhost_user.c +++ b/dpdk/lib/librte_vhost/vhost_user.c @@ -476,7 +476,7 @@ vhost_user_set_vring_addr(struct virtio_net **pdev, VhostUserMsg *msg) if (vq->enabled && (dev->features & (1ULL << VHOST_USER_F_PROTOCOL_FEATURES))) { - dev = translate_ring_addresses(dev, msg->payload.state.index); + dev = translate_ring_addresses(dev, msg->payload.addr.index); if (!dev) return -1; @@ -501,7 +501,7 @@ vhost_user_set_vring_base(struct virtio_net *dev, return 0; } -static void +static int add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr, uint64_t host_phys_addr, uint64_t size) { @@ -511,6 +511,10 @@ add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr, dev->max_guest_pages *= 2; dev->guest_pages = realloc(dev->guest_pages, dev->max_guest_pages * sizeof(*page)); + if (!dev->guest_pages) { + RTE_LOG(ERR, VHOST_CONFIG, "cannot realloc guest_pages\n"); + return -1; + } } if (dev->nr_guest_pages > 0) { @@ -519,7 +523,7 @@ add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr, if (host_phys_addr == last_page->host_phys_addr + last_page->size) { last_page->size += size; - return; + return 0; } } @@ -527,9 +531,11 @@ add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr, page->guest_phys_addr = guest_phys_addr; page->host_phys_addr = host_phys_addr; page->size = size; + + return 0; } -static void +static int add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg, uint64_t page_size) { @@ -543,7 +549,9 @@ add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg, size = page_size - (guest_phys_addr & (page_size - 1)); size = RTE_MIN(size, reg_size); - add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size); + if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size) < 0) + return -1; + host_user_addr += size; guest_phys_addr += size; reg_size -= size; @@ -552,12 +560,16 @@ add_guest_pages(struct virtio_net *dev, struct rte_vhost_mem_region *reg, size = RTE_MIN(reg_size, page_size); host_phys_addr = rte_mem_virt2iova((void *)(uintptr_t) host_user_addr); - add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size); + if (add_one_guest_page(dev, guest_phys_addr, host_phys_addr, + size) < 0) + return -1; host_user_addr += size; guest_phys_addr += size; reg_size -= size; } + + return 0; } #ifdef RTE_LIBRTE_VHOST_DEBUG @@ -611,8 +623,9 @@ vhost_memory_changed(struct VhostUserMemory *new, } static int -vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg) +vhost_user_set_mem_table(struct virtio_net **pdev, struct VhostUserMsg *pmsg) { + struct virtio_net *dev = *pdev; struct VhostUserMemory memory = pmsg->payload.memory; struct rte_vhost_mem_region *reg; void *mmap_addr; @@ -638,6 +651,11 @@ vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg) dev->mem = NULL; } + /* Flush IOTLB cache as previous HVAs are now invalid */ + if (dev->features & (1ULL << VIRTIO_F_IOMMU_PLATFORM)) + for (i = 0; i < dev->nr_vring; i++) + vhost_user_iotlb_flush_all(dev->virtqueue[i]); + dev->nr_guest_pages = 0; if (!dev->guest_pages) { dev->max_guest_pages = 8; @@ -705,7 +723,12 @@ vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg) mmap_offset; if (dev->dequeue_zero_copy) - add_guest_pages(dev, reg, alignment); + if (add_guest_pages(dev, reg, alignment) < 0) { + RTE_LOG(ERR, VHOST_CONFIG, + "adding guest pages to region %u failed.\n", + i); + goto err_mmap; + } RTE_LOG(INFO, VHOST_CONFIG, "guest memory region %u, size: 0x%" PRIx64 "\n" @@ -726,6 +749,25 @@ vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg) mmap_offset); } + for (i = 0; i < dev->nr_vring; i++) { + struct vhost_virtqueue *vq = dev->virtqueue[i]; + + if (vq->desc || vq->avail || vq->used) { + /* + * If the memory table got updated, the ring addresses + * need to be translated again as virtual addresses have + * changed. + */ + vring_invalidate(dev, vq); + + dev = translate_ring_addresses(dev, i); + if (!dev) + return -1; + + *pdev = dev; + } + } + dump_guest_pages(dev); return 0; @@ -857,8 +899,8 @@ vhost_user_get_vring_base(struct virtio_net *dev, dev->flags &= ~VIRTIO_DEV_READY; - /* Here we are safe to get the last used index */ - msg->payload.state.num = vq->last_used_idx; + /* Here we are safe to get the last avail index */ + msg->payload.state.num = vq->last_avail_idx; RTE_LOG(INFO, VHOST_CONFIG, "vring base idx:%d file:%d\n", msg->payload.state.index, @@ -873,6 +915,11 @@ vhost_user_get_vring_base(struct virtio_net *dev, vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD; + if (vq->callfd >= 0) + close(vq->callfd); + + vq->callfd = VIRTIO_UNINITIALIZED_EVENTFD; + if (dev->dequeue_zero_copy) free_zmbufs(vq); rte_free(vq->shadow_used_ring); @@ -967,7 +1014,7 @@ vhost_user_set_log_base(struct virtio_net *dev, struct VhostUserMsg *msg) * mmap from 0 to workaround a hugepage mmap bug: mmap will * fail when offset is not page size aligned. */ - addr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + addr = mmap(0, size + off, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); close(fd); if (addr == MAP_FAILED) { RTE_LOG(ERR, VHOST_CONFIG, "mmap log base failed!\n"); @@ -1382,7 +1429,7 @@ vhost_user_msg_handler(int vid, int fd) break; case VHOST_USER_SET_MEM_TABLE: - ret = vhost_user_set_mem_table(dev, &msg); + ret = vhost_user_set_mem_table(&dev, &msg); break; case VHOST_USER_SET_LOG_BASE: diff --git a/dpdk/lib/librte_vhost/virtio_net.c b/dpdk/lib/librte_vhost/virtio_net.c index a013c07b..bde360f3 100644 --- a/dpdk/lib/librte_vhost/virtio_net.c +++ b/dpdk/lib/librte_vhost/virtio_net.c @@ -107,7 +107,7 @@ do_flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq, rte_memcpy(&vq->used->ring[to], &vq->shadow_used_ring[from], size * sizeof(struct vring_used_elem)); - vhost_log_used_vring(dev, vq, + vhost_log_cache_used_vring(dev, vq, offsetof(struct vring_used, ring[to]), size * sizeof(struct vring_used_elem)); } @@ -135,6 +135,8 @@ flush_shadow_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq) rte_smp_wmb(); + vhost_log_cache_sync(dev, vq); + *(volatile uint16_t *)&vq->used->idx += vq->shadow_used_idx; vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx), sizeof(vq->used->idx)); @@ -159,7 +161,7 @@ do_data_copy_enqueue(struct virtio_net *dev, struct vhost_virtqueue *vq) for (i = 0; i < count; i++) { rte_memcpy(elem[i].dst, elem[i].src, elem[i].len); - vhost_log_write(dev, elem[i].log_addr, elem[i].len); + vhost_log_cache_write(dev, vq, elem[i].log_addr, elem[i].len); PRINT_PACKET(dev, (uintptr_t)elem[i].dst, elem[i].len, 0); } } @@ -275,7 +277,7 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, virtio_enqueue_offload(m, (struct virtio_net_hdr *)(uintptr_t)desc_addr); PRINT_PACKET(dev, (uintptr_t)desc_addr, dev->vhost_hlen, 0); - vhost_log_write(dev, desc_gaddr, dev->vhost_hlen); + vhost_log_cache_write(dev, vq, desc_gaddr, dev->vhost_hlen); } else { struct virtio_net_hdr vnet_hdr; uint64_t remain = dev->vhost_hlen; @@ -297,8 +299,8 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, rte_memcpy((void *)(uintptr_t)dst, (void *)(uintptr_t)src, len); - PRINT_PACKET(dev, (uintptr_t)dst, len, 0); - vhost_log_write(dev, guest_addr, len); + PRINT_PACKET(dev, (uintptr_t)dst, (uint32_t)len, 0); + vhost_log_cache_write(dev, vq, guest_addr, len); remain -= len; guest_addr += len; dst += len; @@ -379,7 +381,8 @@ copy_mbuf_to_desc(struct virtio_net *dev, struct vhost_virtqueue *vq, desc_offset)), rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), cpy_len); - vhost_log_write(dev, desc_gaddr + desc_offset, cpy_len); + vhost_log_cache_write(dev, vq, desc_gaddr + desc_offset, + cpy_len); PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), cpy_len, 0); } else { @@ -468,7 +471,7 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, vq->used->ring[used_idx].id = desc_indexes[i]; vq->used->ring[used_idx].len = pkts[i]->pkt_len + dev->vhost_hlen; - vhost_log_used_vring(dev, vq, + vhost_log_cache_used_vring(dev, vq, offsetof(struct vring_used, ring[used_idx]), sizeof(vq->used->ring[used_idx])); } @@ -528,6 +531,8 @@ virtio_dev_rx(struct virtio_net *dev, uint16_t queue_id, rte_smp_wmb(); + vhost_log_cache_sync(dev, vq); + *(volatile uint16_t *)&vq->used->idx += count; vq->last_used_idx += count; vhost_log_used_vring(dev, vq, @@ -796,8 +801,9 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq, len); PRINT_PACKET(dev, (uintptr_t)dst, - len, 0); - vhost_log_write(dev, guest_addr, len); + (uint32_t)len, 0); + vhost_log_cache_write(dev, vq, + guest_addr, len); remain -= len; guest_addr += len; @@ -806,7 +812,7 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq, } else { PRINT_PACKET(dev, (uintptr_t)hdr_addr, dev->vhost_hlen, 0); - vhost_log_write(dev, hdr_phys_addr, + vhost_log_cache_write(dev, vq, hdr_phys_addr, dev->vhost_hlen); } @@ -820,7 +826,8 @@ copy_mbuf_to_desc_mergeable(struct virtio_net *dev, struct vhost_virtqueue *vq, desc_offset)), rte_pktmbuf_mtod_offset(m, void *, mbuf_offset), cpy_len); - vhost_log_write(dev, desc_gaddr + desc_offset, cpy_len); + vhost_log_cache_write(dev, vq, desc_gaddr + desc_offset, + cpy_len); PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), cpy_len, 0); } else { @@ -1234,7 +1241,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, rte_prefetch0((void *)(uintptr_t)(desc_addr + desc_offset)); PRINT_PACKET(dev, (uintptr_t)(desc_addr + desc_offset), - desc_chunck_len, 0); + (uint32_t)desc_chunck_len, 0); mbuf_offset = 0; mbuf_avail = m->buf_len - RTE_PKTMBUF_HEADROOM; @@ -1322,7 +1329,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, desc_avail = desc->len; PRINT_PACKET(dev, (uintptr_t)desc_addr, - desc_chunck_len, 0); + (uint32_t)desc_chunck_len, 0); } else if (unlikely(desc_chunck_len == 0)) { desc_chunck_len = desc_avail; desc_gaddr += desc_offset; @@ -1337,7 +1344,7 @@ copy_desc_to_mbuf(struct virtio_net *dev, struct vhost_virtqueue *vq, desc_offset = 0; PRINT_PACKET(dev, (uintptr_t)desc_addr, - desc_chunck_len, 0); + (uint32_t)desc_chunck_len, 0); } /* @@ -1384,7 +1391,7 @@ update_used_ring(struct virtio_net *dev, struct vhost_virtqueue *vq, { vq->used->ring[used_idx].id = desc_idx; vq->used->ring[used_idx].len = 0; - vhost_log_used_vring(dev, vq, + vhost_log_cache_used_vring(dev, vq, offsetof(struct vring_used, ring[used_idx]), sizeof(vq->used->ring[used_idx])); } @@ -1399,6 +1406,8 @@ update_used_idx(struct virtio_net *dev, struct vhost_virtqueue *vq, rte_smp_wmb(); rte_smp_rmb(); + vhost_log_cache_sync(dev, vq); + vq->used->idx += count; vhost_log_used_vring(dev, vq, offsetof(struct vring_used, idx), sizeof(vq->used->idx)); @@ -1557,7 +1566,7 @@ rte_vhost_dequeue_burst(int vid, uint16_t queue_id, if (rarp_mbuf == NULL) { RTE_LOG(ERR, VHOST_DATA, "Failed to allocate memory for mbuf.\n"); - return 0; + goto out; } if (make_rarp_packet(rarp_mbuf, &dev->mac)) { diff --git a/dpdk/mk/rte.sdkconfig.mk b/dpdk/mk/rte.sdkconfig.mk index 97363416..59cf6b19 100644 --- a/dpdk/mk/rte.sdkconfig.mk +++ b/dpdk/mk/rte.sdkconfig.mk @@ -64,7 +64,6 @@ notemplate: @echo "use T=template from the following list:" @$(MAKE) -rR showconfigs | sed 's,^, ,' - .PHONY: defconfig defconfig: @$(MAKE) config T=$(shell \ @@ -75,15 +74,25 @@ defconfig: print "arm-armv7a"} \ else if ($$0 == "ppc64") { \ print "ppc_64-power8"} \ + else if ($$0 == "amd64") { \ + print "x86_64-native"} \ else { \ - printf "%s-native", $$0} }')-$(shell \ + printf "%s-native", $$0} }' \ + )-$(shell \ uname | awk '{ \ if ($$0 == "Linux") { \ print "linuxapp"} \ else { \ - print "bsdapp"} }')-$(shell \ - ${CC} -v 2>&1 | \ - grep " version " | cut -d ' ' -f 1) + print "bsdapp"} }' \ + )-$(shell \ + ${CC} --version | grep -o 'cc\|gcc\|icc\|clang' | awk \ + '{ \ + if ($$1 == "cc") { \ + print "gcc" } \ + else { \ + print $$1 } \ + }' \ + ) .PHONY: config ifeq ($(RTE_CONFIG_TEMPLATE),) diff --git a/dpdk/mk/rte.sdkinstall.mk b/dpdk/mk/rte.sdkinstall.mk index 4e97feff..99c4eacb 100644 --- a/dpdk/mk/rte.sdkinstall.mk +++ b/dpdk/mk/rte.sdkinstall.mk @@ -113,18 +113,22 @@ else @echo Installation in $(DESTDIR)$(prefix)/ complete endif +# when installing we want recursive copies preserving timestamps only, no +# preservation of user/group ids or permissions +CP_FLAGS=-dR --preserve=timestamps +TAR_X_FLAGS=--strip-components=1 --keep-newer-files --no-same-owner --no-same-permissions + install-runtime: $(Q)$(call rte_mkdir, $(DESTDIR)$(libdir)) - $(Q)cp -a $O/lib/* $(DESTDIR)$(libdir) + $(Q)cp $(CP_FLAGS) $O/lib/* $(DESTDIR)$(libdir) $(Q)$(call rte_mkdir, $(DESTDIR)$(bindir)) $(Q)tar -cf - -C $O --exclude 'app/*.map' \ --exclude app/dpdk-pmdinfogen \ --exclude 'app/cmdline*' --exclude app/test \ --exclude app/testacl --exclude app/testpipeline app | \ - tar -xf - -C $(DESTDIR)$(bindir) --strip-components=1 \ - --keep-newer-files + tar -xf - -C $(DESTDIR)$(bindir) $(TAR_X_FLAGS) $(Q)$(call rte_mkdir, $(DESTDIR)$(datadir)) - $(Q)cp -a $(RTE_SDK)/usertools $(DESTDIR)$(datadir) + $(Q)cp $(CP_FLAGS) $(RTE_SDK)/usertools $(DESTDIR)$(datadir) $(Q)$(call rte_mkdir, $(DESTDIR)$(sbindir)) $(Q)$(call rte_symlink, $(DESTDIR)$(datadir)/usertools/dpdk-devbind.py, \ $(DESTDIR)$(sbindir)/dpdk-devbind) @@ -132,30 +136,29 @@ install-runtime: $(DESTDIR)$(bindir)/dpdk-pmdinfo) ifneq ($(wildcard $O/doc/man/*/*.1),) $(Q)$(call rte_mkdir, $(DESTDIR)$(mandir)/man1) - $(Q)cp -a $O/doc/man/*/*.1 $(DESTDIR)$(mandir)/man1 + $(Q)cp $(CP_FLAGS) $O/doc/man/*/*.1 $(DESTDIR)$(mandir)/man1 endif ifneq ($(wildcard $O/doc/man/*/*.8),) $(Q)$(call rte_mkdir, $(DESTDIR)$(mandir)/man8) - $(Q)cp -a $O/doc/man/*/*.8 $(DESTDIR)$(mandir)/man8 + $(Q)cp $(CP_FLAGS) $O/doc/man/*/*.8 $(DESTDIR)$(mandir)/man8 endif install-kmod: ifneq ($(wildcard $O/kmod/*),) $(Q)$(call rte_mkdir, $(DESTDIR)$(kerneldir)) - $(Q)cp -a $O/kmod/* $(DESTDIR)$(kerneldir) + $(Q)cp $(CP_FLAGS) $O/kmod/* $(DESTDIR)$(kerneldir) endif install-sdk: $(Q)$(call rte_mkdir, $(DESTDIR)$(includedir)) $(Q)tar -chf - -C $O include | \ - tar -xf - -C $(DESTDIR)$(includedir) --strip-components=1 \ - --keep-newer-files + tar -xf - -C $(DESTDIR)$(includedir) $(TAR_X_FLAGS) $(Q)$(call rte_mkdir, $(DESTDIR)$(sdkdir)) - $(Q)cp -a $(RTE_SDK)/mk $(DESTDIR)$(sdkdir) - $(Q)cp -a $(RTE_SDK)/buildtools $(DESTDIR)$(sdkdir) + $(Q)cp $(CP_FLAGS) $(RTE_SDK)/mk $(DESTDIR)$(sdkdir) + $(Q)cp $(CP_FLAGS) $(RTE_SDK)/buildtools $(DESTDIR)$(sdkdir) $(Q)$(call rte_mkdir, $(DESTDIR)$(targetdir)/app) - $(Q)cp -a $O/.config $(DESTDIR)$(targetdir) - $(Q)cp -a $O/app/dpdk-pmdinfogen $(DESTDIR)$(targetdir)/app + $(Q)cp $(CP_FLAGS) $O/.config $(DESTDIR)$(targetdir) + $(Q)cp $(CP_FLAGS) $O/app/dpdk-pmdinfogen $(DESTDIR)$(targetdir)/app $(Q)$(call rte_symlink, $(DESTDIR)$(includedir), $(DESTDIR)$(targetdir)/include) $(Q)$(call rte_symlink, $(DESTDIR)$(libdir), $(DESTDIR)$(targetdir)/lib) @@ -163,12 +166,11 @@ install-doc: ifneq ($(wildcard $O/doc/html),) $(Q)$(call rte_mkdir, $(DESTDIR)$(docdir)) $(Q)tar -cf - -C $O/doc --exclude 'html/guides/.*' html | \ - tar -xf - -C $(DESTDIR)$(docdir) --strip-components=1 \ - --keep-newer-files + tar -xf - -C $(DESTDIR)$(docdir) $(TAR_X_FLAGS) endif ifneq ($(wildcard $O/doc/*/*/*pdf),) $(Q)$(call rte_mkdir, $(DESTDIR)$(docdir)/guides) - $(Q)cp -a $O/doc/*/*/*pdf $(DESTDIR)$(docdir)/guides + $(Q)cp $(CP_FLAGS) $O/doc/*/*/*pdf $(DESTDIR)$(docdir)/guides endif $(Q)$(call rte_mkdir, $(DESTDIR)$(datadir)) - $(Q)cp -a $(RTE_SDK)/examples $(DESTDIR)$(datadir) + $(Q)cp $(CP_FLAGS) $(RTE_SDK)/examples $(DESTDIR)$(datadir) diff --git a/dpdk/mk/rte.sdkroot.mk b/dpdk/mk/rte.sdkroot.mk index e2e3effa..2fcf8de8 100644 --- a/dpdk/mk/rte.sdkroot.mk +++ b/dpdk/mk/rte.sdkroot.mk @@ -96,8 +96,8 @@ config defconfig showconfigs showversion showversionum: cscope gtags tags etags: $(Q)$(RTE_SDK)/devtools/build-tags.sh $@ $T -.PHONY: test test-basic test-fast test-ring test-mempool test-perf coverage -test test-basic test-fast test-ring test-mempool test-perf coverage: +.PHONY: test test-fast test-perf coverage test-drivers test-dump +test test-fast test-perf coverage test-drivers test-dump: $(Q)$(MAKE) -f $(RTE_SDK)/mk/rte.sdktest.mk $@ test: test-build diff --git a/dpdk/mk/rte.sdktest.mk b/dpdk/mk/rte.sdktest.mk index daeea90f..18e03d6e 100644 --- a/dpdk/mk/rte.sdktest.mk +++ b/dpdk/mk/rte.sdktest.mk @@ -46,14 +46,34 @@ DIR := $(shell basename $(RTE_OUTPUT)) # # test: launch auto-tests, very simple for now. # -.PHONY: test test-basic test-fast test-perf coverage +.PHONY: test test-fast test-perf test-drivers test-dump coverage -PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf -coverage: BLACKLIST=-$(PERFLIST) -test-fast: BLACKLIST=-$(PERFLIST) -test-perf: WHITELIST=$(PERFLIST) +PERFLIST=ring_perf,mempool_perf,memcpy_perf,hash_perf,timer_perf,\ + reciprocal_division,reciprocal_division_perf,lpm_perf,red_all,\ + barrier,hash_multiwriter,timer_racecond,efd,hash_functions,\ + eventdev_selftest_sw,member_perf,efd_perf,lpm6_perf,red_perf,\ + distributor_perf,ring_pmd_perf,pmd_perf,ring_perf +DRIVERSLIST=link_bonding,link_bonding_mode4,link_bonding_rssconf,\ + cryptodev_sw_mrvl,cryptodev_dpaa2_sec,cryptodev_dpaa_sec,\ + cryptodev_qat,cryptodev_aesni_mb,cryptodev_openssl,\ + cryptodev_scheduler,cryptodev_aesni_gcm,cryptodev_null,\ + cryptodev_sw_snow3g,cryptodev_sw_kasumi,cryptodev_sw_zuc +DUMPLIST=dump_struct_sizes,dump_mempool,dump_malloc_stats,dump_devargs,\ + dump_log_types,dump_ring,dump_physmem,dump_memzone -test test-basic test-fast test-perf: +SPACESTR:= +SPACESTR+= +STRIPPED_PERFLIST=$(subst $(SPACESTR),,$(PERFLIST)) +STRIPPED_DRIVERSLIST=$(subst $(SPACESTR),,$(DRIVERSLIST)) +STRIPPED_DUMPLIST=$(subst $(SPACESTR),,$(DUMPLIST)) + +coverage: BLACKLIST=-$(STRIPPED_PERFLIST) +test-fast: BLACKLIST=-$(STRIPPED_PERFLIST),$(STRIPPED_DRIVERSLIST),$(STRIPPED_DUMPLIST) +test-perf: WHITELIST=$(STRIPPED_PERFLIST) +test-drivers: WHITELIST=$(STRIPPED_DRIVERSLIST) +test-dump: WHITELIST=$(STRIPPED_DUMPLIST) + +test test-fast test-perf test-drivers test-dump: @mkdir -p $(AUTOTEST_DIR) ; \ cd $(AUTOTEST_DIR) ; \ if [ -f $(RTE_OUTPUT)/app/test ]; then \ diff --git a/dpdk/mk/toolchain/gcc/rte.toolchain-compat.mk b/dpdk/mk/toolchain/gcc/rte.toolchain-compat.mk index 01ac7e23..8dbd4417 100644 --- a/dpdk/mk/toolchain/gcc/rte.toolchain-compat.mk +++ b/dpdk/mk/toolchain/gcc/rte.toolchain-compat.mk @@ -42,6 +42,11 @@ GCC_MAJOR = $(shell echo __GNUC__ | $(CC) -E -x c - | tail -n 1) GCC_MINOR = $(shell echo __GNUC_MINOR__ | $(CC) -E -x c - | tail -n 1) GCC_VERSION = $(GCC_MAJOR)$(GCC_MINOR) +HOST_GCC_MAJOR = $(shell echo __GNUC__ | $(HOSTCC) -E -x c - | tail -n 1) +HOST_GCC_MINOR = $(shell echo __GNUC_MINOR__ | $(HOSTCC) -E -x c - | tail -n 1) +HOST_GCC_PATCHLEVEL = $(shell echo __GNUC_PATCHLEVEL__ | $(HOSTCC) -E -x c - | tail -n 1) +HOST_GCC_VERSION = $(HOST_GCC_MAJOR)$(HOST_GCC_MINOR) + # if GCC is older than 4.x ifeq ($(shell test $(GCC_VERSION) -lt 40 && echo 1), 1) MACHINE_CFLAGS = diff --git a/dpdk/mk/toolchain/gcc/rte.vars.mk b/dpdk/mk/toolchain/gcc/rte.vars.mk index 3b907e20..e7008c05 100644 --- a/dpdk/mk/toolchain/gcc/rte.vars.mk +++ b/dpdk/mk/toolchain/gcc/rte.vars.mk @@ -99,6 +99,15 @@ ifeq ($(shell test $(GCC_VERSION) -lt 47 && echo 1), 1) WERROR_FLAGS += -Wno-uninitialized endif +HOST_WERROR_FLAGS := $(WERROR_FLAGS) + +ifeq ($(shell test $(HOST_GCC_VERSION) -gt 70 && echo 1), 1) +# Tell GCC only to error for switch fallthroughs without a suitable comment +HOST_WERROR_FLAGS += -Wimplicit-fallthrough=2 +# Ignore errors for snprintf truncation +HOST_WERROR_FLAGS += -Wno-format-truncation +endif + ifeq ($(shell test $(GCC_VERSION) -gt 70 && echo 1), 1) # Tell GCC only to error for switch fallthroughs without a suitable comment WERROR_FLAGS += -Wimplicit-fallthrough=2 diff --git a/dpdk/pkg/dpdk.spec b/dpdk/pkg/dpdk.spec index 8681e4c8..f2486b9a 100644 --- a/dpdk/pkg/dpdk.spec +++ b/dpdk/pkg/dpdk.spec @@ -30,7 +30,7 @@ # OF THE POSSIBILITY OF SUCH DAMAGE. Name: dpdk -Version: 17.11.2 +Version: 17.11.4 Release: 1 Packager: packaging@6wind.com URL: http://dpdk.org diff --git a/dpdk/test/cmdline_test/cmdline_test.py b/dpdk/test/cmdline_test/cmdline_test.py old mode 100644 new mode 100755 diff --git a/dpdk/test/test/autotest_runner.py b/dpdk/test/test/autotest_runner.py index fc882ec0..c6a9105b 100644 --- a/dpdk/test/test/autotest_runner.py +++ b/dpdk/test/test/autotest_runner.py @@ -31,6 +31,7 @@ # The main logic behind running autotests in parallel +from __future__ import print_function import StringIO import csv import multiprocessing @@ -69,7 +70,7 @@ def wait_prompt(child): # quite a bit of effort to make it work). -def run_test_group(cmdline, test_group): +def run_test_group(cmdline, target, test_group): results = [] child = None start_time = time.time() @@ -80,8 +81,8 @@ def run_test_group(cmdline, test_group): # prepare logging of init startuplog = StringIO.StringIO() - print >>startuplog, "\n%s %s\n" % ("=" * 20, test_group["Prefix"]) - print >>startuplog, "\ncmdline=%s" % cmdline + print("\n%s %s\n" % ("=" * 20, test_group["Prefix"]), file=startuplog) + print("\ncmdline=%s" % cmdline, file=startuplog) child = pexpect.spawn(cmdline, logfile=startuplog) @@ -122,13 +123,6 @@ def run_test_group(cmdline, test_group): results.append((0, "Success", "Start %s" % test_group["Prefix"], time.time() - start_time, startuplog.getvalue(), None)) - # parse the binary for available test commands - binary = cmdline.split()[0] - stripped = 'not stripped' not in subprocess.check_output(['file', binary]) - if not stripped: - symbols = subprocess.check_output(['nm', binary]).decode('utf-8') - avail_cmds = re.findall('test_register_(\w+)', symbols) - # run all tests in test group for test in test_group["Tests"]: @@ -145,25 +139,23 @@ def run_test_group(cmdline, test_group): try: # print test name to log buffer - print >>logfile, "\n%s %s\n" % ("-" * 20, test["Name"]) + print("\n%s %s\n" % ("-" * 20, test["Name"]), file=logfile) # run test function associated with the test - if stripped or test["Command"] in avail_cmds: - result = test["Func"](child, test["Command"]) - else: - result = (0, "Skipped [Not Available]") + result = test["Func"](child, test["Command"]) # make a note when the test was finished end_time = time.time() + log = logfile.getvalue() + # append test data to the result tuple - result += (test["Name"], end_time - start_time, - logfile.getvalue()) + result += (test["Name"], end_time - start_time, log) # call report function, if any defined, and supply it with # target and complete log for test run if test["Report"]: - report = test["Report"](self.target, log) + report = test["Report"](target, log) # append report to results tuple result += (report,) @@ -212,8 +204,10 @@ class AutotestRunner: def __init__(self, cmdline, target, blacklist, whitelist): self.cmdline = cmdline self.target = target + self.binary = cmdline.split()[0] self.blacklist = blacklist self.whitelist = whitelist + self.skipped = [] # log file filename logfile = "%s.log" % target @@ -275,7 +269,7 @@ class AutotestRunner: # don't print out total time every line, it's the same anyway if i == len(results) - 1: - print(result, + print(result + "[%02dm %02ds]" % (total_time / 60, total_time % 60)) else: print(result) @@ -302,53 +296,58 @@ class AutotestRunner: if i != 0: self.csvwriter.writerow([test_name, test_result, result_str]) - # this function iterates over test groups and removes each - # test that is not in whitelist/blacklist - def __filter_groups(self, test_groups): - groups_to_remove = [] + # this function checks individual test and decides if this test should be in + # the group by comparing it against whitelist/blacklist. it also checks if + # the test is compiled into the binary, and marks it as skipped if necessary + def __filter_test(self, test): + test_cmd = test["Command"] + test_id = test_cmd - # filter out tests from parallel test groups - for i, test_group in enumerate(test_groups): + # dump tests are specified in full e.g. "Dump_mempool" + if "_autotest" in test_id: + test_id = test_id[:-len("_autotest")] - # iterate over a copy so that we could safely delete individual - # tests - for test in test_group["Tests"][:]: - test_id = test["Command"] + # filter out blacklisted/whitelisted tests + if self.blacklist and test_id in self.blacklist: + return False + if self.whitelist and test_id not in self.whitelist: + return False - # dump tests are specified in full e.g. "Dump_mempool" - if "_autotest" in test_id: - test_id = test_id[:-len("_autotest")] + # if test wasn't compiled in, remove it as well - # filter out blacklisted/whitelisted tests - if self.blacklist and test_id in self.blacklist: - test_group["Tests"].remove(test) - continue - if self.whitelist and test_id not in self.whitelist: - test_group["Tests"].remove(test) - continue + # parse the binary for available test commands + stripped = 'not stripped' not in \ + subprocess.check_output(['file', self.binary]) + if not stripped: + symbols = subprocess.check_output(['nm', + self.binary]).decode('utf-8') + avail_cmds = re.findall('test_register_(\w+)', symbols) - # modify or remove original group - if len(test_group["Tests"]) > 0: - test_groups[i] = test_group - else: - # remember which groups should be deleted - # put the numbers backwards so that we start - # deleting from the end, not from the beginning - groups_to_remove.insert(0, i) + if test_cmd not in avail_cmds: + # notify user + result = 0, "Skipped [Not compiled]", test_id, 0, "", None + self.skipped.append(tuple(result)) + return False - # remove test groups that need to be removed - for i in groups_to_remove: - del test_groups[i] + return True - return test_groups + def __filter_group(self, group): + group["Tests"] = list(filter(self.__filter_test, group["Tests"])) + return len(group["Tests"]) > 0 # iterate over test groups and run tests associated with them def run_all_tests(self): # filter groups - self.parallel_test_groups = \ - self.__filter_groups(self.parallel_test_groups) - self.non_parallel_test_groups = \ - self.__filter_groups(self.non_parallel_test_groups) + # for each test group, check all tests against the filter, then remove + # all groups that don't have any tests + self.parallel_test_groups = list( + filter(self.__filter_group, + self.parallel_test_groups) + ) + self.non_parallel_test_groups = list( + filter(self.__filter_group, + self.non_parallel_test_groups) + ) # create a pool of worker threads pool = multiprocessing.Pool(processes=1) @@ -360,17 +359,36 @@ class AutotestRunner: # create table header print("") - print("Test name".ljust(30), "Test result".ljust(29), - "Test".center(9), "Total".center(9)) + print("Test name".ljust(30) + "Test result".ljust(29) + + "Test".center(9) + "Total".center(9)) print("=" * 80) + # print out skipped autotests if there were any + if len(self.skipped): + print("Skipped autotests:") + + # print out any skipped tests + for result in self.skipped: + # unpack result tuple + test_result, result_str, test_name, _, _, _ = result + self.csvwriter.writerow([test_name, test_result, + result_str]) + + t = ("%s:" % test_name).ljust(30) + t += result_str.ljust(29) + t += "[00m 00s]" + + print(t) + # make a note of tests start time self.start = time.time() + print("Parallel autotests:") # assign worker threads to run test groups for test_group in self.parallel_test_groups: result = pool.apply_async(run_test_group, [self.__get_cmdline(test_group), + self.target, test_group]) results.append(result) @@ -392,10 +410,11 @@ class AutotestRunner: # remove result from results list once we're done with it results.remove(group_result) + print("Non-parallel autotests:") # run non_parallel tests. they are run one by one, synchronously for test_group in self.non_parallel_test_groups: group_result = run_test_group( - self.__get_cmdline(test_group), test_group) + self.__get_cmdline(test_group), self.target, test_group) self.__process_results(group_result) diff --git a/dpdk/test/test/test_cryptodev.c b/dpdk/test/test/test_cryptodev.c index 7997b488..28f982f4 100644 --- a/dpdk/test/test/test_cryptodev.c +++ b/dpdk/test/test/test_cryptodev.c @@ -567,7 +567,7 @@ test_device_configure_invalid_dev_id(void) dev_id = ts_params->valid_devs[ts_params->valid_dev_count - 1]; /* Stop the device in case it's started so it can be configured */ - rte_cryptodev_stop(ts_params->valid_devs[dev_id]); + rte_cryptodev_stop(dev_id); TEST_ASSERT_SUCCESS(rte_cryptodev_configure(dev_id, &ts_params->conf), "Failed test for rte_cryptodev_configure: " @@ -6484,7 +6484,7 @@ test_multi_session_random_usage(void) sessions[i] = rte_cryptodev_sym_session_create( ts_params->session_mpool); - rte_memcpy(&ut_paramz[i].ut_params, &testsuite_params, + rte_memcpy(&ut_paramz[i].ut_params, &unittest_params, sizeof(struct crypto_unittest_params)); test_AES_CBC_HMAC_SHA512_decrypt_create_session_params( diff --git a/dpdk/test/test/test_distributor_perf.c b/dpdk/test/test/test_distributor_perf.c index 7d69887b..4a62a922 100644 --- a/dpdk/test/test/test_distributor_perf.c +++ b/dpdk/test/test/test_distributor_perf.c @@ -60,7 +60,7 @@ struct worker_stats worker_stats[RTE_MAX_LCORE]; * worker thread used for testing the time to do a round-trip of a cache * line between two cores and back again */ -static void +static int flip_bit(volatile uint64_t *arg) { uint64_t old_val = 0; @@ -70,6 +70,7 @@ flip_bit(volatile uint64_t *arg) old_val = *arg; *arg = 0; } + return 0; } /* diff --git a/dpdk/test/test/test_eal_flags.c b/dpdk/test/test/test_eal_flags.c index 310109e6..e82bb963 100644 --- a/dpdk/test/test/test_eal_flags.c +++ b/dpdk/test/test/test_eal_flags.c @@ -61,7 +61,7 @@ #define memtest "memtest" #define memtest1 "memtest1" #define memtest2 "memtest2" -#define SOCKET_MEM_STRLEN (RTE_MAX_NUMA_NODES * 10) +#define SOCKET_MEM_STRLEN (RTE_MAX_NUMA_NODES * 20) #define launch_proc(ARGV) process_dup(ARGV, \ sizeof(ARGV)/(sizeof(ARGV[0])), __func__) @@ -404,17 +404,17 @@ test_invalid_vdev_flag(void) #endif /* Test with invalid vdev option */ - const char *vdevinval[] = {prgname, prefix, "-n", "1", + const char *vdevinval[] = {prgname, prefix, no_huge, "-n", "1", "-c", "1", vdev, "eth_dummy"}; /* Test with valid vdev option */ - const char *vdevval1[] = {prgname, prefix, "-n", "1", + const char *vdevval1[] = {prgname, prefix, no_huge, "-n", "1", "-c", "1", vdev, "net_ring0"}; - const char *vdevval2[] = {prgname, prefix, "-n", "1", + const char *vdevval2[] = {prgname, prefix, no_huge, "-n", "1", "-c", "1", vdev, "net_ring0,args=test"}; - const char *vdevval3[] = {prgname, prefix, "-n", "1", + const char *vdevval3[] = {prgname, prefix, no_huge, "-n", "1", "-c", "1", vdev, "net_ring0,nodeaction=r1:0:CREATE"}; if (launch_proc(vdevinval) == 0) { @@ -877,13 +877,10 @@ test_misc_flags(void) const char *argv4[] = {prgname, prefix, mp_flag, "-c", "1", "--syslog"}; /* With invalid --syslog */ const char *argv5[] = {prgname, prefix, mp_flag, "-c", "1", "--syslog", "error"}; - /* With no-sh-conf */ + /* With no-sh-conf, also use no-huge to ensure this test runs on BSD */ const char *argv6[] = {prgname, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE, - no_shconf, nosh_prefix }; + no_shconf, nosh_prefix, no_huge}; -#ifdef RTE_EXEC_ENV_BSDAPP - return 0; -#endif /* With --huge-dir */ const char *argv7[] = {prgname, "-c", "1", "-n", "2", "-m", DEFAULT_MEM_SIZE, "--file-prefix=hugedir", "--huge-dir", hugepath}; @@ -917,6 +914,7 @@ test_misc_flags(void) const char *argv15[] = {prgname, "--file-prefix=intr", "-c", "1", "-n", "2", "--vfio-intr=invalid"}; + /* run all tests also applicable to FreeBSD first */ if (launch_proc(argv0) == 0) { printf("Error - process ran ok with invalid flag\n"); @@ -930,6 +928,16 @@ test_misc_flags(void) printf("Error - process did not run ok with -v flag\n"); return -1; } + if (launch_proc(argv6) != 0) { + printf("Error - process did not run ok with --no-shconf flag\n"); + return -1; + } + +#ifdef RTE_EXEC_ENV_BSDAPP + /* no more tests to be done on FreeBSD */ + return 0; +#endif + if (launch_proc(argv3) != 0) { printf("Error - process did not run ok with --syslog flag\n"); return -1; @@ -942,13 +950,6 @@ test_misc_flags(void) printf("Error - process run ok with invalid --syslog flag\n"); return -1; } - if (launch_proc(argv6) != 0) { - printf("Error - process did not run ok with --no-shconf flag\n"); - return -1; - } -#ifdef RTE_EXEC_ENV_BSDAPP - return 0; -#endif if (launch_proc(argv7) != 0) { printf("Error - process did not run ok with --huge-dir flag\n"); return -1; @@ -1166,10 +1167,11 @@ test_memory_flags(void) #ifdef RTE_EXEC_ENV_BSDAPP int i, num_sockets = 1; #else - int i, num_sockets = get_number_of_sockets(); + int i, num_sockets = RTE_MIN(get_number_of_sockets(), + RTE_MAX_NUMA_NODES); #endif - if (num_sockets <= 0 || num_sockets > RTE_MAX_NUMA_NODES) { + if (num_sockets <= 0) { printf("Error - cannot get number of sockets!\n"); return -1; } diff --git a/dpdk/test/test/test_flow_classify.c b/dpdk/test/test/test_flow_classify.c index 9f331cd8..a8a1cad4 100644 --- a/dpdk/test/test/test_flow_classify.c +++ b/dpdk/test/test/test_flow_classify.c @@ -641,32 +641,32 @@ test_flow_classify(void) printf("Line %i: f_create has failed!\n", __LINE__); rte_flow_classifier_free(cls->cls); rte_free(cls); - return -1; + return TEST_FAILED; } printf("Created table_acl for for IPv4 five tuple packets\n"); ret = init_mbufpool(); if (ret) { printf("Line %i: init_mbufpool has failed!\n", __LINE__); - return -1; + return TEST_FAILED; } if (test_invalid_parameters() < 0) - return -1; + return TEST_FAILED; if (test_valid_parameters() < 0) - return -1; + return TEST_FAILED; if (test_invalid_patterns() < 0) - return -1; + return TEST_FAILED; if (test_invalid_actions() < 0) - return -1; + return TEST_FAILED; if (test_query_udp() < 0) - return -1; + return TEST_FAILED; if (test_query_tcp() < 0) - return -1; + return TEST_FAILED; if (test_query_sctp() < 0) - return -1; + return TEST_FAILED; - return 0; + return TEST_SUCCESS; } REGISTER_TEST_COMMAND(flow_classify_autotest, test_flow_classify); diff --git a/dpdk/test/test/test_hash_multiwriter.c b/dpdk/test/test/test_hash_multiwriter.c index 4dcbd9d5..0e9ac716 100644 --- a/dpdk/test/test/test_hash_multiwriter.c +++ b/dpdk/test/test/test_hash_multiwriter.c @@ -76,18 +76,29 @@ static rte_atomic64_t ginsertions; static int use_htm; static int -test_hash_multiwriter_worker(__attribute__((unused)) void *arg) +test_hash_multiwriter_worker(void *arg) { uint64_t i, offset; + uint16_t pos_core; uint32_t lcore_id = rte_lcore_id(); uint64_t begin, cycles; + uint16_t *enabled_core_ids = (uint16_t *)arg; - offset = (lcore_id - rte_get_master_lcore()) - * tbl_multiwriter_test_params.nb_tsx_insertion; + for (pos_core = 0; pos_core < rte_lcore_count(); pos_core++) { + if (enabled_core_ids[pos_core] == lcore_id) + break; + } + + /* + * Calculate offset for entries based on the position of the + * logical core, from the master core (not counting not enabled cores) + */ + offset = pos_core * tbl_multiwriter_test_params.nb_tsx_insertion; printf("Core #%d inserting %d: %'"PRId64" - %'"PRId64"\n", lcore_id, tbl_multiwriter_test_params.nb_tsx_insertion, - offset, offset + tbl_multiwriter_test_params.nb_tsx_insertion); + offset, + offset + tbl_multiwriter_test_params.nb_tsx_insertion - 1); begin = rte_rdtsc_precise(); @@ -116,6 +127,8 @@ test_hash_multiwriter(void) { unsigned int i, rounded_nb_total_tsx_insertion; static unsigned calledCount = 1; + uint16_t enabled_core_ids[RTE_MAX_LCORE]; + uint16_t core_id; uint32_t *keys; uint32_t *found; @@ -168,16 +181,17 @@ test_hash_multiwriter(void) goto err1; } + for (i = 0; i < nb_entries; i++) + keys[i] = i; + + tbl_multiwriter_test_params.keys = keys; + found = rte_zmalloc(NULL, sizeof(uint32_t) * nb_entries, 0); if (found == NULL) { printf("RTE_ZMALLOC failed\n"); goto err2; } - for (i = 0; i < nb_entries; i++) - keys[i] = i; - - tbl_multiwriter_test_params.keys = keys; tbl_multiwriter_test_params.found = found; rte_atomic64_init(&gcycles); @@ -186,9 +200,27 @@ test_hash_multiwriter(void) rte_atomic64_init(&ginsertions); rte_atomic64_clear(&ginsertions); + /* Get list of enabled cores */ + i = 0; + for (core_id = 0; core_id < RTE_MAX_LCORE; core_id++) { + if (i == rte_lcore_count()) + break; + + if (rte_lcore_is_enabled(core_id)) { + enabled_core_ids[i] = core_id; + i++; + } + } + + if (i != rte_lcore_count()) { + printf("Number of enabled cores in list is different from " + "number given by rte_lcore_count()\n"); + goto err3; + } + /* Fire all threads. */ rte_eal_mp_remote_launch(test_hash_multiwriter_worker, - NULL, CALL_MASTER); + enabled_core_ids, CALL_MASTER); rte_eal_mp_wait_lcore(); while (rte_hash_iterate(handle, &next_key, &next_data, &iter) >= 0) { diff --git a/dpdk/test/test/test_mempool.c b/dpdk/test/test/test_mempool.c index 37ead503..56b0fd02 100644 --- a/dpdk/test/test/test_mempool.c +++ b/dpdk/test/test/test_mempool.c @@ -355,17 +355,17 @@ test_mempool_sp_sc(void) } if (rte_mempool_lookup("test_mempool_sp_sc") != mp_spsc) { printf("Cannot lookup mempool from its name\n"); - rte_mempool_free(mp_spsc); - RET_ERR(); + ret = -1; + goto err; } lcore_next = rte_get_next_lcore(lcore_id, 0, 1); if (lcore_next >= RTE_MAX_LCORE) { - rte_mempool_free(mp_spsc); - RET_ERR(); + ret = -1; + goto err; } if (rte_eal_lcore_role(lcore_next) != ROLE_RTE) { - rte_mempool_free(mp_spsc); - RET_ERR(); + ret = -1; + goto err; } rte_spinlock_init(&scsp_spinlock); memset(scsp_obj_table, 0, sizeof(scsp_obj_table)); @@ -376,7 +376,10 @@ test_mempool_sp_sc(void) if (rte_eal_wait_lcore(lcore_next) < 0) ret = -1; + +err: rte_mempool_free(mp_spsc); + mp_spsc = NULL; return ret; } diff --git a/dpdk/test/test/test_pmd_ring.c b/dpdk/test/test/test_pmd_ring.c index 2cdf60d1..8d5813cf 100644 --- a/dpdk/test/test/test_pmd_ring.c +++ b/dpdk/test/test/test_pmd_ring.c @@ -247,6 +247,8 @@ test_pmd_ring_pair_create_attach(int portd, int porte) struct rte_mbuf buf, *pbuf = &buf; struct rte_eth_conf null_conf; + memset(&null_conf, 0, sizeof(struct rte_eth_conf)); + if ((rte_eth_dev_configure(portd, 1, 1, &null_conf) < 0) || (rte_eth_dev_configure(porte, 1, 1, &null_conf) < 0)) { printf("Configure failed for port\n"); diff --git a/dpdk/test/test/test_reorder.c b/dpdk/test/test/test_reorder.c index 429f6eb4..6f2c2306 100644 --- a/dpdk/test/test/test_reorder.c +++ b/dpdk/test/test/test_reorder.c @@ -175,11 +175,11 @@ test_reorder_insert(void) b = rte_reorder_create("test_insert", rte_socket_id(), size); TEST_ASSERT_NOT_NULL(b, "Failed to create reorder buffer"); - ret = rte_mempool_get_bulk(p, (void *)bufs, num_bufs); - TEST_ASSERT_SUCCESS(ret, "Error getting mbuf from pool"); - - for (i = 0; i < num_bufs; i++) + for (i = 0; i < num_bufs; i++) { + bufs[i] = rte_pktmbuf_alloc(p); + TEST_ASSERT_NOT_NULL(bufs[i], "Packet allocation failed\n"); bufs[i]->seqn = i; + } /* This should fill up order buffer: * reorder_seq = 0 @@ -194,6 +194,7 @@ test_reorder_insert(void) ret = -1; goto exit; } + bufs[i] = NULL; } /* early packet - should move mbufs to ready buf and move sequence window @@ -208,6 +209,7 @@ test_reorder_insert(void) ret = -1; goto exit; } + bufs[4] = NULL; /* early packet from current sequence window - full ready buffer */ bufs[5]->seqn = 2 * size; @@ -218,6 +220,7 @@ test_reorder_insert(void) ret = -1; goto exit; } + bufs[5] = NULL; /* late packet */ bufs[6]->seqn = 3 * size; @@ -228,11 +231,15 @@ test_reorder_insert(void) ret = -1; goto exit; } + bufs[6] = NULL; ret = 0; exit: - rte_mempool_put_bulk(p, (void *)bufs, num_bufs); rte_reorder_free(b); + for (i = 0; i < num_bufs; i++) { + if (bufs[i] != NULL) + rte_pktmbuf_free(bufs[i]); + } return ret; } @@ -248,6 +255,10 @@ test_reorder_drain(void) int ret = 0; unsigned i, cnt; + /* initialize all robufs to NULL */ + for (i = 0; i < num_bufs; i++) + robufs[i] = NULL; + /* This would create a reorder buffer instance consisting of: * reorder_seq = 0 * ready_buf: RB[size] = {NULL, NULL, NULL, NULL} @@ -256,9 +267,6 @@ test_reorder_drain(void) b = rte_reorder_create("test_drain", rte_socket_id(), size); TEST_ASSERT_NOT_NULL(b, "Failed to create reorder buffer"); - ret = rte_mempool_get_bulk(p, (void *)bufs, num_bufs); - TEST_ASSERT_SUCCESS(ret, "Error getting mbuf from pool"); - /* Check no drained packets if reorder is empty */ cnt = rte_reorder_drain(b, robufs, 1); if (cnt != 0) { @@ -268,8 +276,11 @@ test_reorder_drain(void) goto exit; } - for (i = 0; i < num_bufs; i++) + for (i = 0; i < num_bufs; i++) { + bufs[i] = rte_pktmbuf_alloc(p); + TEST_ASSERT_NOT_NULL(bufs[i], "Packet allocation failed\n"); bufs[i]->seqn = i; + } /* Insert packet with seqn 1: * reorder_seq = 0 @@ -277,6 +288,7 @@ test_reorder_drain(void) * OB[] = {1, NULL, NULL, NULL} */ rte_reorder_insert(b, bufs[1]); + bufs[1] = NULL; cnt = rte_reorder_drain(b, robufs, 1); if (cnt != 1) { @@ -285,6 +297,8 @@ test_reorder_drain(void) ret = -1; goto exit; } + if (robufs[0] != NULL) + rte_pktmbuf_free(robufs[i]); /* Insert more packets * RB[] = {NULL, NULL, NULL, NULL} @@ -292,18 +306,22 @@ test_reorder_drain(void) */ rte_reorder_insert(b, bufs[2]); rte_reorder_insert(b, bufs[3]); + bufs[2] = NULL; + bufs[3] = NULL; /* Insert more packets * RB[] = {NULL, NULL, NULL, NULL} * OB[] = {NULL, 2, 3, 4} */ rte_reorder_insert(b, bufs[4]); + bufs[4] = NULL; /* Insert more packets * RB[] = {2, 3, 4, NULL} * OB[] = {NULL, NULL, 7, NULL} */ rte_reorder_insert(b, bufs[7]); + bufs[7] = NULL; /* drained expected packets */ cnt = rte_reorder_drain(b, robufs, 4); @@ -313,6 +331,10 @@ test_reorder_drain(void) ret = -1; goto exit; } + for (i = 0; i < 3; i++) { + if (robufs[i] != NULL) + rte_pktmbuf_free(robufs[i]); + } /* * RB[] = {NULL, NULL, NULL, NULL} @@ -327,8 +349,13 @@ test_reorder_drain(void) } ret = 0; exit: - rte_mempool_put_bulk(p, (void *)bufs, num_bufs); rte_reorder_free(b); + for (i = 0; i < num_bufs; i++) { + if (bufs[i] != NULL) + rte_pktmbuf_free(bufs[i]); + if (robufs[i] != NULL) + rte_pktmbuf_free(robufs[i]); + } return ret; } diff --git a/dpdk/test/test/test_table_pipeline.c b/dpdk/test/test/test_table_pipeline.c index a6fef721..fcd3f374 100644 --- a/dpdk/test/test/test_table_pipeline.c +++ b/dpdk/test/test/test_table_pipeline.c @@ -98,9 +98,9 @@ rte_pipeline_table_action_handler_hit table_action_stub_hit(struct rte_pipeline *p, struct rte_mbuf **pkts, uint64_t pkts_mask, struct rte_pipeline_table_entry **entry, void *arg); -rte_pipeline_table_action_handler_miss +static int table_action_stub_miss(struct rte_pipeline *p, struct rte_mbuf **pkts, - uint64_t pkts_mask, struct rte_pipeline_table_entry **entry, void *arg); + uint64_t pkts_mask, struct rte_pipeline_table_entry *entry, void *arg); rte_pipeline_table_action_handler_hit table_action_0x00(__attribute__((unused)) struct rte_pipeline *p, @@ -130,11 +130,11 @@ table_action_stub_hit(__attribute__((unused)) struct rte_pipeline *p, return 0; } -rte_pipeline_table_action_handler_miss +static int table_action_stub_miss(struct rte_pipeline *p, __attribute__((unused)) struct rte_mbuf **pkts, uint64_t pkts_mask, - __attribute__((unused)) struct rte_pipeline_table_entry **entry, + __attribute__((unused)) struct rte_pipeline_table_entry *entry, __attribute__((unused)) void *arg) { printf("STUB Table Action Miss - setting mask to 0x%"PRIx64"\n", @@ -546,8 +546,7 @@ test_table_pipeline(void) /* TEST - one packet per port */ action_handler_hit = NULL; - action_handler_miss = - (rte_pipeline_table_action_handler_miss) table_action_stub_miss; + action_handler_miss = table_action_stub_miss; table_entry_default_action = RTE_PIPELINE_ACTION_PORT; override_miss_mask = 0x01; /* one packet per port */ setup_pipeline(e_TEST_STUB); @@ -582,8 +581,7 @@ test_table_pipeline(void) printf("TEST - two tables, hitmask override to 0x01\n"); connect_miss_action_to_table = 1; - action_handler_miss = - (rte_pipeline_table_action_handler_miss)table_action_stub_miss; + action_handler_miss = table_action_stub_miss; override_miss_mask = 0x01; setup_pipeline(e_TEST_STUB); if (test_pipeline_single_filter(e_TEST_STUB, 2) < 0) From 8891572376e7b0a52a7e6fdc0de614ea48cf049d Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Mon, 26 Nov 2018 21:48:12 +0800 Subject: [PATCH 4/7] Fix crash bug of DPDK's driver when working on multi_process with AWS --- dpdk/drivers/net/ena/ena_ethdev.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dpdk/drivers/net/ena/ena_ethdev.c b/dpdk/drivers/net/ena/ena_ethdev.c index 4e526567..25aa3653 100644 --- a/dpdk/drivers/net/ena/ena_ethdev.c +++ b/dpdk/drivers/net/ena/ena_ethdev.c @@ -1278,7 +1278,8 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) static int adapters_found; - memset(adapter, 0, sizeof(struct ena_adapter)); + // Temporary modification for multi process + // memset(adapter, 0, sizeof(struct ena_adapter)); ena_dev = &adapter->ena_dev; eth_dev->dev_ops = &ena_dev_ops; From c4c89eff93ed815d672af1d304b2090f172fec37 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Tue, 27 Nov 2018 02:02:38 +0800 Subject: [PATCH 5/7] CONTAINER(DOCKER): support aggregation model while running in container with OVS-DPDK, refer to #298. --- config.ini | 19 +++++++++++ lib/ff_config.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/ff_config.h | 17 ++++++++- 3 files changed, 126 insertions(+), 1 deletion(-) diff --git a/config.ini b/config.ini index 0a5a6bea..e935267a 100644 --- a/config.ini +++ b/config.ini @@ -36,6 +36,9 @@ idle_sleep=0 # 1-3,4,7 ports 1,2,3,4,7 are enabled port_list=0 +# Number of vdev. +nb_vdev=0 + # Port config section # Correspond to dpdk.port_list's index: port0, port1... [port0] @@ -51,6 +54,22 @@ gateway=192.168.1.1 # Packet capture path, this will hurt performance #pcap=./a.pcap +# Vdev config section +# orrespond to dpdk.nb_vdev's index: vdev0, vdev1... +# iface : Shouldn't set always. +# path : The vuser device path in container. Required. +# queues : The max queues of vuser. Optional, default 1, greater or equal to the number of processes. +# queue_size : Queue size.Optional, default 256. +# mac : The mac address of vuser. Optional, default random, if vhost use phy NIC, it should be set to the phy NIC's mac. +# cq : Optional, if queues = 1, default 0; if queues > 1 default 1. +#[vdev0] +##iface=/usr/local/var/run/openvswitch/vhost-user0 +#path=/var/run/openvswitch/vhost-user0 +#queues=1 +#queue_size=256 +#mac=00:00:00:00:00:01 +#cq=0 + # Kni config: if enabled and method=reject, # all packets that do not belong to the following tcp_port and udp_port # will transmit to kernel; if method=accept, all packets that belong to diff --git a/lib/ff_config.c b/lib/ff_config.c index d9a2f6f6..b3183e65 100644 --- a/lib/ff_config.c +++ b/lib/ff_config.c @@ -402,6 +402,61 @@ port_cfg_handler(struct ff_config *cfg, const char *section, return 1; } +static int +vdev_cfg_handler(struct ff_config *cfg, const char *section, + const char *name, const char *value) { + + if (cfg->dpdk.nb_vdev == 0) { + fprintf(stderr, "vdev_cfg_handler: must config dpdk.nb_vdev first\n"); + return 0; + } + + if (cfg->dpdk.vdev_cfgs == NULL) { + struct ff_vdev_cfg *vc = calloc(RTE_MAX_ETHPORTS, sizeof(struct ff_vdev_cfg)); + if (vc == NULL) { + fprintf(stderr, "vdev_cfg_handler malloc failed\n"); + return 0; + } + cfg->dpdk.vdev_cfgs = vc; + } + + int vdevid; + int ret = sscanf(section, "vdev%d", &vdevid); + if (ret != 1) { + fprintf(stderr, "vdev_cfg_handler section[%s] error\n", section); + return 0; + } + + /* just return true if vdevid >= nb_vdev because it has no effect */ + if (vdevid > cfg->dpdk.nb_vdev) { + fprintf(stderr, "vdev_cfg_handler section[%s] bigger than max vdev id\n", section); + return 1; + } + + struct ff_vdev_cfg *cur = &cfg->dpdk.vdev_cfgs[vdevid]; + if (cur->name == NULL) { + cur->name = strdup(section); + cur->vdev_id = vdevid; + } + + if (strcmp(name, "iface") == 0) { + cur->iface = strdup(value); + } else if (strcmp(name, "path") == 0) { + cur->path = strdup(value); + } else if (strcmp(name, "queues") == 0) { + cur->nb_queues = atoi(value); + } else if (strcmp(name, "queue_size") == 0) { + cur->queue_size = atoi(value); + } else if (strcmp(name, "mac") == 0) { + cur->mac = strdup(value); + } else if (strcmp(name, "cq") == 0) { + cur->nb_cq = atoi(value); + } + + return 1; +} + + static int ini_parse_handler(void* user, const char* section, const char* name, const char* value) @@ -424,6 +479,8 @@ ini_parse_handler(void* user, const char* section, const char* name, pconfig->dpdk.base_virtaddr= strdup(value); } else if (MATCH("dpdk", "port_list")) { return parse_port_list(pconfig, value); + } else if (MATCH("dpdk", "nb_vdev")) { + pconfig->dpdk.nb_vdev = atoi(value); } else if (MATCH("dpdk", "promiscuous")) { pconfig->dpdk.promiscuous = atoi(value); } else if (MATCH("dpdk", "numa_on")) { @@ -456,6 +513,8 @@ ini_parse_handler(void* user, const char* section, const char* name, return freebsd_conf_handler(pconfig, "sysctl", name, value); } else if (strncmp(section, "port", 4) == 0) { return port_cfg_handler(pconfig, section, name, value); + } else if (strncmp(section, "vdev", 4) == 0) { + return vdev_cfg_handler(pconfig, section, name, value); } return 1; @@ -492,8 +551,40 @@ dpdk_args_setup(struct ff_config *cfg) dpdk_argv[n++] = strdup(temp); } + if (cfg->dpdk.nb_vdev) { + for (i=0; idpdk.nb_vdev; i++) { + sprintf(temp, "--vdev=virtio_user%d,path=%s", + cfg->dpdk.vdev_cfgs[i].vdev_id, + cfg->dpdk.vdev_cfgs[i].path); + if (cfg->dpdk.vdev_cfgs[i].nb_queues) { + sprintf(temp, "%s,queues=%u", + temp, cfg->dpdk.vdev_cfgs[i].nb_queues); + } + if (cfg->dpdk.vdev_cfgs[i].nb_cq) { + sprintf(temp, "%s,cq=%u", + temp, cfg->dpdk.vdev_cfgs[i].nb_cq); + } + if (cfg->dpdk.vdev_cfgs[i].queue_size) { + sprintf(temp, "%s,queue_size=%u", + temp, cfg->dpdk.vdev_cfgs[i].queue_size); + } + if (cfg->dpdk.vdev_cfgs[i].mac) { + sprintf(temp, "%s,mac=%s", + temp, cfg->dpdk.vdev_cfgs[i].mac); + } + dpdk_argv[n++] = strdup(temp); + } + sprintf(temp, "--no-pci"); + dpdk_argv[n++] = strdup(temp); + sprintf(temp, "--file-prefix=container"); + dpdk_argv[n++] = strdup(temp); + } + dpdk_argc = n; + for (i=0; i struct ff_port_cfg*) struct ff_port_cfg *port_cfgs; + struct ff_vdev_cfg *vdev_cfgs; } dpdk; struct { From 9da1cd96481cc2d533ae3d65ffae56d5c129a7d0 Mon Sep 17 00:00:00 2001 From: fengbojiang Date: Tue, 27 Nov 2018 02:05:10 +0800 Subject: [PATCH 6/7] // CONTAINER(DOCKER): Fixed virtio_user can't run in multi-progress mode. --- dpdk/drivers/net/virtio/virtio_user_ethdev.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dpdk/drivers/net/virtio/virtio_user_ethdev.c b/dpdk/drivers/net/virtio/virtio_user_ethdev.c index 7be57ce6..48700afd 100644 --- a/dpdk/drivers/net/virtio/virtio_user_ethdev.c +++ b/dpdk/drivers/net/virtio/virtio_user_ethdev.c @@ -511,6 +511,8 @@ virtio_user_pmd_probe(struct rte_vdev_device *dev) eth_dev = rte_eth_dev_attach_secondary(rte_vdev_device_name(dev)); if (!eth_dev) goto end; + // CONTAINER(DOCKER): Fixed virtio_user can't run in multi-progress mode. + eth_dev->device = &dev->device; } /* previously called by rte_pci_probe() for physical dev */ From 8850115bd87ac2e168ca2ab33159528818d45fe8 Mon Sep 17 00:00:00 2001 From: jfb8856606 Date: Thu, 6 Dec 2018 22:17:51 +0800 Subject: [PATCH 7/7] DPDK:upgrade to 18.11.0 LTS. --- dpdk/.gitignore | 1 + dpdk/GNUmakefile | 34 +- dpdk/LICENSE.GPL | 339 - dpdk/LICENSE.LGPL | 502 - dpdk/MAINTAINERS | 424 +- dpdk/Makefile | 32 +- dpdk/README | 4 +- dpdk/app/Makefile | 38 +- dpdk/app/meson.build | 62 + dpdk/app/pdump/Makefile | 33 +- dpdk/app/pdump/main.c | 199 +- dpdk/app/pdump/meson.build | 6 + dpdk/app/proc-info/Makefile | 15 + dpdk/app/proc-info/main.c | 672 + dpdk/app/proc-info/meson.build | 6 + dpdk/app/proc_info/Makefile | 42 - dpdk/app/proc_info/main.c | 693 - dpdk/app/test-bbdev/Makefile | 25 + dpdk/app/test-bbdev/main.c | 326 + dpdk/app/test-bbdev/main.h | 120 + dpdk/app/test-bbdev/meson.build | 9 + dpdk/app/test-bbdev/test-bbdev.py | 111 + dpdk/app/test-bbdev/test_bbdev.c | 1371 + dpdk/app/test-bbdev/test_bbdev_perf.c | 2399 ++ dpdk/app/test-bbdev/test_bbdev_vector.c | 939 + dpdk/app/test-bbdev/test_bbdev_vector.h | 71 + .../test-bbdev/test_vectors/bbdev_null.data | 5 + ...turbo_dec_c1_k40_r0_e17280_sbd_negllr.data | 57 + ..._r0_e10376_crc24b_sbd_negllr_high_snr.data | 643 + ...4_r0_e10376_crc24b_sbd_negllr_low_snr.data | 643 + .../turbo_dec_c1_k6144_r0_e34560_negllr.data | 645 + .../turbo_dec_c1_k6144_r0_e34560_posllr.data | 645 + ...rbo_dec_c1_k6144_r0_e34560_sbd_negllr.data | 1224 + ...rbo_dec_c1_k6144_r0_e34560_sbd_posllr.data | 1225 + ...urbo_dec_c2_k3136_r0_e4920_sbd_negllr.data | 676 + ...c_c2_k3136_r0_e4920_sbd_negllr_crc24b.data | 677 + .../turbo_enc_c1_k40_r0_e1190_rm.data | 36 + .../turbo_enc_c1_k40_r0_e1194_rm.data | 36 + .../turbo_enc_c1_k40_r0_e1196_rm.data | 36 + .../turbo_enc_c1_k40_r0_e272_rm.data | 33 + .../turbo_enc_c1_k6144_r0_e120_rm_rvidx.data | 63 + .../turbo_enc_c1_k6144_r0_e18444.data | 156 + .../turbo_enc_c1_k6144_r0_e18448_crc24a.data | 159 + ...urbo_enc_c1_k6144_r0_e32256_crc24b_rm.data | 180 + .../turbo_enc_c2_k5952_r0_e17868_crc24b.data | 300 + .../turbo_enc_c3_k4800_r2_e14412_crc24b.data | 153 + .../turbo_enc_c4_k4800_r2_e14412_crc24b.data | 252 + dpdk/app/test-bbdev/turbo_dec_default.data | 1 + dpdk/app/test-bbdev/turbo_enc_default.data | 1 + dpdk/app/test-crypto-perf/Makefile | 33 +- dpdk/app/test-crypto-perf/cperf.h | 32 +- dpdk/app/test-crypto-perf/cperf_ops.c | 109 +- dpdk/app/test-crypto-perf/cperf_ops.h | 34 +- dpdk/app/test-crypto-perf/cperf_options.h | 12 +- .../test-crypto-perf/cperf_options_parsing.c | 93 +- dpdk/app/test-crypto-perf/cperf_test_common.c | 71 +- dpdk/app/test-crypto-perf/cperf_test_common.h | 32 +- .../app/test-crypto-perf/cperf_test_latency.c | 38 +- .../app/test-crypto-perf/cperf_test_latency.h | 32 +- .../cperf_test_pmd_cyclecount.c | 46 +- .../cperf_test_pmd_cyclecount.h | 32 +- .../test-crypto-perf/cperf_test_throughput.c | 37 +- .../test-crypto-perf/cperf_test_throughput.h | 32 +- .../cperf_test_vector_parsing.c | 32 +- .../cperf_test_vector_parsing.h | 32 +- .../app/test-crypto-perf/cperf_test_vectors.c | 54 +- .../app/test-crypto-perf/cperf_test_vectors.h | 32 +- dpdk/app/test-crypto-perf/cperf_test_verify.c | 37 +- dpdk/app/test-crypto-perf/cperf_test_verify.h | 32 +- dpdk/app/test-crypto-perf/main.c | 189 +- dpdk/app/test-crypto-perf/meson.build | 15 + dpdk/app/test-eventdev/Makefile | 35 +- dpdk/app/test-eventdev/evt_common.h | 73 +- dpdk/app/test-eventdev/evt_main.c | 73 +- dpdk/app/test-eventdev/evt_options.c | 171 +- dpdk/app/test-eventdev/evt_options.h | 96 +- dpdk/app/test-eventdev/evt_test.c | 32 +- dpdk/app/test-eventdev/evt_test.h | 35 +- dpdk/app/test-eventdev/meson.build | 15 + dpdk/app/test-eventdev/parser.h | 33 +- dpdk/app/test-eventdev/test_order_atq.c | 46 +- dpdk/app/test-eventdev/test_order_common.c | 32 +- dpdk/app/test-eventdev/test_order_common.h | 32 +- dpdk/app/test-eventdev/test_order_queue.c | 46 +- dpdk/app/test-eventdev/test_perf_atq.c | 103 +- dpdk/app/test-eventdev/test_perf_common.c | 558 +- dpdk/app/test-eventdev/test_perf_common.h | 53 +- dpdk/app/test-eventdev/test_perf_queue.c | 99 +- dpdk/app/test-eventdev/test_pipeline_atq.c | 527 + dpdk/app/test-eventdev/test_pipeline_common.c | 467 + dpdk/app/test-eventdev/test_pipeline_common.h | 173 + dpdk/app/test-eventdev/test_pipeline_queue.c | 542 + dpdk/app/test-pmd/Makefile | 49 +- dpdk/app/test-pmd/bpf_cmd.c | 198 + dpdk/app/test-pmd/bpf_cmd.h | 16 + dpdk/app/test-pmd/cmdline.c | 3778 +- dpdk/app/test-pmd/cmdline_flow.c | 2318 +- dpdk/app/test-pmd/cmdline_mtr.c | 697 +- dpdk/app/test-pmd/cmdline_mtr.h | 39 +- dpdk/app/test-pmd/cmdline_tm.c | 496 +- dpdk/app/test-pmd/cmdline_tm.h | 38 +- dpdk/app/test-pmd/config.c | 1001 +- dpdk/app/test-pmd/csumonly.c | 218 +- dpdk/app/test-pmd/flowgen.c | 12 +- dpdk/app/test-pmd/icmpecho.c | 34 +- dpdk/app/test-pmd/ieee1588fwd.c | 33 +- dpdk/app/test-pmd/iofwd.c | 33 +- dpdk/app/test-pmd/macfwd.c | 44 +- dpdk/app/test-pmd/macswap.c | 11 +- dpdk/app/test-pmd/meson.build | 50 + dpdk/app/test-pmd/noisy_vnf.c | 279 + dpdk/app/test-pmd/parameters.c | 289 +- dpdk/app/test-pmd/rxonly.c | 171 +- dpdk/app/test-pmd/softnicfwd.c | 676 + dpdk/app/test-pmd/testpmd.c | 1134 +- dpdk/app/test-pmd/testpmd.h | 346 +- dpdk/app/test-pmd/tm.c | 865 - dpdk/app/test-pmd/txonly.c | 41 +- dpdk/app/test-pmd/util.c | 219 + dpdk/buildtools/Makefile | 33 +- dpdk/buildtools/auto-config-h.sh | 36 +- dpdk/buildtools/check-experimental-syms.sh | 40 + dpdk/buildtools/gen-build-mk.sh | 38 +- dpdk/buildtools/gen-config-h.sh | 33 +- dpdk/buildtools/gen-pmdinfo-cfile.sh | 13 + dpdk/buildtools/meson.build | 6 + dpdk/buildtools/pmdinfogen/Makefile | 33 +- dpdk/buildtools/pmdinfogen/meson.build | 9 + dpdk/buildtools/pmdinfogen/pmdinfogen.c | 49 +- dpdk/buildtools/relpath.sh | 33 +- dpdk/buildtools/symlink-drivers-solibs.sh | 12 + dpdk/config/arm/arm64_armv8_linuxapp_gcc | 24 + dpdk/config/arm/arm64_dpaa2_linuxapp_gcc | 15 + dpdk/config/arm/arm64_dpaa_linuxapp_gcc | 15 + dpdk/config/arm/arm64_thunderx_linuxapp_gcc | 14 + dpdk/config/arm/armv8_machine.py | 18 + dpdk/config/arm/meson.build | 182 + dpdk/config/common_armv8a_linuxapp | 45 +- dpdk/config/common_base | 676 +- dpdk/config/common_bsdapp | 33 +- dpdk/config/common_linuxapp | 67 +- dpdk/config/defconfig_arm-armv7a-linuxapp-gcc | 31 +- .../defconfig_arm64-armv8a-linuxapp-clang | 31 +- .../defconfig_arm64-armv8a-linuxapp-gcc | 32 +- dpdk/config/defconfig_arm64-dpaa-linuxapp-gcc | 62 +- .../config/defconfig_arm64-dpaa2-linuxapp-gcc | 78 +- .../defconfig_arm64-stingray-linuxapp-gcc | 16 + .../defconfig_arm64-thunderx-linuxapp-gcc | 34 +- .../defconfig_arm64-xgene1-linuxapp-gcc | 31 +- .../config/defconfig_i686-native-linuxapp-gcc | 36 +- .../config/defconfig_i686-native-linuxapp-icc | 36 +- .../defconfig_ppc_64-power8-linuxapp-gcc | 1 + .../defconfig_x86_64-native-bsdapp-clang | 33 +- .../config/defconfig_x86_64-native-bsdapp-gcc | 33 +- .../defconfig_x86_64-native-linuxapp-clang | 33 +- .../defconfig_x86_64-native-linuxapp-gcc | 33 +- .../defconfig_x86_64-native-linuxapp-icc | 33 +- .../defconfig_x86_x32-native-linuxapp-gcc | 36 +- dpdk/config/meson.build | 130 + dpdk/config/ppc_64/meson.build | 11 + dpdk/config/rte_config.h | 122 + dpdk/config/x86/meson.build | 52 + dpdk/devtools/build-tags.sh | 34 +- dpdk/devtools/check-dup-includes.sh | 23 +- dpdk/devtools/check-forbidden-tokens.awk | 69 + dpdk/devtools/check-git-log.sh | 43 +- dpdk/devtools/check-includes.sh | 37 +- dpdk/devtools/check-maintainers.sh | 30 +- dpdk/devtools/check-symbol-change.sh | 158 + dpdk/devtools/check-symbol-maps.sh | 30 + dpdk/devtools/checkpatches.sh | 115 +- dpdk/devtools/cocci.sh | 2 +- dpdk/devtools/cocci/strlcpy.cocci | 8 + dpdk/devtools/get-maintainer.sh | 51 +- dpdk/devtools/git-log-fixes.sh | 30 +- dpdk/devtools/test-build.sh | 63 +- dpdk/devtools/test-meson-builds.sh | 61 + dpdk/devtools/test-null.sh | 30 +- dpdk/devtools/validate-abi.sh | 32 +- dpdk/doc/api/doxy-api-index.md | 53 +- dpdk/doc/api/doxy-api.conf | 105 - dpdk/doc/api/doxy-api.conf.in | 97 + dpdk/doc/api/doxy-html-custom.sh | 30 +- dpdk/doc/api/generate_doxygen.sh | 10 + dpdk/doc/api/generate_examples.sh | 12 + dpdk/doc/api/meson.build | 55 + dpdk/doc/build-sdk-meson.txt | 211 + dpdk/doc/guides/bbdevs/index.rst | 12 + dpdk/doc/guides/bbdevs/null.rst | 49 + dpdk/doc/guides/bbdevs/turbo_sw.rst | 157 + .../guides/compressdevs/features/default.ini | 26 + .../doc/guides/compressdevs/features/isal.ini | 16 + .../guides/compressdevs/features/octeontx.ini | 10 + dpdk/doc/guides/compressdevs/features/qat.ini | 16 + .../doc/guides/compressdevs/features/zlib.ini | 10 + dpdk/doc/guides/compressdevs/index.rst | 16 + dpdk/doc/guides/compressdevs/isal.rst | 123 + dpdk/doc/guides/compressdevs/octeontx.rst | 105 + dpdk/doc/guides/compressdevs/overview.rst | 32 + dpdk/doc/guides/compressdevs/qat_comp.rst | 44 + dpdk/doc/guides/compressdevs/zlib.rst | 69 + dpdk/doc/guides/conf.py | 57 +- dpdk/doc/guides/contributing/cheatsheet.rst | 3 + dpdk/doc/guides/contributing/coding_style.rst | 298 +- dpdk/doc/guides/contributing/design.rst | 3 + .../doc/guides/contributing/documentation.rst | 26 +- dpdk/doc/guides/contributing/index.rst | 3 + dpdk/doc/guides/contributing/patches.rst | 128 +- dpdk/doc/guides/contributing/stable.rst | 29 +- dpdk/doc/guides/contributing/versioning.rst | 41 +- dpdk/doc/guides/cryptodevs/aesni_gcm.rst | 43 +- dpdk/doc/guides/cryptodevs/aesni_mb.rst | 55 +- dpdk/doc/guides/cryptodevs/armv8.rst | 31 +- dpdk/doc/guides/cryptodevs/caam_jr.rst | 150 + dpdk/doc/guides/cryptodevs/ccp.rst | 140 + dpdk/doc/guides/cryptodevs/dpaa2_sec.rst | 95 +- dpdk/doc/guides/cryptodevs/dpaa_sec.rst | 94 +- .../guides/cryptodevs/features/aesni_gcm.ini | 2 + .../guides/cryptodevs/features/aesni_mb.ini | 3 + .../guides/cryptodevs/features/caam_jr.ini | 46 + dpdk/doc/guides/cryptodevs/features/ccp.ini | 59 + .../guides/cryptodevs/features/default.ini | 23 + .../guides/cryptodevs/features/dpaa2_sec.ini | 5 + .../guides/cryptodevs/features/dpaa_sec.ini | 6 + dpdk/doc/guides/cryptodevs/features/mrvl.ini | 42 - dpdk/doc/guides/cryptodevs/features/mvsam.ini | 54 + dpdk/doc/guides/cryptodevs/features/null.ini | 1 + .../guides/cryptodevs/features/octeontx.ini | 62 + .../guides/cryptodevs/features/openssl.ini | 13 + dpdk/doc/guides/cryptodevs/features/qat.ini | 9 + .../doc/guides/cryptodevs/features/virtio.ini | 26 + dpdk/doc/guides/cryptodevs/index.rst | 36 +- dpdk/doc/guides/cryptodevs/kasumi.rst | 40 +- dpdk/doc/guides/cryptodevs/mrvl.rst | 206 - dpdk/doc/guides/cryptodevs/mvsam.rst | 140 + dpdk/doc/guides/cryptodevs/null.rst | 30 +- dpdk/doc/guides/cryptodevs/octeontx.rst | 122 + dpdk/doc/guides/cryptodevs/openssl.rst | 31 +- dpdk/doc/guides/cryptodevs/overview.rst | 57 +- dpdk/doc/guides/cryptodevs/qat.rst | 356 +- dpdk/doc/guides/cryptodevs/scheduler.rst | 43 +- dpdk/doc/guides/cryptodevs/snow3g.rst | 40 +- dpdk/doc/guides/cryptodevs/virtio.rst | 117 + dpdk/doc/guides/cryptodevs/zuc.rst | 40 +- dpdk/doc/guides/custom.css | 7 + dpdk/doc/guides/eventdevs/dpaa.rst | 102 + dpdk/doc/guides/eventdevs/dpaa2.rst | 97 +- dpdk/doc/guides/eventdevs/dsw.rst | 96 + dpdk/doc/guides/eventdevs/index.rst | 33 +- dpdk/doc/guides/eventdevs/octeontx.rst | 97 +- dpdk/doc/guides/eventdevs/opdl.rst | 136 + dpdk/doc/guides/eventdevs/sw.rst | 30 +- dpdk/doc/guides/faq/faq.rst | 56 +- dpdk/doc/guides/faq/index.rst | 31 +- dpdk/doc/guides/freebsd_gsg/build_dpdk.rst | 31 +- .../guides/freebsd_gsg/build_sample_apps.rst | 38 +- .../freebsd_gsg/freebsd_eal_parameters.rst | 20 + dpdk/doc/guides/freebsd_gsg/index.rst | 32 +- .../guides/freebsd_gsg/install_from_ports.rst | 33 +- dpdk/doc/guides/freebsd_gsg/intro.rst | 31 +- dpdk/doc/guides/howto/flow_bifurcation.rst | 34 +- .../guides/howto/img/vf_daemon_overview.svg | 36 +- dpdk/doc/guides/howto/index.rst | 32 +- .../doc/guides/howto/lm_bond_virtio_sriov.rst | 31 +- .../doc/guides/howto/lm_virtio_vhost_user.rst | 34 +- .../guides/howto/packet_capture_framework.rst | 33 +- .../guides/howto/pvp_reference_benchmark.rst | 4 +- dpdk/doc/guides/howto/rte_flow.rst | 66 +- dpdk/doc/guides/howto/telemetry.rst | 85 + dpdk/doc/guides/howto/vfd.rst | 32 +- .../howto/virtio_user_as_exceptional_path.rst | 54 +- .../virtio_user_for_container_networking.rst | 36 +- dpdk/doc/guides/index.rst | 34 +- dpdk/doc/guides/linux_gsg/build_dpdk.rst | 31 +- .../guides/linux_gsg/build_sample_apps.rst | 47 +- .../linux_gsg/cross_build_dpdk_for_arm64.rst | 132 + .../doc/guides/linux_gsg/eal_args.include.rst | 146 + dpdk/doc/guides/linux_gsg/enable_func.rst | 31 +- dpdk/doc/guides/linux_gsg/index.rst | 33 +- dpdk/doc/guides/linux_gsg/intro.rst | 31 +- dpdk/doc/guides/linux_gsg/linux_drivers.rst | 37 +- .../guides/linux_gsg/linux_eal_parameters.rst | 118 + .../linux_gsg/nic_perf_intel_platform.rst | 4 +- dpdk/doc/guides/linux_gsg/quick_start.rst | 31 +- dpdk/doc/guides/linux_gsg/sys_reqs.rst | 59 +- dpdk/doc/guides/mempool/index.rst | 30 +- dpdk/doc/guides/mempool/octeontx.rst | 52 +- dpdk/doc/guides/meson.build | 28 + dpdk/doc/guides/nics/atlantic.rst | 47 + dpdk/doc/guides/nics/axgbe.rst | 89 + dpdk/doc/guides/nics/bnx2x.rst | 11 +- dpdk/doc/guides/nics/bnxt.rst | 36 +- dpdk/doc/guides/nics/build_and_test.rst | 34 +- dpdk/doc/guides/nics/cxgbe.rst | 194 +- dpdk/doc/guides/nics/dpaa.rst | 109 +- dpdk/doc/guides/nics/dpaa2.rst | 131 +- dpdk/doc/guides/nics/e1000em.rst | 31 +- dpdk/doc/guides/nics/ena.rst | 19 +- dpdk/doc/guides/nics/enetc.rst | 110 + dpdk/doc/guides/nics/enic.rst | 280 +- dpdk/doc/guides/nics/fail_safe.rst | 54 +- dpdk/doc/guides/nics/features.rst | 103 +- dpdk/doc/guides/nics/features/atlantic.ini | 37 + dpdk/doc/guides/nics/features/avf.ini | 36 + dpdk/doc/guides/nics/features/avf_vec.ini | 36 + dpdk/doc/guides/nics/features/axgbe.ini | 19 + dpdk/doc/guides/nics/features/bnxt.ini | 7 + dpdk/doc/guides/nics/features/cxgbe.ini | 3 + dpdk/doc/guides/nics/features/cxgbevf.ini | 29 + dpdk/doc/guides/nics/features/default.ini | 4 + dpdk/doc/guides/nics/features/ena.ini | 1 + dpdk/doc/guides/nics/features/enetc.ini | 11 + dpdk/doc/guides/nics/features/enic.ini | 10 +- dpdk/doc/guides/nics/features/failsafe.ini | 4 + dpdk/doc/guides/nics/features/fm10k.ini | 4 + dpdk/doc/guides/nics/features/fm10k_vf.ini | 2 + dpdk/doc/guides/nics/features/i40e.ini | 3 + dpdk/doc/guides/nics/features/i40e_vec.ini | 1 + dpdk/doc/guides/nics/features/i40e_vf.ini | 1 + dpdk/doc/guides/nics/features/i40e_vf_vec.ini | 1 + dpdk/doc/guides/nics/features/ifcvf.ini | 8 + dpdk/doc/guides/nics/features/igb.ini | 1 + dpdk/doc/guides/nics/features/igb_vf.ini | 1 + dpdk/doc/guides/nics/features/ixgbe.ini | 1 + dpdk/doc/guides/nics/features/ixgbe_vec.ini | 1 + dpdk/doc/guides/nics/features/mlx4.ini | 1 + dpdk/doc/guides/nics/features/mlx5.ini | 4 + dpdk/doc/guides/nics/features/mrvl.ini | 23 - dpdk/doc/guides/nics/features/mvneta.ini | 19 + dpdk/doc/guides/nics/features/mvpp2.ini | 25 + dpdk/doc/guides/nics/features/netvsc.ini | 24 + .../nics/features/{nfp_pf.ini => nfp.ini} | 0 dpdk/doc/guides/nics/features/qede.ini | 12 +- dpdk/doc/guides/nics/features/qede_vf.ini | 2 +- dpdk/doc/guides/nics/features/sfc_efx.ini | 5 + dpdk/doc/guides/nics/features/softnic.ini | 9 + dpdk/doc/guides/nics/features/tap.ini | 2 +- dpdk/doc/guides/nics/features/vdev_netvsc.ini | 12 + dpdk/doc/guides/nics/features/vhost.ini | 1 - dpdk/doc/guides/nics/features/virtio.ini | 1 + dpdk/doc/guides/nics/features/virtio_vec.ini | 1 + dpdk/doc/guides/nics/fm10k.rst | 57 +- dpdk/doc/guides/nics/i40e.rst | 259 +- dpdk/doc/guides/nics/ifc.rst | 96 + dpdk/doc/guides/nics/igb.rst | 38 + dpdk/doc/guides/nics/img/mvpp2_tm.svg | 71 + .../img/szedata2_nfb200g_architecture.svg | 214 + dpdk/doc/guides/nics/index.rst | 62 +- dpdk/doc/guides/nics/intel_vf.rst | 70 +- dpdk/doc/guides/nics/ixgbe.rst | 101 +- dpdk/doc/guides/nics/kni.rst | 31 +- dpdk/doc/guides/nics/liquidio.rst | 43 +- dpdk/doc/guides/nics/mlx4.rst | 81 +- dpdk/doc/guides/nics/mlx5.rst | 395 +- dpdk/doc/guides/nics/mrvl.rst | 253 - dpdk/doc/guides/nics/mvneta.rst | 171 + dpdk/doc/guides/nics/mvpp2.rst | 809 + dpdk/doc/guides/nics/netvsc.rst | 118 + dpdk/doc/guides/nics/nfp.rst | 34 +- dpdk/doc/guides/nics/octeontx.rst | 85 +- dpdk/doc/guides/nics/overview.rst | 28 +- dpdk/doc/guides/nics/pcap_ring.rst | 74 +- dpdk/doc/guides/nics/qede.rst | 72 +- dpdk/doc/guides/nics/sfc_efx.rst | 184 +- dpdk/doc/guides/nics/softnic.rst | 370 + dpdk/doc/guides/nics/szedata2.rst | 112 +- dpdk/doc/guides/nics/tap.rst | 164 +- dpdk/doc/guides/nics/thunderx.rst | 73 +- dpdk/doc/guides/nics/vdev_netvsc.rst | 104 + dpdk/doc/guides/nics/vhost.rst | 5 + dpdk/doc/guides/nics/virtio.rst | 78 +- dpdk/doc/guides/nics/vmxnet3.rst | 31 +- dpdk/doc/guides/platform/dpaa.rst | 103 + dpdk/doc/guides/platform/dpaa2.rst | 109 + dpdk/doc/guides/platform/index.rst | 32 +- dpdk/doc/guides/platform/octeontx.rst | 164 +- dpdk/doc/guides/prog_guide/bbdev.rst | 790 + dpdk/doc/guides/prog_guide/bpf_lib.rst | 38 + dpdk/doc/guides/prog_guide/build_app.rst | 31 +- dpdk/doc/guides/prog_guide/compressdev.rst | 623 + dpdk/doc/guides/prog_guide/cryptodev_lib.rst | 344 +- .../prog_guide/dev_kit_build_system.rst | 31 +- .../prog_guide/dev_kit_root_make_help.rst | 31 +- dpdk/doc/guides/prog_guide/efd_lib.rst | 31 +- .../prog_guide/env_abstraction_layer.rst | 334 +- .../prog_guide/event_crypto_adapter.rst | 296 + .../prog_guide/event_ethernet_rx_adapter.rst | 77 +- .../prog_guide/event_ethernet_tx_adapter.rst | 165 + .../guides/prog_guide/event_timer_adapter.rst | 296 + dpdk/doc/guides/prog_guide/eventdev.rst | 87 +- .../prog_guide/ext_app_lib_make_help.rst | 31 +- dpdk/doc/guides/prog_guide/extend_dpdk.rst | 31 +- .../guides/prog_guide/flow_classify_lib.rst | 118 +- .../generic_receive_offload_lib.rst | 274 +- .../generic_segmentation_offload_lib.rst | 41 +- dpdk/doc/guides/prog_guide/glossary.rst | 34 +- dpdk/doc/guides/prog_guide/hash_lib.rst | 210 +- .../prog_guide/img/architecture-overview.svg | 35 +- .../doc/guides/prog_guide/img/bond-mode-0.svg | 36 +- .../doc/guides/prog_guide/img/bond-mode-1.svg | 36 +- .../doc/guides/prog_guide/img/bond-mode-2.svg | 36 +- .../doc/guides/prog_guide/img/bond-mode-3.svg | 36 +- .../doc/guides/prog_guide/img/bond-mode-4.svg | 36 +- .../doc/guides/prog_guide/img/bond-mode-5.svg | 36 +- .../guides/prog_guide/img/bond-overview.svg | 36 +- .../img/event_crypto_adapter_op_forward.svg | 1078 + .../img/event_crypto_adapter_op_new.svg | 1061 + .../guides/prog_guide/img/eventdev_usage.svg | 1519 +- .../prog_guide/img/gro-key-algorithm.svg | 223 + .../guides/prog_guide/img/linuxapp_launch.svg | 35 +- .../doc/guides/prog_guide/img/malloc_heap.svg | 1379 +- dpdk/doc/guides/prog_guide/img/mbuf1.svg | 37 +- dpdk/doc/guides/prog_guide/img/mbuf2.svg | 37 +- .../prog_guide/img/memory-management.svg | 35 +- .../prog_guide/img/memory-management2.svg | 35 +- dpdk/doc/guides/prog_guide/img/mempool.svg | 35 +- .../prog_guide/img/multi_process_memory.svg | 35 +- .../guides/prog_guide/img/ring-dequeue1.svg | 35 +- .../guides/prog_guide/img/ring-dequeue2.svg | 35 +- .../guides/prog_guide/img/ring-dequeue3.svg | 35 +- .../guides/prog_guide/img/ring-enqueue1.svg | 35 +- .../guides/prog_guide/img/ring-enqueue2.svg | 35 +- .../guides/prog_guide/img/ring-enqueue3.svg | 35 +- .../guides/prog_guide/img/ring-modulo1.svg | 36 +- .../guides/prog_guide/img/ring-modulo2.svg | 35 +- .../prog_guide/img/ring-mp-enqueue1.svg | 35 +- .../prog_guide/img/ring-mp-enqueue2.svg | 35 +- .../prog_guide/img/ring-mp-enqueue3.svg | 35 +- .../prog_guide/img/ring-mp-enqueue4.svg | 35 +- .../prog_guide/img/ring-mp-enqueue5.svg | 35 +- dpdk/doc/guides/prog_guide/img/ring1.svg | 35 +- .../doc/guides/prog_guide/img/stateful-op.svg | 116 + .../prog_guide/img/stateless-op-shared.svg | 124 + .../guides/prog_guide/img/stateless-op.svg | 140 + .../guides/prog_guide/img/turbo_tb_decode.svg | 1471 + .../guides/prog_guide/img/turbo_tb_encode.svg | 1948 + dpdk/doc/guides/prog_guide/index.rst | 240 +- dpdk/doc/guides/prog_guide/intro.rst | 31 +- .../prog_guide/ip_fragment_reassembly_lib.rst | 31 +- .../prog_guide/kernel_nic_interface.rst | 270 +- .../link_bonding_poll_mode_drv_lib.rst | 88 +- dpdk/doc/guides/prog_guide/lpm6_lib.rst | 31 +- dpdk/doc/guides/prog_guide/lpm_lib.rst | 31 +- dpdk/doc/guides/prog_guide/mbuf_lib.rst | 42 +- dpdk/doc/guides/prog_guide/member_lib.rst | 32 +- dpdk/doc/guides/prog_guide/mempool_lib.rst | 31 +- dpdk/doc/guides/prog_guide/metrics_lib.rst | 40 +- .../guides/prog_guide/multi_proc_support.rst | 207 +- dpdk/doc/guides/prog_guide/overview.rst | 35 +- .../prog_guide/packet_classif_access_ctrl.rst | 31 +- .../guides/prog_guide/packet_distrib_lib.rst | 31 +- .../guides/prog_guide/packet_framework.rst | 42 +- dpdk/doc/guides/prog_guide/pdump_lib.rst | 31 +- .../guides/prog_guide/perf_opt_guidelines.rst | 31 +- dpdk/doc/guides/prog_guide/poll_mode_drv.rst | 107 +- .../prog_guide/port_hotplug_framework.rst | 106 - dpdk/doc/guides/prog_guide/power_man.rst | 125 +- dpdk/doc/guides/prog_guide/profile_app.rst | 65 +- dpdk/doc/guides/prog_guide/qos_framework.rst | 35 +- dpdk/doc/guides/prog_guide/rawdev.rst | 107 + dpdk/doc/guides/prog_guide/reorder_lib.rst | 31 +- dpdk/doc/guides/prog_guide/ring_lib.rst | 31 +- dpdk/doc/guides/prog_guide/rte_flow.rst | 1525 +- dpdk/doc/guides/prog_guide/rte_security.rst | 158 +- dpdk/doc/guides/prog_guide/service_cores.rst | 33 +- dpdk/doc/guides/prog_guide/source_org.rst | 35 +- .../prog_guide/switch_representation.rst | 837 + .../thread_safety_dpdk_functions.rst | 31 +- dpdk/doc/guides/prog_guide/timer_lib.rst | 31 +- .../guides/prog_guide/traffic_management.rst | 32 +- .../traffic_metering_and_policing.rst | 34 +- dpdk/doc/guides/prog_guide/vhost_lib.rst | 167 +- .../prog_guide/writing_efficient_code.rst | 31 +- dpdk/doc/guides/rawdevs/dpaa2_cmdif.rst | 104 + dpdk/doc/guides/rawdevs/dpaa2_qdma.rst | 101 + dpdk/doc/guides/rawdevs/ifpga_rawdev.rst | 112 + dpdk/doc/guides/rawdevs/index.rst | 16 + dpdk/doc/guides/rel_notes/deprecation.rst | 102 +- dpdk/doc/guides/rel_notes/index.rst | 36 +- dpdk/doc/guides/rel_notes/known_issues.rst | 144 +- dpdk/doc/guides/rel_notes/rel_description.rst | 40 - dpdk/doc/guides/rel_notes/release_16_04.rst | 3 + dpdk/doc/guides/rel_notes/release_16_07.rst | 3 + dpdk/doc/guides/rel_notes/release_16_11.rst | 3 + dpdk/doc/guides/rel_notes/release_17_02.rst | 3 + dpdk/doc/guides/rel_notes/release_17_05.rst | 9 + dpdk/doc/guides/rel_notes/release_17_08.rst | 3 + dpdk/doc/guides/rel_notes/release_17_11.rst | 662 +- dpdk/doc/guides/rel_notes/release_18_02.rst | 570 + dpdk/doc/guides/rel_notes/release_18_05.rst | 983 + dpdk/doc/guides/rel_notes/release_18_08.rst | 549 + dpdk/doc/guides/rel_notes/release_18_11.rst | 863 + dpdk/doc/guides/rel_notes/release_1_8.rst | 32 +- dpdk/doc/guides/rel_notes/release_2_0.rst | 32 +- dpdk/doc/guides/rel_notes/release_2_1.rst | 34 +- dpdk/doc/guides/rel_notes/release_2_2.rst | 3 + dpdk/doc/guides/sample_app_ug/bbdev_app.rst | 133 + dpdk/doc/guides/sample_app_ug/cmd_line.rst | 31 +- dpdk/doc/guides/sample_app_ug/compiling.rst | 46 +- dpdk/doc/guides/sample_app_ug/dist_app.rst | 31 +- dpdk/doc/guides/sample_app_ug/ethtool.rst | 35 +- .../sample_app_ug/eventdev_pipeline.rst | 145 + .../eventdev_pipeline_sw_pmd.rst | 173 - .../guides/sample_app_ug/exception_path.rst | 31 +- .../guides/sample_app_ug/fips_validation.rst | 132 + .../guides/sample_app_ug/flow_classify.rst | 53 +- .../guides/sample_app_ug/flow_filtering.rst | 136 +- dpdk/doc/guides/sample_app_ug/hello_world.rst | 31 +- .../doc/guides/sample_app_ug/img/dist_app.svg | 36 +- .../guides/sample_app_ug/img/dist_perf.svg | 36 +- .../img/exception_path_example.svg | 36 +- .../sample_app_ug/img/ip_pipelines_1.svg | 738 - .../sample_app_ug/img/ip_pipelines_2.svg | 997 - .../sample_app_ug/img/ip_pipelines_3.svg | 826 - .../img/l2_fwd_benchmark_setup.svg | 35 +- .../sample_app_ug/img/master_slave_proc.png | Bin 195232 -> 0 bytes .../sample_app_ug/img/slave_proc_recov.png | Bin 85287 -> 0 bytes .../img/vm_power_mgr_highlevel.svg | 36 +- .../img/vm_power_mgr_vm_request_seq.svg | 36 +- .../sample_app_ug/img/vmdq_dcb_example.svg | 35 +- dpdk/doc/guides/sample_app_ug/index.rst | 101 +- dpdk/doc/guides/sample_app_ug/intro.rst | 41 +- dpdk/doc/guides/sample_app_ug/ip_frag.rst | 31 +- dpdk/doc/guides/sample_app_ug/ip_pipeline.rst | 1596 +- .../guides/sample_app_ug/ip_reassembly.rst | 35 +- dpdk/doc/guides/sample_app_ug/ipsec_secgw.rst | 35 +- .../guides/sample_app_ug/ipv4_multicast.rst | 32 +- dpdk/doc/guides/sample_app_ug/keep_alive.rst | 34 +- .../sample_app_ug/kernel_nic_interface.rst | 673 +- .../guides/sample_app_ug/l2_forward_cat.rst | 32 +- .../sample_app_ug/l2_forward_crypto.rst | 39 +- .../sample_app_ug/l2_forward_job_stats.rst | 57 +- .../sample_app_ug/l2_forward_real_virtual.rst | 57 +- dpdk/doc/guides/sample_app_ug/l3_forward.rst | 31 +- .../sample_app_ug/l3_forward_access_ctrl.rst | 31 +- .../sample_app_ug/l3_forward_power_man.rst | 100 +- .../sample_app_ug/l3_forward_virtual.rst | 31 +- .../guides/sample_app_ug/link_status_intr.rst | 41 +- .../guides/sample_app_ug/load_balancer.rst | 31 +- .../guides/sample_app_ug/multi_process.rst | 437 +- .../sample_app_ug/netmap_compatibility.rst | 32 +- .../guides/sample_app_ug/packet_ordering.rst | 31 +- .../sample_app_ug/performance_thread.rst | 34 +- dpdk/doc/guides/sample_app_ug/ptpclient.rst | 32 +- .../doc/guides/sample_app_ug/qos_metering.rst | 35 +- .../guides/sample_app_ug/qos_scheduler.rst | 31 +- .../guides/sample_app_ug/quota_watermark.rst | 33 +- .../guides/sample_app_ug/rxtx_callbacks.rst | 35 +- .../guides/sample_app_ug/server_node_efd.rst | 31 +- .../guides/sample_app_ug/service_cores.rst | 31 +- dpdk/doc/guides/sample_app_ug/skeleton.rst | 43 +- .../guides/sample_app_ug/tep_termination.rst | 33 +- .../guides/sample_app_ug/test_pipeline.rst | 31 +- dpdk/doc/guides/sample_app_ug/timer.rst | 31 +- dpdk/doc/guides/sample_app_ug/vdpa.rst | 120 + dpdk/doc/guides/sample_app_ug/vhost.rst | 69 +- .../doc/guides/sample_app_ug/vhost_crypto.rst | 80 + dpdk/doc/guides/sample_app_ug/vhost_scsi.rst | 33 +- .../sample_app_ug/vm_power_management.rst | 517 +- .../sample_app_ug/vmdq_dcb_forwarding.rst | 35 +- dpdk/doc/guides/testpmd_app_ug/build_app.rst | 31 +- dpdk/doc/guides/testpmd_app_ug/index.rst | 31 +- dpdk/doc/guides/testpmd_app_ug/intro.rst | 31 +- dpdk/doc/guides/testpmd_app_ug/run_app.rst | 260 +- .../guides/testpmd_app_ug/testpmd_funcs.rst | 1229 +- dpdk/doc/guides/tools/cryptoperf.rst | 47 +- .../tools/img/eventdev_order_atq_test.svg | 34 +- .../tools/img/eventdev_order_queue_test.svg | 34 +- .../tools/img/eventdev_perf_atq_test.svg | 34 +- .../tools/img/eventdev_perf_queue_test.svg | 34 +- .../eventdev_pipeline_atq_test_generic.svg | 3465 ++ ...entdev_pipeline_atq_test_internal_port.svg | 3344 ++ .../eventdev_pipeline_queue_test_generic.svg | 3688 ++ ...tdev_pipeline_queue_test_internal_port.svg | 3826 ++ dpdk/doc/guides/tools/index.rst | 1 + dpdk/doc/guides/tools/pdump.rst | 41 +- dpdk/doc/guides/tools/proc_info.rst | 47 +- dpdk/doc/guides/tools/testbbdev.rst | 618 + dpdk/doc/guides/tools/testeventdev.rst | 309 +- dpdk/doc/meson.build | 15 + dpdk/drivers/Makefile | 50 +- dpdk/drivers/baseband/Makefile | 14 + dpdk/drivers/baseband/meson.build | 7 + dpdk/drivers/baseband/null/Makefile | 25 + dpdk/drivers/baseband/null/bbdev_null.c | 356 + dpdk/drivers/baseband/null/meson.build | 7 + .../null/rte_pmd_bbdev_null_version.map | 3 + dpdk/drivers/baseband/turbo_sw/Makefile | 42 + .../baseband/turbo_sw/bbdev_turbo_software.c | 1307 + .../rte_pmd_bbdev_turbo_sw_version.map | 3 + dpdk/drivers/bus/Makefile | 35 +- dpdk/drivers/bus/dpaa/Makefile | 36 +- dpdk/drivers/bus/dpaa/base/fman/fman.c | 46 +- dpdk/drivers/bus/dpaa/base/fman/fman_hw.c | 69 +- .../drivers/bus/dpaa/base/fman/netcfg_layer.c | 60 +- dpdk/drivers/bus/dpaa/base/fman/of.c | 76 +- dpdk/drivers/bus/dpaa/base/qbman/bman.c | 37 +- dpdk/drivers/bus/dpaa/base/qbman/bman.h | 37 +- .../drivers/bus/dpaa/base/qbman/bman_driver.c | 60 +- dpdk/drivers/bus/dpaa/base/qbman/bman_priv.h | 37 +- dpdk/drivers/bus/dpaa/base/qbman/dpaa_alloc.c | 37 +- dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.c | 37 +- dpdk/drivers/bus/dpaa/base/qbman/dpaa_sys.h | 37 +- dpdk/drivers/bus/dpaa/base/qbman/process.c | 37 +- dpdk/drivers/bus/dpaa/base/qbman/qman.c | 439 +- dpdk/drivers/bus/dpaa/base/qbman/qman.h | 41 +- .../drivers/bus/dpaa/base/qbman/qman_driver.c | 209 +- dpdk/drivers/bus/dpaa/base/qbman/qman_priv.h | 44 +- dpdk/drivers/bus/dpaa/dpaa_bus.c | 292 +- dpdk/drivers/bus/dpaa/include/compat.h | 91 +- dpdk/drivers/bus/dpaa/include/dpaa_bits.h | 30 +- dpdk/drivers/bus/dpaa/include/dpaa_list.h | 30 +- dpdk/drivers/bus/dpaa/include/dpaa_rbtree.h | 30 +- dpdk/drivers/bus/dpaa/include/fman.h | 37 +- dpdk/drivers/bus/dpaa/include/fsl_bman.h | 35 +- dpdk/drivers/bus/dpaa/include/fsl_fman.h | 43 +- .../drivers/bus/dpaa/include/fsl_fman_crc64.h | 43 +- dpdk/drivers/bus/dpaa/include/fsl_qman.h | 180 +- dpdk/drivers/bus/dpaa/include/fsl_usd.h | 45 +- dpdk/drivers/bus/dpaa/include/netcfg.h | 35 +- dpdk/drivers/bus/dpaa/include/of.h | 39 +- dpdk/drivers/bus/dpaa/include/process.h | 46 +- dpdk/drivers/bus/dpaa/meson.build | 30 + .../drivers/bus/dpaa/rte_bus_dpaa_version.map | 52 + dpdk/drivers/bus/dpaa/rte_dpaa_bus.h | 105 +- dpdk/drivers/bus/dpaa/rte_dpaa_logs.h | 57 +- dpdk/drivers/bus/fslmc/Makefile | 56 +- dpdk/drivers/bus/fslmc/fslmc_bus.c | 305 +- dpdk/drivers/bus/fslmc/fslmc_logs.h | 87 +- dpdk/drivers/bus/fslmc/fslmc_vfio.c | 436 +- dpdk/drivers/bus/fslmc/fslmc_vfio.h | 32 +- dpdk/drivers/bus/fslmc/mc/dpbp.c | 47 +- dpdk/drivers/bus/fslmc/mc/dpci.c | 232 +- dpdk/drivers/bus/fslmc/mc/dpcon.c | 60 +- dpdk/drivers/bus/fslmc/mc/dpdmai.c | 443 + dpdk/drivers/bus/fslmc/mc/dpio.c | 46 +- dpdk/drivers/bus/fslmc/mc/dpmng.c | 37 +- dpdk/drivers/bus/fslmc/mc/fsl_dpbp.h | 38 +- dpdk/drivers/bus/fslmc/mc/fsl_dpbp_cmd.h | 53 +- dpdk/drivers/bus/fslmc/mc/fsl_dpci.h | 82 +- dpdk/drivers/bus/fslmc/mc/fsl_dpci_cmd.h | 97 +- dpdk/drivers/bus/fslmc/mc/fsl_dpcon.h | 54 +- dpdk/drivers/bus/fslmc/mc/fsl_dpcon_cmd.h | 37 +- dpdk/drivers/bus/fslmc/mc/fsl_dpdmai.h | 194 + dpdk/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h | 111 + dpdk/drivers/bus/fslmc/mc/fsl_dpio.h | 37 +- dpdk/drivers/bus/fslmc/mc/fsl_dpio_cmd.h | 37 +- dpdk/drivers/bus/fslmc/mc/fsl_dpmng.h | 39 +- dpdk/drivers/bus/fslmc/mc/fsl_dpmng_cmd.h | 37 +- dpdk/drivers/bus/fslmc/mc/fsl_dpopr.h | 85 + dpdk/drivers/bus/fslmc/mc/fsl_mc_cmd.h | 39 +- dpdk/drivers/bus/fslmc/mc/fsl_mc_sys.h | 37 +- dpdk/drivers/bus/fslmc/mc/mc_sys.c | 37 +- dpdk/drivers/bus/fslmc/meson.build | 28 + dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 56 +- dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 132 +- dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 334 +- dpdk/drivers/bus/fslmc/portal/dpaa2_hw_dpio.h | 45 +- dpdk/drivers/bus/fslmc/portal/dpaa2_hw_pvt.h | 202 +- dpdk/drivers/bus/fslmc/qbman/include/compat.h | 29 +- .../bus/fslmc/qbman/include/fsl_qbman_base.h | 26 +- .../bus/fslmc/qbman/include/fsl_qbman_debug.h | 30 + .../fslmc/qbman/include/fsl_qbman_portal.h | 73 +- dpdk/drivers/bus/fslmc/qbman/qbman_debug.c | 66 + dpdk/drivers/bus/fslmc/qbman/qbman_portal.c | 828 +- dpdk/drivers/bus/fslmc/qbman/qbman_portal.h | 53 +- dpdk/drivers/bus/fslmc/qbman/qbman_sys.h | 150 +- dpdk/drivers/bus/fslmc/qbman/qbman_sys_decl.h | 51 +- .../bus/fslmc/rte_bus_fslmc_version.map | 46 +- dpdk/drivers/bus/fslmc/rte_fslmc.h | 85 +- dpdk/drivers/bus/ifpga/Makefile | 32 + dpdk/drivers/bus/ifpga/ifpga_bus.c | 464 + dpdk/drivers/bus/ifpga/ifpga_common.c | 88 + dpdk/drivers/bus/ifpga/ifpga_common.h | 18 + dpdk/drivers/bus/ifpga/ifpga_logs.h | 31 + dpdk/drivers/bus/ifpga/meson.build | 10 + dpdk/drivers/bus/ifpga/rte_bus_ifpga.h | 146 + .../bus/ifpga/rte_bus_ifpga_version.map | 10 + dpdk/drivers/bus/meson.build | 7 + dpdk/drivers/bus/pci/Makefile | 41 +- dpdk/drivers/bus/pci/bsd/Makefile | 32 +- dpdk/drivers/bus/pci/bsd/pci.c | 39 +- dpdk/drivers/bus/pci/linux/Makefile | 34 +- dpdk/drivers/bus/pci/linux/pci.c | 167 +- dpdk/drivers/bus/pci/linux/pci_init.h | 53 +- dpdk/drivers/bus/pci/linux/pci_uio.c | 82 +- dpdk/drivers/bus/pci/linux/pci_vfio.c | 447 +- dpdk/drivers/bus/pci/meson.build | 24 + dpdk/drivers/bus/pci/pci_common.c | 273 +- dpdk/drivers/bus/pci/pci_common_uio.c | 66 +- dpdk/drivers/bus/pci/pci_params.c | 78 + dpdk/drivers/bus/pci/private.h | 122 +- dpdk/drivers/bus/pci/rte_bus_pci.h | 50 +- dpdk/drivers/bus/vdev/Makefile | 38 +- dpdk/drivers/bus/vdev/meson.build | 12 + dpdk/drivers/bus/vdev/rte_bus_vdev.h | 73 +- .../drivers/bus/vdev/rte_bus_vdev_version.map | 8 + dpdk/drivers/bus/vdev/vdev.c | 371 +- dpdk/drivers/bus/vdev/vdev_logs.h | 33 +- dpdk/drivers/bus/vdev/vdev_params.c | 66 + dpdk/drivers/bus/vdev/vdev_private.h | 26 + dpdk/drivers/bus/vmbus/Makefile | 36 + dpdk/drivers/bus/vmbus/linux/Makefile | 3 + dpdk/drivers/bus/vmbus/linux/vmbus_bus.c | 358 + dpdk/drivers/bus/vmbus/linux/vmbus_uio.c | 400 + dpdk/drivers/bus/vmbus/meson.build | 20 + dpdk/drivers/bus/vmbus/private.h | 138 + dpdk/drivers/bus/vmbus/rte_bus_vmbus.h | 421 + .../bus/vmbus/rte_bus_vmbus_version.map | 36 + dpdk/drivers/bus/vmbus/rte_vmbus_reg.h | 344 + dpdk/drivers/bus/vmbus/vmbus_bufring.c | 244 + dpdk/drivers/bus/vmbus/vmbus_channel.c | 431 + dpdk/drivers/bus/vmbus/vmbus_common.c | 307 + dpdk/drivers/bus/vmbus/vmbus_common_uio.c | 232 + dpdk/drivers/common/Makefile | 26 + dpdk/drivers/common/cpt/Makefile | 25 + dpdk/drivers/common/cpt/cpt_common.h | 91 + dpdk/drivers/common/cpt/cpt_hw_types.h | 522 + dpdk/drivers/common/cpt/cpt_mcode_defines.h | 386 + dpdk/drivers/common/cpt/cpt_pmd_logs.h | 50 + dpdk/drivers/common/cpt/cpt_pmd_ops_helper.c | 41 + dpdk/drivers/common/cpt/cpt_pmd_ops_helper.h | 34 + dpdk/drivers/common/cpt/cpt_request_mgr.h | 185 + dpdk/drivers/common/cpt/cpt_ucode.h | 3643 ++ dpdk/drivers/common/cpt/meson.build | 8 + .../common/cpt/rte_common_cpt_version.map | 6 + dpdk/drivers/common/dpaax/Makefile | 31 + dpdk/drivers/common/dpaax/dpaax_iova_table.c | 465 + dpdk/drivers/common/dpaax/dpaax_iova_table.h | 105 + dpdk/drivers/common/dpaax/dpaax_logs.h | 39 + dpdk/drivers/common/dpaax/meson.build | 12 + .../common/dpaax/rte_common_dpaax_version.map | 11 + dpdk/drivers/common/meson.build | 7 + dpdk/drivers/common/mvep/Makefile | 38 + dpdk/drivers/common/mvep/meson.build | 19 + dpdk/drivers/common/mvep/mvep_common.c | 45 + .../common/mvep/rte_common_mvep_version.map | 6 + dpdk/drivers/common/mvep/rte_mvep_common.h | 21 + dpdk/drivers/common/octeontx/Makefile | 24 + dpdk/drivers/common/octeontx/meson.build | 5 + dpdk/drivers/common/octeontx/octeontx_mbox.c | 249 + dpdk/drivers/common/octeontx/octeontx_mbox.h | 37 + .../octeontx/rte_common_octeontx_version.map | 7 + dpdk/drivers/common/qat/Makefile | 66 + dpdk/drivers/common/qat/meson.build | 14 + .../qat/qat_adf/adf_transport_access_macros.h | 136 + dpdk/drivers/common/qat/qat_adf/icp_qat_fw.h | 318 + .../common/qat/qat_adf/icp_qat_fw_comp.h | 482 + .../common/qat/qat_adf/icp_qat_fw_la.h | 361 + dpdk/drivers/common/qat/qat_adf/icp_qat_hw.h | 386 + dpdk/drivers/common/qat/qat_common.c | 123 + dpdk/drivers/common/qat/qat_common.h | 80 + dpdk/drivers/common/qat/qat_device.c | 303 + dpdk/drivers/common/qat/qat_device.h | 114 + dpdk/drivers/common/qat/qat_logs.c | 38 + dpdk/drivers/common/qat/qat_logs.h | 34 + dpdk/drivers/common/qat/qat_qp.c | 688 + dpdk/drivers/common/qat/qat_qp.h | 112 + dpdk/drivers/compress/Makefile | 10 + dpdk/drivers/compress/isal/Makefile | 31 + .../drivers/compress/isal/isal_compress_pmd.c | 695 + .../compress/isal/isal_compress_pmd_ops.c | 351 + .../compress/isal/isal_compress_pmd_private.h | 57 + dpdk/drivers/compress/isal/meson.build | 14 + .../compress/isal/rte_pmd_isal_version.map | 3 + dpdk/drivers/compress/meson.build | 8 + dpdk/drivers/compress/octeontx/Makefile | 30 + .../compress/octeontx/include/zip_regs.h | 711 + dpdk/drivers/compress/octeontx/meson.build | 9 + dpdk/drivers/compress/octeontx/otx_zip.c | 180 + dpdk/drivers/compress/octeontx/otx_zip.h | 277 + dpdk/drivers/compress/octeontx/otx_zip_pmd.c | 655 + .../rte_pmd_octeontx_compress_version.map | 3 + dpdk/drivers/compress/qat/meson.build | 18 + dpdk/drivers/compress/qat/qat_comp.c | 418 + dpdk/drivers/compress/qat/qat_comp.h | 78 + dpdk/drivers/compress/qat/qat_comp_pmd.c | 573 + dpdk/drivers/compress/qat/qat_comp_pmd.h | 42 + .../compress/qat/rte_pmd_qat_version.map | 3 + dpdk/drivers/compress/zlib/Makefile | 29 + dpdk/drivers/compress/zlib/meson.build | 14 + .../compress/zlib/rte_pmd_zlib_version.map | 3 + dpdk/drivers/compress/zlib/zlib_pmd.c | 434 + dpdk/drivers/compress/zlib/zlib_pmd_ops.c | 307 + dpdk/drivers/compress/zlib/zlib_pmd_private.h | 71 + dpdk/drivers/crypto/Makefile | 45 +- dpdk/drivers/crypto/aesni_gcm/Makefile | 41 +- dpdk/drivers/crypto/aesni_gcm/aesni_gcm_ops.h | 35 +- dpdk/drivers/crypto/aesni_gcm/aesni_gcm_pmd.c | 173 +- .../crypto/aesni_gcm/aesni_gcm_pmd_ops.c | 94 +- .../crypto/aesni_gcm/aesni_gcm_pmd_private.h | 76 +- dpdk/drivers/crypto/aesni_gcm/meson.build | 12 + dpdk/drivers/crypto/aesni_mb/Makefile | 41 +- dpdk/drivers/crypto/aesni_mb/aesni_mb_ops.h | 144 +- dpdk/drivers/crypto/aesni_mb/meson.build | 12 + .../crypto/aesni_mb/rte_aesni_mb_pmd.c | 583 +- .../crypto/aesni_mb/rte_aesni_mb_pmd_ops.c | 275 +- .../aesni_mb/rte_aesni_mb_pmd_private.h | 117 +- dpdk/drivers/crypto/armv8/Makefile | 32 +- dpdk/drivers/crypto/armv8/rte_armv8_pmd.c | 47 +- dpdk/drivers/crypto/armv8/rte_armv8_pmd_ops.c | 79 +- .../crypto/armv8/rte_armv8_pmd_private.h | 36 +- dpdk/drivers/crypto/caam_jr/Makefile | 44 + dpdk/drivers/crypto/caam_jr/caam_jr.c | 2508 ++ .../crypto/caam_jr/caam_jr_capabilities.c | 266 + .../crypto/caam_jr/caam_jr_capabilities.h | 18 + dpdk/drivers/crypto/caam_jr/caam_jr_config.h | 207 + dpdk/drivers/crypto/caam_jr/caam_jr_desc.h | 285 + dpdk/drivers/crypto/caam_jr/caam_jr_hw.c | 367 + .../crypto/caam_jr/caam_jr_hw_specific.h | 503 + dpdk/drivers/crypto/caam_jr/caam_jr_log.h | 42 + dpdk/drivers/crypto/caam_jr/caam_jr_pvt.h | 291 + dpdk/drivers/crypto/caam_jr/caam_jr_uio.c | 501 + dpdk/drivers/crypto/caam_jr/meson.build | 17 + .../caam_jr/rte_pmd_caam_jr_version.map | 4 + dpdk/drivers/crypto/ccp/Makefile | 35 + dpdk/drivers/crypto/ccp/ccp_crypto.c | 2951 ++ dpdk/drivers/crypto/ccp/ccp_crypto.h | 388 + dpdk/drivers/crypto/ccp/ccp_dev.c | 810 + dpdk/drivers/crypto/ccp/ccp_dev.h | 495 + dpdk/drivers/crypto/ccp/ccp_pci.c | 239 + dpdk/drivers/crypto/ccp/ccp_pci.h | 27 + dpdk/drivers/crypto/ccp/ccp_pmd_ops.c | 833 + dpdk/drivers/crypto/ccp/ccp_pmd_private.h | 107 + dpdk/drivers/crypto/ccp/meson.build | 21 + dpdk/drivers/crypto/ccp/rte_ccp_pmd.c | 397 + .../crypto/ccp/rte_pmd_ccp_version.map | 4 + dpdk/drivers/crypto/dpaa2_sec/Makefile | 49 +- .../crypto/dpaa2_sec/dpaa2_sec_dpseci.c | 1836 +- .../crypto/dpaa2_sec/dpaa2_sec_event.h | 18 + .../drivers/crypto/dpaa2_sec/dpaa2_sec_logs.h | 82 +- .../drivers/crypto/dpaa2_sec/dpaa2_sec_priv.h | 242 +- dpdk/drivers/crypto/dpaa2_sec/hw/compat.h | 37 +- dpdk/drivers/crypto/dpaa2_sec/hw/desc.h | 853 +- dpdk/drivers/crypto/dpaa2_sec/hw/desc/algo.h | 95 +- .../drivers/crypto/dpaa2_sec/hw/desc/common.h | 37 +- dpdk/drivers/crypto/dpaa2_sec/hw/desc/ipsec.h | 232 +- dpdk/drivers/crypto/dpaa2_sec/hw/desc/pdcp.h | 2796 ++ dpdk/drivers/crypto/dpaa2_sec/hw/rta.h | 37 +- .../dpaa2_sec/hw/rta/fifo_load_store_cmd.h | 37 +- .../crypto/dpaa2_sec/hw/rta/header_cmd.h | 37 +- .../crypto/dpaa2_sec/hw/rta/jump_cmd.h | 37 +- .../drivers/crypto/dpaa2_sec/hw/rta/key_cmd.h | 37 +- .../crypto/dpaa2_sec/hw/rta/load_cmd.h | 37 +- .../crypto/dpaa2_sec/hw/rta/math_cmd.h | 37 +- .../crypto/dpaa2_sec/hw/rta/move_cmd.h | 37 +- .../crypto/dpaa2_sec/hw/rta/nfifo_cmd.h | 37 +- .../crypto/dpaa2_sec/hw/rta/operation_cmd.h | 41 +- .../crypto/dpaa2_sec/hw/rta/protocol_cmd.h | 383 +- .../dpaa2_sec/hw/rta/sec_run_time_asm.h | 59 +- .../dpaa2_sec/hw/rta/seq_in_out_ptr_cmd.h | 37 +- .../crypto/dpaa2_sec/hw/rta/signature_cmd.h | 37 +- .../crypto/dpaa2_sec/hw/rta/store_cmd.h | 37 +- dpdk/drivers/crypto/dpaa2_sec/mc/dpseci.c | 165 +- dpdk/drivers/crypto/dpaa2_sec/mc/fsl_dpseci.h | 62 +- .../crypto/dpaa2_sec/mc/fsl_dpseci_cmd.h | 110 +- dpdk/drivers/crypto/dpaa2_sec/meson.build | 16 + .../dpaa2_sec/rte_pmd_dpaa2_sec_version.map | 8 + dpdk/drivers/crypto/dpaa_sec/Makefile | 41 +- dpdk/drivers/crypto/dpaa_sec/dpaa_sec.c | 1549 +- dpdk/drivers/crypto/dpaa_sec/dpaa_sec.h | 205 +- dpdk/drivers/crypto/dpaa_sec/dpaa_sec_log.h | 83 +- dpdk/drivers/crypto/dpaa_sec/meson.build | 13 + dpdk/drivers/crypto/kasumi/Makefile | 31 +- dpdk/drivers/crypto/kasumi/meson.build | 12 + dpdk/drivers/crypto/kasumi/rte_kasumi_pmd.c | 80 +- .../crypto/kasumi/rte_kasumi_pmd_ops.c | 83 +- .../crypto/kasumi/rte_kasumi_pmd_private.h | 60 +- dpdk/drivers/crypto/meson.build | 10 + dpdk/drivers/crypto/mrvl/Makefile | 66 - dpdk/drivers/crypto/mrvl/rte_mrvl_compat.h | 50 - dpdk/drivers/crypto/mrvl/rte_mrvl_pmd.c | 857 - dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c | 778 - .../crypto/mrvl/rte_mrvl_pmd_private.h | 123 - dpdk/drivers/crypto/mvsam/Makefile | 43 + dpdk/drivers/crypto/mvsam/meson.build | 21 + dpdk/drivers/crypto/mvsam/rte_mrvl_compat.h | 23 + dpdk/drivers/crypto/mvsam/rte_mrvl_pmd.c | 992 + dpdk/drivers/crypto/mvsam/rte_mrvl_pmd_ops.c | 828 + .../crypto/mvsam/rte_mrvl_pmd_private.h | 89 + .../rte_pmd_mvsam_version.map} | 0 dpdk/drivers/crypto/null/Makefile | 31 +- dpdk/drivers/crypto/null/meson.build | 6 + dpdk/drivers/crypto/null/null_crypto_pmd.c | 60 +- .../drivers/crypto/null/null_crypto_pmd_ops.c | 102 +- .../crypto/null/null_crypto_pmd_private.h | 58 +- dpdk/drivers/crypto/octeontx/Makefile | 46 + dpdk/drivers/crypto/octeontx/meson.build | 18 + dpdk/drivers/crypto/octeontx/otx_cryptodev.c | 133 + dpdk/drivers/crypto/octeontx/otx_cryptodev.h | 20 + .../octeontx/otx_cryptodev_capabilities.c | 604 + .../octeontx/otx_cryptodev_capabilities.h | 17 + .../crypto/octeontx/otx_cryptodev_hw_access.c | 601 + .../crypto/octeontx/otx_cryptodev_hw_access.h | 320 + .../crypto/octeontx/otx_cryptodev_mbox.c | 178 + .../crypto/octeontx/otx_cryptodev_mbox.h | 92 + .../crypto/octeontx/otx_cryptodev_ops.c | 531 + .../crypto/octeontx/otx_cryptodev_ops.h | 18 + .../rte_pmd_octeontx_crypto_version.map | 4 + dpdk/drivers/crypto/openssl/Makefile | 31 +- dpdk/drivers/crypto/openssl/compat.h | 197 + dpdk/drivers/crypto/openssl/meson.build | 11 + dpdk/drivers/crypto/openssl/rte_openssl_pmd.c | 586 +- .../crypto/openssl/rte_openssl_pmd_ops.c | 641 +- .../crypto/openssl/rte_openssl_pmd_private.h | 89 +- dpdk/drivers/crypto/qat/Makefile | 62 - dpdk/drivers/crypto/qat/README | 7 + dpdk/drivers/crypto/qat/meson.build | 18 + .../qat/qat_adf/adf_transport_access_macros.h | 175 - dpdk/drivers/crypto/qat/qat_adf/icp_qat_fw.h | 316 - .../crypto/qat/qat_adf/icp_qat_fw_la.h | 404 - dpdk/drivers/crypto/qat/qat_adf/icp_qat_hw.h | 329 - dpdk/drivers/crypto/qat/qat_adf/qat_algs.h | 169 - .../crypto/qat/qat_adf/qat_algs_build_desc.c | 1059 - dpdk/drivers/crypto/qat/qat_crypto.c | 1725 - dpdk/drivers/crypto/qat/qat_crypto.h | 179 - .../crypto/qat/qat_crypto_capabilities.h | 586 - dpdk/drivers/crypto/qat/qat_logs.h | 78 - dpdk/drivers/crypto/qat/qat_qp.c | 499 - dpdk/drivers/crypto/qat/qat_sym.c | 569 + dpdk/drivers/crypto/qat/qat_sym.h | 174 + .../drivers/crypto/qat/qat_sym_capabilities.h | 577 + dpdk/drivers/crypto/qat/qat_sym_pmd.c | 332 + dpdk/drivers/crypto/qat/qat_sym_pmd.h | 41 + dpdk/drivers/crypto/qat/qat_sym_session.c | 1807 + dpdk/drivers/crypto/qat/qat_sym_session.h | 148 + .../crypto/qat/rte_pmd_qat_version.map | 3 - dpdk/drivers/crypto/qat/rte_qat_cryptodev.c | 209 - dpdk/drivers/crypto/scheduler/Makefile | 31 +- dpdk/drivers/crypto/scheduler/meson.build | 19 + .../scheduler/rte_cryptodev_scheduler.c | 151 +- .../scheduler/rte_cryptodev_scheduler.h | 44 +- .../rte_cryptodev_scheduler_operations.h | 33 +- .../crypto/scheduler/scheduler_failover.c | 42 +- .../crypto/scheduler/scheduler_multicore.c | 49 +- .../scheduler/scheduler_pkt_size_distr.c | 52 +- dpdk/drivers/crypto/scheduler/scheduler_pmd.c | 162 +- .../crypto/scheduler/scheduler_pmd_ops.c | 126 +- .../crypto/scheduler/scheduler_pmd_private.h | 57 +- .../crypto/scheduler/scheduler_roundrobin.c | 40 +- dpdk/drivers/crypto/snow3g/Makefile | 31 +- dpdk/drivers/crypto/snow3g/rte_snow3g_pmd.c | 72 +- .../crypto/snow3g/rte_snow3g_pmd_ops.c | 81 +- .../crypto/snow3g/rte_snow3g_pmd_private.h | 60 +- dpdk/drivers/crypto/virtio/Makefile | 35 + dpdk/drivers/crypto/virtio/meson.build | 8 + .../virtio/rte_pmd_virtio_crypto_version.map | 3 + .../crypto/virtio/virtio_crypto_algs.h | 28 + .../virtio/virtio_crypto_capabilities.h | 51 + dpdk/drivers/crypto/virtio/virtio_cryptodev.c | 1505 + dpdk/drivers/crypto/virtio/virtio_cryptodev.h | 64 + dpdk/drivers/crypto/virtio/virtio_logs.h | 91 + dpdk/drivers/crypto/virtio/virtio_pci.c | 462 + dpdk/drivers/crypto/virtio/virtio_pci.h | 253 + dpdk/drivers/crypto/virtio/virtio_ring.h | 137 + dpdk/drivers/crypto/virtio/virtio_rxtx.c | 527 + dpdk/drivers/crypto/virtio/virtqueue.c | 43 + dpdk/drivers/crypto/virtio/virtqueue.h | 171 + dpdk/drivers/crypto/zuc/Makefile | 31 +- dpdk/drivers/crypto/zuc/meson.build | 12 + dpdk/drivers/crypto/zuc/rte_zuc_pmd.c | 74 +- dpdk/drivers/crypto/zuc/rte_zuc_pmd_ops.c | 82 +- dpdk/drivers/crypto/zuc/rte_zuc_pmd_private.h | 61 +- dpdk/drivers/event/Makefile | 38 +- dpdk/drivers/event/dpaa/Makefile | 39 + dpdk/drivers/event/dpaa/dpaa_eventdev.c | 852 + dpdk/drivers/event/dpaa/dpaa_eventdev.h | 74 + dpdk/drivers/event/dpaa/meson.build | 10 + .../event/dpaa/rte_pmd_dpaa_event_version.map | 4 + dpdk/drivers/event/dpaa2/Makefile | 45 +- dpdk/drivers/event/dpaa2/dpaa2_eventdev.c | 663 +- dpdk/drivers/event/dpaa2/dpaa2_eventdev.h | 65 +- .../drivers/event/dpaa2/dpaa2_eventdev_logs.h | 39 + dpdk/drivers/event/dpaa2/dpaa2_hw_dpcon.c | 46 +- dpdk/drivers/event/dpaa2/meson.build | 14 + dpdk/drivers/event/dsw/Makefile | 29 + dpdk/drivers/event/dsw/dsw_evdev.c | 435 + dpdk/drivers/event/dsw/dsw_evdev.h | 279 + dpdk/drivers/event/dsw/dsw_event.c | 1253 + dpdk/drivers/event/dsw/dsw_sort.h | 48 + dpdk/drivers/event/dsw/dsw_xstats.c | 288 + dpdk/drivers/event/dsw/meson.build | 6 + .../event/dsw/rte_pmd_dsw_event_version.map | 3 + dpdk/drivers/event/meson.build | 7 + dpdk/drivers/event/octeontx/Makefile | 51 +- dpdk/drivers/event/octeontx/meson.build | 14 + ...map => rte_pmd_octeontx_event_version.map} | 0 dpdk/drivers/event/octeontx/ssovf_evdev.c | 235 +- dpdk/drivers/event/octeontx/ssovf_evdev.h | 81 +- .../event/octeontx/ssovf_evdev_selftest.c | 1523 + dpdk/drivers/event/octeontx/ssovf_probe.c | 290 + dpdk/drivers/event/octeontx/ssovf_worker.c | 108 +- dpdk/drivers/event/octeontx/ssovf_worker.h | 36 +- dpdk/drivers/event/octeontx/timvf_evdev.c | 405 + dpdk/drivers/event/octeontx/timvf_evdev.h | 225 + dpdk/drivers/event/octeontx/timvf_probe.c | 148 + dpdk/drivers/event/octeontx/timvf_worker.c | 199 + dpdk/drivers/event/octeontx/timvf_worker.h | 443 + dpdk/drivers/event/opdl/Makefile | 39 + dpdk/drivers/event/opdl/meson.build | 11 + dpdk/drivers/event/opdl/opdl_evdev.c | 766 + dpdk/drivers/event/opdl/opdl_evdev.h | 314 + dpdk/drivers/event/opdl/opdl_evdev_init.c | 943 + dpdk/drivers/event/opdl/opdl_evdev_xstats.c | 180 + dpdk/drivers/event/opdl/opdl_log.h | 21 + dpdk/drivers/event/opdl/opdl_ring.c | 1272 + dpdk/drivers/event/opdl/opdl_ring.h | 614 + dpdk/drivers/event/opdl/opdl_test.c | 1057 + .../event/opdl/rte_pmd_opdl_event_version.map | 3 + dpdk/drivers/event/skeleton/Makefile | 32 +- dpdk/drivers/event/skeleton/meson.build | 5 + .../event/skeleton/skeleton_eventdev.c | 35 +- .../event/skeleton/skeleton_eventdev.h | 32 +- dpdk/drivers/event/sw/Makefile | 34 +- dpdk/drivers/event/sw/event_ring.h | 32 +- dpdk/drivers/event/sw/iq_chunk.h | 196 + dpdk/drivers/event/sw/iq_ring.h | 172 - dpdk/drivers/event/sw/meson.build | 11 + dpdk/drivers/event/sw/sw_evdev.c | 312 +- dpdk/drivers/event/sw/sw_evdev.h | 79 +- dpdk/drivers/event/sw/sw_evdev_log.h | 23 + dpdk/drivers/event/sw/sw_evdev_scheduler.c | 109 +- dpdk/drivers/event/sw/sw_evdev_selftest.c | 3401 ++ dpdk/drivers/event/sw/sw_evdev_worker.c | 76 +- dpdk/drivers/event/sw/sw_evdev_xstats.c | 44 +- dpdk/drivers/mempool/Makefile | 36 +- dpdk/drivers/mempool/bucket/Makefile | 27 + dpdk/drivers/mempool/bucket/meson.build | 9 + .../mempool/bucket/rte_mempool_bucket.c | 628 + .../bucket/rte_mempool_bucket_version.map | 4 + dpdk/drivers/mempool/dpaa/Makefile | 36 +- dpdk/drivers/mempool/dpaa/dpaa_mempool.c | 145 +- dpdk/drivers/mempool/dpaa/dpaa_mempool.h | 41 +- dpdk/drivers/mempool/dpaa/meson.build | 12 + .../mempool/dpaa/rte_mempool_dpaa_version.map | 2 +- dpdk/drivers/mempool/dpaa2/Makefile | 44 +- dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.c | 147 +- dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool.h | 32 +- .../mempool/dpaa2/dpaa2_hw_mempool_logs.h | 38 + dpdk/drivers/mempool/dpaa2/meson.build | 14 + .../drivers/mempool/dpaa2/rte_dpaa2_mempool.h | 53 + .../dpaa2/rte_mempool_dpaa2_version.map | 8 + dpdk/drivers/mempool/meson.build | 7 + dpdk/drivers/mempool/octeontx/Makefile | 36 +- dpdk/drivers/mempool/octeontx/meson.build | 8 + .../drivers/mempool/octeontx/octeontx_fpavf.c | 97 +- .../drivers/mempool/octeontx/octeontx_fpavf.h | 43 +- dpdk/drivers/mempool/octeontx/octeontx_mbox.c | 242 - dpdk/drivers/mempool/octeontx/octeontx_mbox.h | 64 - .../mempool/octeontx/octeontx_pool_logs.h | 68 +- .../drivers/mempool/octeontx/octeontx_ssovf.c | 299 - .../mempool/octeontx/rte_mempool_octeontx.c | 97 +- .../octeontx/rte_mempool_octeontx_version.map | 6 - dpdk/drivers/mempool/ring/Makefile | 31 +- dpdk/drivers/mempool/ring/meson.build | 4 + dpdk/drivers/mempool/ring/rte_mempool_ring.c | 33 +- dpdk/drivers/mempool/stack/Makefile | 31 +- dpdk/drivers/mempool/stack/meson.build | 4 + .../drivers/mempool/stack/rte_mempool_stack.c | 33 +- dpdk/drivers/meson.build | 155 + dpdk/drivers/net/Makefile | 50 +- dpdk/drivers/net/af_packet/Makefile | 37 +- dpdk/drivers/net/af_packet/meson.build | 7 + .../drivers/net/af_packet/rte_eth_af_packet.c | 223 +- dpdk/drivers/net/ark/Makefile | 32 +- dpdk/drivers/net/ark/ark_ddm.c | 45 +- dpdk/drivers/net/ark/ark_ddm.h | 40 +- dpdk/drivers/net/ark/ark_ethdev.c | 50 +- dpdk/drivers/net/ark/ark_ethdev_rx.c | 88 +- dpdk/drivers/net/ark/ark_ethdev_rx.h | 35 +- dpdk/drivers/net/ark/ark_ethdev_tx.c | 35 +- dpdk/drivers/net/ark/ark_ethdev_tx.h | 35 +- dpdk/drivers/net/ark/ark_ext.h | 35 +- dpdk/drivers/net/ark/ark_global.h | 35 +- dpdk/drivers/net/ark/ark_logs.h | 33 +- dpdk/drivers/net/ark/ark_mpu.c | 33 +- dpdk/drivers/net/ark/ark_mpu.h | 33 +- dpdk/drivers/net/ark/ark_pktchkr.c | 35 +- dpdk/drivers/net/ark/ark_pktchkr.h | 33 +- dpdk/drivers/net/ark/ark_pktdir.c | 33 +- dpdk/drivers/net/ark/ark_pktdir.h | 33 +- dpdk/drivers/net/ark/ark_pktgen.c | 35 +- dpdk/drivers/net/ark/ark_pktgen.h | 33 +- dpdk/drivers/net/ark/ark_rqp.c | 33 +- dpdk/drivers/net/ark/ark_rqp.h | 33 +- dpdk/drivers/net/ark/ark_udm.c | 33 +- dpdk/drivers/net/ark/ark_udm.h | 33 +- dpdk/drivers/net/ark/meson.build | 13 + dpdk/drivers/net/atlantic/Makefile | 35 + dpdk/drivers/net/atlantic/atl_common.h | 96 + dpdk/drivers/net/atlantic/atl_ethdev.c | 1539 + dpdk/drivers/net/atlantic/atl_ethdev.h | 107 + dpdk/drivers/net/atlantic/atl_hw_regs.c | 52 + dpdk/drivers/net/atlantic/atl_hw_regs.h | 53 + dpdk/drivers/net/atlantic/atl_logs.h | 31 + dpdk/drivers/net/atlantic/atl_rxtx.c | 1357 + dpdk/drivers/net/atlantic/atl_types.h | 186 + dpdk/drivers/net/atlantic/hw_atl/hw_atl_b0.c | 510 + dpdk/drivers/net/atlantic/hw_atl/hw_atl_b0.h | 40 + .../net/atlantic/hw_atl/hw_atl_b0_internal.h | 145 + dpdk/drivers/net/atlantic/hw_atl/hw_atl_llh.c | 1490 + dpdk/drivers/net/atlantic/hw_atl/hw_atl_llh.h | 714 + .../net/atlantic/hw_atl/hw_atl_llh_internal.h | 2407 ++ .../net/atlantic/hw_atl/hw_atl_utils.c | 942 + .../net/atlantic/hw_atl/hw_atl_utils.h | 510 + .../net/atlantic/hw_atl/hw_atl_utils_fw2x.c | 618 + dpdk/drivers/net/atlantic/meson.build | 12 + .../net/atlantic/rte_pmd_atlantic_version.map | 4 + dpdk/drivers/net/avf/Makefile | 54 + dpdk/drivers/net/avf/avf.h | 216 + dpdk/drivers/net/avf/avf_ethdev.c | 1428 + dpdk/drivers/net/avf/avf_log.h | 44 + dpdk/drivers/net/avf/avf_rxtx.c | 1955 + dpdk/drivers/net/avf/avf_rxtx.h | 268 + dpdk/drivers/net/avf/avf_rxtx_vec_common.h | 210 + dpdk/drivers/net/avf/avf_rxtx_vec_sse.c | 656 + dpdk/drivers/net/avf/avf_vchnl.c | 810 + dpdk/drivers/net/avf/base/README | 19 + dpdk/drivers/net/avf/base/avf_adminq.c | 1010 + dpdk/drivers/net/avf/base/avf_adminq.h | 166 + dpdk/drivers/net/avf/base/avf_adminq_cmd.h | 2842 ++ dpdk/drivers/net/avf/base/avf_alloc.h | 65 + dpdk/drivers/net/avf/base/avf_common.c | 1845 + dpdk/drivers/net/avf/base/avf_devids.h | 43 + dpdk/drivers/net/avf/base/avf_hmc.h | 245 + dpdk/drivers/net/avf/base/avf_lan_hmc.h | 200 + dpdk/drivers/net/avf/base/avf_osdep.h | 187 + dpdk/drivers/net/avf/base/avf_prototype.h | 206 + dpdk/drivers/net/avf/base/avf_register.h | 346 + dpdk/drivers/net/avf/base/avf_status.h | 108 + dpdk/drivers/net/avf/base/avf_type.h | 2024 + dpdk/drivers/net/avf/base/meson.build | 23 + dpdk/drivers/net/avf/base/virtchnl.h | 787 + dpdk/drivers/net/avf/meson.build | 20 + dpdk/drivers/net/avf/rte_pmd_avf_version.map | 4 + dpdk/drivers/net/avp/Makefile | 33 +- dpdk/drivers/net/avp/avp_ethdev.c | 79 +- dpdk/drivers/net/avp/avp_logs.h | 41 +- dpdk/drivers/net/avp/meson.build | 8 + dpdk/drivers/net/avp/rte_avp_common.h | 57 +- dpdk/drivers/net/avp/rte_avp_fifo.h | 57 +- dpdk/drivers/net/avp/rte_pmd_avp_version.map | 3 +- dpdk/drivers/net/axgbe/Makefile | 35 + dpdk/drivers/net/axgbe/axgbe_common.h | 1710 + dpdk/drivers/net/axgbe/axgbe_dev.c | 1103 + dpdk/drivers/net/axgbe/axgbe_ethdev.c | 766 + dpdk/drivers/net/axgbe/axgbe_ethdev.h | 586 + dpdk/drivers/net/axgbe/axgbe_i2c.c | 331 + dpdk/drivers/net/axgbe/axgbe_logs.h | 26 + dpdk/drivers/net/axgbe/axgbe_mdio.c | 1066 + dpdk/drivers/net/axgbe/axgbe_phy.h | 192 + dpdk/drivers/net/axgbe/axgbe_phy_impl.c | 2191 + dpdk/drivers/net/axgbe/axgbe_rxtx.c | 674 + dpdk/drivers/net/axgbe/axgbe_rxtx.h | 186 + dpdk/drivers/net/axgbe/axgbe_rxtx_vec_sse.c | 93 + dpdk/drivers/net/axgbe/meson.build | 19 + .../net/axgbe/rte_pmd_axgbe_version.map | 3 + dpdk/drivers/net/bnx2x/LICENSE.bnx2x_pmd | 28 - dpdk/drivers/net/bnx2x/Makefile | 8 +- dpdk/drivers/net/bnx2x/bnx2x.c | 858 +- dpdk/drivers/net/bnx2x/bnx2x.h | 50 +- dpdk/drivers/net/bnx2x/bnx2x_ethdev.c | 255 +- dpdk/drivers/net/bnx2x/bnx2x_ethdev.h | 11 +- dpdk/drivers/net/bnx2x/bnx2x_logs.h | 62 +- dpdk/drivers/net/bnx2x/bnx2x_rxtx.c | 46 +- dpdk/drivers/net/bnx2x/bnx2x_rxtx.h | 9 +- dpdk/drivers/net/bnx2x/bnx2x_stats.c | 45 +- dpdk/drivers/net/bnx2x/bnx2x_stats.h | 10 +- dpdk/drivers/net/bnx2x/bnx2x_vfpf.c | 59 +- dpdk/drivers/net/bnx2x/bnx2x_vfpf.h | 9 +- dpdk/drivers/net/bnx2x/ecore_fw_defs.h | 10 +- dpdk/drivers/net/bnx2x/ecore_hsi.h | 30 +- dpdk/drivers/net/bnx2x/ecore_init.h | 16 +- dpdk/drivers/net/bnx2x/ecore_init_ops.h | 18 +- dpdk/drivers/net/bnx2x/ecore_mfw_req.h | 10 +- dpdk/drivers/net/bnx2x/ecore_reg.h | 11 +- dpdk/drivers/net/bnx2x/ecore_sp.c | 272 +- dpdk/drivers/net/bnx2x/ecore_sp.h | 14 +- dpdk/drivers/net/bnx2x/elink.c | 7916 ++-- dpdk/drivers/net/bnx2x/elink.h | 246 +- dpdk/drivers/net/bnx2x/meson.build | 14 + dpdk/drivers/net/bnxt/Makefile | 39 +- dpdk/drivers/net/bnxt/bnxt.h | 169 +- dpdk/drivers/net/bnxt/bnxt_cpr.c | 141 +- dpdk/drivers/net/bnxt/bnxt_cpr.h | 51 +- dpdk/drivers/net/bnxt/bnxt_ethdev.c | 1247 +- dpdk/drivers/net/bnxt/bnxt_filter.c | 1142 +- dpdk/drivers/net/bnxt/bnxt_filter.h | 34 +- dpdk/drivers/net/bnxt/bnxt_flow.c | 1171 + dpdk/drivers/net/bnxt/bnxt_hwrm.c | 1112 +- dpdk/drivers/net/bnxt/bnxt_hwrm.h | 66 +- dpdk/drivers/net/bnxt/bnxt_irq.c | 64 +- dpdk/drivers/net/bnxt/bnxt_irq.h | 34 +- dpdk/drivers/net/bnxt/bnxt_nvm_defs.h | 35 +- dpdk/drivers/net/bnxt/bnxt_ring.c | 249 +- dpdk/drivers/net/bnxt/bnxt_ring.h | 43 +- dpdk/drivers/net/bnxt/bnxt_rxq.c | 188 +- dpdk/drivers/net/bnxt/bnxt_rxq.h | 48 +- dpdk/drivers/net/bnxt/bnxt_rxr.c | 225 +- dpdk/drivers/net/bnxt/bnxt_rxr.h | 39 +- dpdk/drivers/net/bnxt/bnxt_stats.c | 301 +- dpdk/drivers/net/bnxt/bnxt_stats.h | 36 +- dpdk/drivers/net/bnxt/bnxt_txq.c | 62 +- dpdk/drivers/net/bnxt/bnxt_txq.h | 38 +- dpdk/drivers/net/bnxt/bnxt_txr.c | 172 +- dpdk/drivers/net/bnxt/bnxt_txr.h | 36 +- dpdk/drivers/net/bnxt/bnxt_util.c | 18 + dpdk/drivers/net/bnxt/bnxt_util.h | 11 + dpdk/drivers/net/bnxt/bnxt_vnic.c | 99 +- dpdk/drivers/net/bnxt/bnxt_vnic.h | 36 +- dpdk/drivers/net/bnxt/hsi_struct_def_dpdk.h | 33308 +++++++++++----- dpdk/drivers/net/bnxt/meson.build | 20 + dpdk/drivers/net/bnxt/rte_pmd_bnxt.c | 86 +- dpdk/drivers/net/bnxt/rte_pmd_bnxt.h | 36 +- dpdk/drivers/net/bonding/Makefile | 34 +- dpdk/drivers/net/bonding/meson.build | 13 + dpdk/drivers/net/bonding/rte_eth_bond.h | 33 +- .../drivers/net/bonding/rte_eth_bond_8023ad.c | 207 +- .../drivers/net/bonding/rte_eth_bond_8023ad.h | 33 +- .../net/bonding/rte_eth_bond_8023ad_private.h | 35 +- dpdk/drivers/net/bonding/rte_eth_bond_alb.c | 37 +- dpdk/drivers/net/bonding/rte_eth_bond_alb.h | 33 +- dpdk/drivers/net/bonding/rte_eth_bond_api.c | 349 +- dpdk/drivers/net/bonding/rte_eth_bond_args.c | 42 +- dpdk/drivers/net/bonding/rte_eth_bond_flow.c | 245 + dpdk/drivers/net/bonding/rte_eth_bond_pmd.c | 1289 +- .../net/bonding/rte_eth_bond_private.h | 102 +- .../net/bonding/rte_pmd_bond_version.map | 9 +- dpdk/drivers/net/cxgbe/Makefile | 49 +- dpdk/drivers/net/cxgbe/base/adapter.h | 175 +- dpdk/drivers/net/cxgbe/base/common.h | 213 +- dpdk/drivers/net/cxgbe/base/t4_chip_type.h | 34 +- dpdk/drivers/net/cxgbe/base/t4_hw.c | 1166 +- dpdk/drivers/net/cxgbe/base/t4_hw.h | 38 +- dpdk/drivers/net/cxgbe/base/t4_msg.h | 288 +- dpdk/drivers/net/cxgbe/base/t4_pci_id_tbl.h | 34 +- dpdk/drivers/net/cxgbe/base/t4_regs.h | 180 +- dpdk/drivers/net/cxgbe/base/t4_regs_values.h | 34 +- dpdk/drivers/net/cxgbe/base/t4_tcb.h | 31 + dpdk/drivers/net/cxgbe/base/t4fw_interface.h | 709 +- dpdk/drivers/net/cxgbe/base/t4vf_hw.c | 880 + dpdk/drivers/net/cxgbe/base/t4vf_hw.h | 15 + dpdk/drivers/net/cxgbe/clip_tbl.c | 193 + dpdk/drivers/net/cxgbe/clip_tbl.h | 31 + dpdk/drivers/net/cxgbe/cxgbe.h | 80 +- dpdk/drivers/net/cxgbe/cxgbe_compat.h | 46 +- dpdk/drivers/net/cxgbe/cxgbe_ethdev.c | 423 +- dpdk/drivers/net/cxgbe/cxgbe_filter.c | 1340 + dpdk/drivers/net/cxgbe/cxgbe_filter.h | 270 + dpdk/drivers/net/cxgbe/cxgbe_flow.c | 1116 + dpdk/drivers/net/cxgbe/cxgbe_flow.h | 44 + dpdk/drivers/net/cxgbe/cxgbe_main.c | 815 +- dpdk/drivers/net/cxgbe/cxgbe_ofld.h | 89 + dpdk/drivers/net/cxgbe/cxgbe_pfvf.h | 45 + dpdk/drivers/net/cxgbe/cxgbevf_ethdev.c | 211 + dpdk/drivers/net/cxgbe/cxgbevf_main.c | 294 + dpdk/drivers/net/cxgbe/l2t.c | 227 + dpdk/drivers/net/cxgbe/l2t.h | 57 + dpdk/drivers/net/cxgbe/meson.build | 16 + dpdk/drivers/net/cxgbe/mps_tcam.c | 241 + dpdk/drivers/net/cxgbe/mps_tcam.h | 52 + dpdk/drivers/net/cxgbe/sge.c | 646 +- dpdk/drivers/net/dpaa/Makefile | 39 +- dpdk/drivers/net/dpaa/dpaa_ethdev.c | 686 +- dpdk/drivers/net/dpaa/dpaa_ethdev.h | 103 +- dpdk/drivers/net/dpaa/dpaa_rxtx.c | 439 +- dpdk/drivers/net/dpaa/dpaa_rxtx.h | 41 +- dpdk/drivers/net/dpaa/meson.build | 14 + dpdk/drivers/net/dpaa/rte_pmd_dpaa.h | 36 + .../drivers/net/dpaa/rte_pmd_dpaa_version.map | 8 + dpdk/drivers/net/dpaa2/Makefile | 44 +- dpdk/drivers/net/dpaa2/base/dpaa2_hw_dpni.c | 62 +- .../net/dpaa2/base/dpaa2_hw_dpni_annot.h | 92 +- dpdk/drivers/net/dpaa2/dpaa2_ethdev.c | 805 +- dpdk/drivers/net/dpaa2/dpaa2_ethdev.h | 68 +- dpdk/drivers/net/dpaa2/dpaa2_pmd_logs.h | 41 + dpdk/drivers/net/dpaa2/dpaa2_rxtx.c | 510 +- dpdk/drivers/net/dpaa2/mc/dpkg.c | 35 +- dpdk/drivers/net/dpaa2/mc/dpni.c | 171 +- dpdk/drivers/net/dpaa2/mc/fsl_dpkg.h | 109 +- dpdk/drivers/net/dpaa2/mc/fsl_dpni.h | 415 +- dpdk/drivers/net/dpaa2/mc/fsl_dpni_cmd.h | 124 +- dpdk/drivers/net/dpaa2/mc/fsl_net.h | 37 +- dpdk/drivers/net/dpaa2/meson.build | 20 + dpdk/drivers/net/e1000/Makefile | 37 +- dpdk/drivers/net/e1000/base/e1000_82571.c | 5 + dpdk/drivers/net/e1000/base/e1000_82575.c | 7 +- dpdk/drivers/net/e1000/base/e1000_defines.h | 1 + dpdk/drivers/net/e1000/base/e1000_i210.c | 1 + dpdk/drivers/net/e1000/base/e1000_ich8lan.c | 2 +- dpdk/drivers/net/e1000/base/e1000_osdep.h | 2 +- dpdk/drivers/net/e1000/base/e1000_phy.h | 8 + dpdk/drivers/net/e1000/base/meson.build | 40 + dpdk/drivers/net/e1000/e1000_ethdev.h | 78 +- dpdk/drivers/net/e1000/e1000_logs.c | 26 + dpdk/drivers/net/e1000/e1000_logs.h | 57 +- dpdk/drivers/net/e1000/em_ethdev.c | 228 +- dpdk/drivers/net/e1000/em_rxtx.c | 155 +- dpdk/drivers/net/e1000/igb_ethdev.c | 390 +- dpdk/drivers/net/e1000/igb_flow.c | 281 +- dpdk/drivers/net/e1000/igb_pf.c | 35 +- dpdk/drivers/net/e1000/igb_regs.h | 33 +- dpdk/drivers/net/e1000/igb_rxtx.c | 297 +- dpdk/drivers/net/e1000/meson.build | 19 + dpdk/drivers/net/ena/Makefile | 4 + dpdk/drivers/net/ena/base/ena_com.c | 707 +- dpdk/drivers/net/ena/base/ena_com.h | 112 +- .../net/ena/base/ena_defs/ena_admin_defs.h | 1172 +- .../net/ena/base/ena_defs/ena_common_defs.h | 8 +- .../net/ena/base/ena_defs/ena_eth_io_defs.h | 758 +- .../net/ena/base/ena_defs/ena_gen_info.h | 4 +- .../net/ena/base/ena_defs/ena_includes.h | 2 - .../net/ena/base/ena_defs/ena_regs_defs.h | 36 + dpdk/drivers/net/ena/base/ena_eth_com.c | 78 +- dpdk/drivers/net/ena/base/ena_eth_com.h | 10 +- dpdk/drivers/net/ena/base/ena_plat.h | 4 + dpdk/drivers/net/ena/base/ena_plat_dpdk.h | 47 +- dpdk/drivers/net/ena/ena_ethdev.c | 1005 +- dpdk/drivers/net/ena/ena_ethdev.h | 38 +- dpdk/drivers/net/ena/ena_logs.h | 14 +- dpdk/drivers/net/ena/ena_platform.h | 2 +- dpdk/drivers/net/ena/meson.build | 11 + dpdk/drivers/net/enetc/Makefile | 23 + dpdk/drivers/net/enetc/base/enetc_hw.h | 226 + dpdk/drivers/net/enetc/enetc.h | 113 + dpdk/drivers/net/enetc/enetc_ethdev.c | 629 + dpdk/drivers/net/enetc/enetc_logs.h | 40 + dpdk/drivers/net/enetc/enetc_rxtx.c | 239 + dpdk/drivers/net/enetc/meson.build | 11 + .../net/enetc/rte_pmd_enetc_version.map | 4 + dpdk/drivers/net/enic/LICENSE | 27 - dpdk/drivers/net/enic/Makefile | 60 +- dpdk/drivers/net/enic/base/cq_desc.h | 34 +- dpdk/drivers/net/enic/base/cq_enet_desc.h | 33 +- dpdk/drivers/net/enic/base/rq_enet_desc.h | 33 +- dpdk/drivers/net/enic/base/vnic_cq.c | 33 +- dpdk/drivers/net/enic/base/vnic_cq.h | 33 +- dpdk/drivers/net/enic/base/vnic_dev.c | 213 +- dpdk/drivers/net/enic/base/vnic_dev.h | 54 +- dpdk/drivers/net/enic/base/vnic_devcmd.h | 143 +- dpdk/drivers/net/enic/base/vnic_enet.h | 36 +- dpdk/drivers/net/enic/base/vnic_intr.c | 33 +- dpdk/drivers/net/enic/base/vnic_intr.h | 33 +- dpdk/drivers/net/enic/base/vnic_nic.h | 39 +- dpdk/drivers/net/enic/base/vnic_resource.h | 33 +- dpdk/drivers/net/enic/base/vnic_rq.c | 33 +- dpdk/drivers/net/enic/base/vnic_rq.h | 37 +- dpdk/drivers/net/enic/base/vnic_rss.c | 33 +- dpdk/drivers/net/enic/base/vnic_rss.h | 32 +- dpdk/drivers/net/enic/base/vnic_stats.h | 33 +- dpdk/drivers/net/enic/base/vnic_wq.c | 42 +- dpdk/drivers/net/enic/base/vnic_wq.h | 46 +- dpdk/drivers/net/enic/base/wq_enet_desc.h | 33 +- dpdk/drivers/net/enic/enic.h | 187 +- dpdk/drivers/net/enic/enic_clsf.c | 56 +- dpdk/drivers/net/enic/enic_compat.h | 40 +- dpdk/drivers/net/enic/enic_ethdev.c | 514 +- dpdk/drivers/net/enic/enic_flow.c | 307 +- dpdk/drivers/net/enic/enic_main.c | 733 +- dpdk/drivers/net/enic/enic_res.c | 132 +- dpdk/drivers/net/enic/enic_res.h | 55 +- dpdk/drivers/net/enic/enic_rxtx.c | 648 +- dpdk/drivers/net/enic/enic_rxtx_common.h | 271 + dpdk/drivers/net/enic/enic_rxtx_vec_avx2.c | 831 + dpdk/drivers/net/enic/meson.build | 35 + dpdk/drivers/net/failsafe/Makefile | 41 +- dpdk/drivers/net/failsafe/failsafe.c | 160 +- dpdk/drivers/net/failsafe/failsafe_args.c | 133 +- dpdk/drivers/net/failsafe/failsafe_eal.c | 150 +- dpdk/drivers/net/failsafe/failsafe_ether.c | 184 +- dpdk/drivers/net/failsafe/failsafe_flow.c | 103 +- dpdk/drivers/net/failsafe/failsafe_intr.c | 536 + dpdk/drivers/net/failsafe/failsafe_ops.c | 648 +- dpdk/drivers/net/failsafe/failsafe_private.h | 194 +- dpdk/drivers/net/failsafe/failsafe_rxtx.c | 89 +- dpdk/drivers/net/failsafe/meson.build | 23 + dpdk/drivers/net/fm10k/Makefile | 35 +- dpdk/drivers/net/fm10k/base/fm10k_mbx.c | 2 +- dpdk/drivers/net/fm10k/base/meson.build | 30 + dpdk/drivers/net/fm10k/fm10k.h | 46 +- dpdk/drivers/net/fm10k/fm10k_ethdev.c | 333 +- dpdk/drivers/net/fm10k/fm10k_logs.h | 49 +- dpdk/drivers/net/fm10k/fm10k_rxtx.c | 113 +- dpdk/drivers/net/fm10k/fm10k_rxtx_vec.c | 41 +- dpdk/drivers/net/fm10k/meson.build | 16 + dpdk/drivers/net/i40e/Makefile | 56 +- dpdk/drivers/net/i40e/base/README | 2 +- dpdk/drivers/net/i40e/base/i40e_adminq.c | 94 +- dpdk/drivers/net/i40e/base/i40e_adminq.h | 38 +- dpdk/drivers/net/i40e/base/i40e_adminq_cmd.h | 263 +- dpdk/drivers/net/i40e/base/i40e_alloc.h | 35 +- dpdk/drivers/net/i40e/base/i40e_common.c | 618 +- dpdk/drivers/net/i40e/base/i40e_dcb.c | 126 +- dpdk/drivers/net/i40e/base/i40e_dcb.h | 35 +- dpdk/drivers/net/i40e/base/i40e_devids.h | 41 +- dpdk/drivers/net/i40e/base/i40e_diag.c | 37 +- dpdk/drivers/net/i40e/base/i40e_diag.h | 37 +- dpdk/drivers/net/i40e/base/i40e_hmc.c | 36 +- dpdk/drivers/net/i40e/base/i40e_hmc.h | 35 +- dpdk/drivers/net/i40e/base/i40e_lan_hmc.c | 50 +- dpdk/drivers/net/i40e/base/i40e_lan_hmc.h | 35 +- dpdk/drivers/net/i40e/base/i40e_nvm.c | 486 +- dpdk/drivers/net/i40e/base/i40e_osdep.h | 40 +- dpdk/drivers/net/i40e/base/i40e_prototype.h | 120 +- dpdk/drivers/net/i40e/base/i40e_register.h | 35 +- dpdk/drivers/net/i40e/base/i40e_status.h | 36 +- dpdk/drivers/net/i40e/base/i40e_type.h | 112 +- dpdk/drivers/net/i40e/base/meson.build | 31 + dpdk/drivers/net/i40e/base/virtchnl.h | 47 +- dpdk/drivers/net/i40e/i40e_ethdev.c | 1382 +- dpdk/drivers/net/i40e/i40e_ethdev.h | 167 +- dpdk/drivers/net/i40e/i40e_ethdev_vf.c | 396 +- dpdk/drivers/net/i40e/i40e_fdir.c | 74 +- dpdk/drivers/net/i40e/i40e_flow.c | 706 +- dpdk/drivers/net/i40e/i40e_logs.h | 33 +- dpdk/drivers/net/i40e/i40e_pf.c | 178 +- dpdk/drivers/net/i40e/i40e_pf.h | 39 +- dpdk/drivers/net/i40e/i40e_regs.h | 33 +- dpdk/drivers/net/i40e/i40e_rxtx.c | 603 +- dpdk/drivers/net/i40e/i40e_rxtx.h | 44 +- dpdk/drivers/net/i40e/i40e_rxtx_vec_altivec.c | 2 +- dpdk/drivers/net/i40e/i40e_rxtx_vec_avx2.c | 792 + dpdk/drivers/net/i40e/i40e_rxtx_vec_common.h | 81 +- dpdk/drivers/net/i40e/i40e_rxtx_vec_neon.c | 37 +- dpdk/drivers/net/i40e/i40e_rxtx_vec_sse.c | 35 +- dpdk/drivers/net/i40e/i40e_tm.c | 33 +- dpdk/drivers/net/i40e/i40e_vf_representor.c | 533 + dpdk/drivers/net/i40e/meson.build | 50 + dpdk/drivers/net/i40e/rte_pmd_i40e.c | 237 +- dpdk/drivers/net/i40e/rte_pmd_i40e.h | 190 +- .../drivers/net/i40e/rte_pmd_i40e_version.map | 7 + dpdk/drivers/net/ifc/Makefile | 35 + dpdk/drivers/net/ifc/base/ifcvf.c | 329 + dpdk/drivers/net/ifc/base/ifcvf.h | 161 + dpdk/drivers/net/ifc/base/ifcvf_osdep.h | 52 + dpdk/drivers/net/ifc/ifcvf_vdpa.c | 861 + dpdk/drivers/net/ifc/meson.build | 8 + dpdk/drivers/net/ifc/rte_pmd_ifc_version.map | 4 + dpdk/drivers/net/ixgbe/Makefile | 39 +- dpdk/drivers/net/ixgbe/base/README | 4 +- dpdk/drivers/net/ixgbe/base/ixgbe_82598.c | 38 +- dpdk/drivers/net/ixgbe/base/ixgbe_82598.h | 35 +- dpdk/drivers/net/ixgbe/base/ixgbe_82599.c | 62 +- dpdk/drivers/net/ixgbe/base/ixgbe_82599.h | 35 +- dpdk/drivers/net/ixgbe/base/ixgbe_api.c | 64 +- dpdk/drivers/net/ixgbe/base/ixgbe_api.h | 36 +- dpdk/drivers/net/ixgbe/base/ixgbe_common.c | 212 +- dpdk/drivers/net/ixgbe/base/ixgbe_common.h | 41 +- dpdk/drivers/net/ixgbe/base/ixgbe_dcb.c | 45 +- dpdk/drivers/net/ixgbe/base/ixgbe_dcb.h | 35 +- dpdk/drivers/net/ixgbe/base/ixgbe_dcb_82598.c | 57 +- dpdk/drivers/net/ixgbe/base/ixgbe_dcb_82598.h | 35 +- dpdk/drivers/net/ixgbe/base/ixgbe_dcb_82599.c | 60 +- dpdk/drivers/net/ixgbe/base/ixgbe_dcb_82599.h | 35 +- dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.c | 52 +- dpdk/drivers/net/ixgbe/base/ixgbe_hv_vf.h | 35 +- dpdk/drivers/net/ixgbe/base/ixgbe_mbx.c | 57 +- dpdk/drivers/net/ixgbe/base/ixgbe_mbx.h | 35 +- dpdk/drivers/net/ixgbe/base/ixgbe_osdep.h | 38 +- dpdk/drivers/net/ixgbe/base/ixgbe_phy.c | 53 +- dpdk/drivers/net/ixgbe/base/ixgbe_phy.h | 35 +- dpdk/drivers/net/ixgbe/base/ixgbe_type.h | 108 +- dpdk/drivers/net/ixgbe/base/ixgbe_vf.c | 42 +- dpdk/drivers/net/ixgbe/base/ixgbe_vf.h | 35 +- dpdk/drivers/net/ixgbe/base/ixgbe_x540.c | 62 +- dpdk/drivers/net/ixgbe/base/ixgbe_x540.h | 35 +- dpdk/drivers/net/ixgbe/base/ixgbe_x550.c | 213 +- dpdk/drivers/net/ixgbe/base/ixgbe_x550.h | 36 +- dpdk/drivers/net/ixgbe/base/meson.build | 35 + dpdk/drivers/net/ixgbe/ixgbe_82599_bypass.c | 33 +- dpdk/drivers/net/ixgbe/ixgbe_bypass.c | 35 +- dpdk/drivers/net/ixgbe/ixgbe_bypass.h | 33 +- dpdk/drivers/net/ixgbe/ixgbe_bypass_api.h | 33 +- dpdk/drivers/net/ixgbe/ixgbe_bypass_defines.h | 33 +- dpdk/drivers/net/ixgbe/ixgbe_ethdev.c | 873 +- dpdk/drivers/net/ixgbe/ixgbe_ethdev.h | 71 +- dpdk/drivers/net/ixgbe/ixgbe_fdir.c | 57 +- dpdk/drivers/net/ixgbe/ixgbe_flow.c | 527 +- dpdk/drivers/net/ixgbe/ixgbe_ipsec.c | 161 +- dpdk/drivers/net/ixgbe/ixgbe_ipsec.h | 40 +- dpdk/drivers/net/ixgbe/ixgbe_logs.h | 49 +- dpdk/drivers/net/ixgbe/ixgbe_pf.c | 57 +- dpdk/drivers/net/ixgbe/ixgbe_regs.h | 33 +- dpdk/drivers/net/ixgbe/ixgbe_rxtx.c | 536 +- dpdk/drivers/net/ixgbe/ixgbe_rxtx.h | 57 +- .../drivers/net/ixgbe/ixgbe_rxtx_vec_common.h | 40 +- dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c | 37 +- dpdk/drivers/net/ixgbe/ixgbe_rxtx_vec_sse.c | 35 +- dpdk/drivers/net/ixgbe/ixgbe_tm.c | 33 +- dpdk/drivers/net/ixgbe/ixgbe_vf_representor.c | 235 + dpdk/drivers/net/ixgbe/meson.build | 36 + dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.c | 265 +- dpdk/drivers/net/ixgbe/rte_pmd_ixgbe.h | 118 +- .../net/ixgbe/rte_pmd_ixgbe_version.map | 10 + dpdk/drivers/net/kni/Makefile | 32 +- dpdk/drivers/net/kni/meson.build | 8 + dpdk/drivers/net/kni/rte_eth_kni.c | 105 +- dpdk/drivers/net/liquidio/Makefile | 35 +- dpdk/drivers/net/liquidio/base/lio_23xx_reg.h | 33 +- dpdk/drivers/net/liquidio/base/lio_23xx_vf.c | 89 +- dpdk/drivers/net/liquidio/base/lio_23xx_vf.h | 38 +- dpdk/drivers/net/liquidio/base/lio_hw_defs.h | 35 +- dpdk/drivers/net/liquidio/base/lio_mbox.c | 35 +- dpdk/drivers/net/liquidio/base/lio_mbox.h | 33 +- dpdk/drivers/net/liquidio/lio_ethdev.c | 276 +- dpdk/drivers/net/liquidio/lio_ethdev.h | 33 +- dpdk/drivers/net/liquidio/lio_logs.h | 53 +- dpdk/drivers/net/liquidio/lio_rxtx.c | 116 +- dpdk/drivers/net/liquidio/lio_rxtx.h | 33 +- dpdk/drivers/net/liquidio/lio_struct.h | 39 +- dpdk/drivers/net/liquidio/meson.build | 8 + ...rsion.map => rte_pmd_liquidio_version.map} | 0 dpdk/drivers/net/meson.build | 45 + dpdk/drivers/net/mlx4/Makefile | 80 +- dpdk/drivers/net/mlx4/meson.build | 102 + dpdk/drivers/net/mlx4/mlx4.c | 379 +- dpdk/drivers/net/mlx4/mlx4.h | 100 +- dpdk/drivers/net/mlx4/mlx4_ethdev.c | 89 +- dpdk/drivers/net/mlx4/mlx4_flow.c | 293 +- dpdk/drivers/net/mlx4/mlx4_flow.h | 38 +- dpdk/drivers/net/mlx4/mlx4_glue.c | 279 + dpdk/drivers/net/mlx4/mlx4_glue.h | 89 + dpdk/drivers/net/mlx4/mlx4_intr.c | 50 +- dpdk/drivers/net/mlx4/mlx4_mr.c | 1523 +- dpdk/drivers/net/mlx4/mlx4_mr.h | 122 + dpdk/drivers/net/mlx4/mlx4_prm.h | 75 +- dpdk/drivers/net/mlx4/mlx4_rxq.c | 192 +- dpdk/drivers/net/mlx4/mlx4_rxtx.c | 1068 +- dpdk/drivers/net/mlx4/mlx4_rxtx.h | 151 +- dpdk/drivers/net/mlx4/mlx4_txq.c | 209 +- dpdk/drivers/net/mlx4/mlx4_utils.c | 34 +- dpdk/drivers/net/mlx4/mlx4_utils.h | 34 +- dpdk/drivers/net/mlx5/Makefile | 414 +- dpdk/drivers/net/mlx5/meson.build | 280 + dpdk/drivers/net/mlx5/mlx5.c | 1752 +- dpdk/drivers/net/mlx5/mlx5.h | 216 +- dpdk/drivers/net/mlx5/mlx5_defs.h | 105 +- dpdk/drivers/net/mlx5/mlx5_ethdev.c | 398 +- dpdk/drivers/net/mlx5/mlx5_flow.c | 4313 +- dpdk/drivers/net/mlx5/mlx5_flow.h | 420 + dpdk/drivers/net/mlx5/mlx5_flow_dv.c | 2402 ++ dpdk/drivers/net/mlx5/mlx5_flow_tcf.c | 5956 +++ dpdk/drivers/net/mlx5/mlx5_flow_verbs.c | 1737 + dpdk/drivers/net/mlx5/mlx5_glue.c | 538 + dpdk/drivers/net/mlx5/mlx5_glue.h | 171 + dpdk/drivers/net/mlx5/mlx5_mac.c | 178 +- dpdk/drivers/net/mlx5/mlx5_mr.c | 1649 +- dpdk/drivers/net/mlx5/mlx5_mr.h | 120 + dpdk/drivers/net/mlx5/mlx5_nl.c | 916 + dpdk/drivers/net/mlx5/mlx5_prm.h | 307 +- dpdk/drivers/net/mlx5/mlx5_rss.c | 43 +- dpdk/drivers/net/mlx5/mlx5_rxmode.c | 46 +- dpdk/drivers/net/mlx5/mlx5_rxq.c | 1084 +- dpdk/drivers/net/mlx5/mlx5_rxtx.c | 1030 +- dpdk/drivers/net/mlx5/mlx5_rxtx.h | 488 +- dpdk/drivers/net/mlx5/mlx5_rxtx_vec.c | 147 +- dpdk/drivers/net/mlx5/mlx5_rxtx_vec.h | 58 +- dpdk/drivers/net/mlx5/mlx5_rxtx_vec_neon.h | 74 +- dpdk/drivers/net/mlx5/mlx5_rxtx_vec_sse.h | 69 +- dpdk/drivers/net/mlx5/mlx5_socket.c | 34 +- dpdk/drivers/net/mlx5/mlx5_stats.c | 136 +- dpdk/drivers/net/mlx5/mlx5_trigger.c | 74 +- dpdk/drivers/net/mlx5/mlx5_txq.c | 348 +- dpdk/drivers/net/mlx5/mlx5_utils.h | 44 +- dpdk/drivers/net/mlx5/mlx5_vlan.c | 62 +- dpdk/drivers/net/mrvl/Makefile | 68 - dpdk/drivers/net/mrvl/mrvl_ethdev.c | 2390 -- dpdk/drivers/net/mrvl/mrvl_ethdev.h | 113 - dpdk/drivers/net/mrvl/mrvl_qos.c | 636 - dpdk/drivers/net/mrvl/mrvl_qos.h | 113 - dpdk/drivers/net/mvneta/Makefile | 42 + dpdk/drivers/net/mvneta/meson.build | 28 + dpdk/drivers/net/mvneta/mvneta_ethdev.c | 987 + dpdk/drivers/net/mvneta/mvneta_ethdev.h | 79 + dpdk/drivers/net/mvneta/mvneta_rxtx.c | 1030 + dpdk/drivers/net/mvneta/mvneta_rxtx.h | 38 + .../net/mvneta/rte_pmd_mvneta_version.map | 3 + dpdk/drivers/net/mvpp2/Makefile | 45 + dpdk/drivers/net/mvpp2/meson.build | 27 + dpdk/drivers/net/mvpp2/mrvl_ethdev.c | 3033 ++ dpdk/drivers/net/mvpp2/mrvl_ethdev.h | 230 + dpdk/drivers/net/mvpp2/mrvl_flow.c | 2824 ++ dpdk/drivers/net/mvpp2/mrvl_flow.h | 15 + dpdk/drivers/net/mvpp2/mrvl_mtr.c | 512 + dpdk/drivers/net/mvpp2/mrvl_mtr.h | 15 + dpdk/drivers/net/mvpp2/mrvl_qos.c | 912 + dpdk/drivers/net/mvpp2/mrvl_qos.h | 107 + dpdk/drivers/net/mvpp2/mrvl_tm.c | 1009 + dpdk/drivers/net/mvpp2/mrvl_tm.h | 15 + .../rte_pmd_mvpp2_version.map} | 0 dpdk/drivers/net/netvsc/Makefile | 24 + dpdk/drivers/net/netvsc/hn_ethdev.c | 890 + dpdk/drivers/net/netvsc/hn_logs.h | 36 + dpdk/drivers/net/netvsc/hn_nvs.c | 550 + dpdk/drivers/net/netvsc/hn_nvs.h | 238 + dpdk/drivers/net/netvsc/hn_rndis.c | 1134 + dpdk/drivers/net/netvsc/hn_rndis.h | 33 + dpdk/drivers/net/netvsc/hn_rxtx.c | 1414 + dpdk/drivers/net/netvsc/hn_var.h | 212 + dpdk/drivers/net/netvsc/hn_vf.c | 549 + dpdk/drivers/net/netvsc/meson.build | 10 + dpdk/drivers/net/netvsc/ndis.h | 378 + dpdk/drivers/net/netvsc/rndis.h | 414 + .../net/netvsc/rte_pmd_netvsc_version.map | 5 + dpdk/drivers/net/nfp/Makefile | 50 +- dpdk/drivers/net/nfp/meson.build | 21 + dpdk/drivers/net/nfp/nfp_net.c | 1048 +- dpdk/drivers/net/nfp/nfp_net_ctrl.h | 6 + dpdk/drivers/net/nfp/nfp_net_eth.h | 82 - dpdk/drivers/net/nfp/nfp_net_logs.h | 33 +- dpdk/drivers/net/nfp/nfp_net_pmd.h | 46 +- dpdk/drivers/net/nfp/nfp_nfpu.c | 126 - dpdk/drivers/net/nfp/nfp_nfpu.h | 55 - dpdk/drivers/net/nfp/nfp_nspu.c | 642 - dpdk/drivers/net/nfp/nfp_nspu.h | 83 - .../net/nfp/nfpcore/nfp-common/nfp_cppat.h | 722 + .../net/nfp/nfpcore/nfp-common/nfp_platform.h | 35 + .../net/nfp/nfpcore/nfp-common/nfp_resid.h | 592 + .../drivers/net/nfp/nfpcore/nfp6000/nfp6000.h | 40 + .../drivers/net/nfp/nfpcore/nfp6000/nfp_xpb.h | 26 + dpdk/drivers/net/nfp/nfpcore/nfp_cpp.h | 781 + .../net/nfp/nfpcore/nfp_cpp_pcie_ops.c | 845 + dpdk/drivers/net/nfp/nfpcore/nfp_cppcore.c | 858 + dpdk/drivers/net/nfp/nfpcore/nfp_crc.c | 49 + dpdk/drivers/net/nfp/nfpcore/nfp_crc.h | 19 + dpdk/drivers/net/nfp/nfpcore/nfp_hwinfo.c | 199 + dpdk/drivers/net/nfp/nfpcore/nfp_hwinfo.h | 85 + dpdk/drivers/net/nfp/nfpcore/nfp_mip.c | 154 + dpdk/drivers/net/nfp/nfpcore/nfp_mip.h | 21 + dpdk/drivers/net/nfp/nfpcore/nfp_mutex.c | 424 + dpdk/drivers/net/nfp/nfpcore/nfp_nffw.c | 235 + dpdk/drivers/net/nfp/nfpcore/nfp_nffw.h | 86 + dpdk/drivers/net/nfp/nfpcore/nfp_nsp.c | 427 + dpdk/drivers/net/nfp/nfpcore/nfp_nsp.h | 304 + dpdk/drivers/net/nfp/nfpcore/nfp_nsp_cmds.c | 109 + dpdk/drivers/net/nfp/nfpcore/nfp_nsp_eth.c | 665 + dpdk/drivers/net/nfp/nfpcore/nfp_resource.c | 266 + dpdk/drivers/net/nfp/nfpcore/nfp_resource.h | 52 + dpdk/drivers/net/nfp/nfpcore/nfp_rtsym.c | 327 + dpdk/drivers/net/nfp/nfpcore/nfp_rtsym.h | 61 + dpdk/drivers/net/nfp/nfpcore/nfp_target.h | 579 + dpdk/drivers/net/null/meson.build | 5 + dpdk/drivers/net/null/rte_eth_null.c | 95 +- dpdk/drivers/net/octeontx/Makefile | 37 +- dpdk/drivers/net/octeontx/base/meson.build | 25 + dpdk/drivers/net/octeontx/base/octeontx_bgx.c | 54 +- dpdk/drivers/net/octeontx/base/octeontx_bgx.h | 32 +- dpdk/drivers/net/octeontx/base/octeontx_io.h | 34 +- .../net/octeontx/base/octeontx_pki_var.h | 45 +- .../net/octeontx/base/octeontx_pkivf.c | 43 +- .../net/octeontx/base/octeontx_pkivf.h | 42 +- .../net/octeontx/base/octeontx_pkovf.c | 33 +- .../net/octeontx/base/octeontx_pkovf.h | 32 +- dpdk/drivers/net/octeontx/meson.build | 15 + dpdk/drivers/net/octeontx/octeontx_ethdev.c | 222 +- dpdk/drivers/net/octeontx/octeontx_ethdev.h | 38 +- dpdk/drivers/net/octeontx/octeontx_logs.h | 75 +- dpdk/drivers/net/octeontx/octeontx_rxtx.c | 69 +- dpdk/drivers/net/octeontx/octeontx_rxtx.h | 67 +- .../net/octeontx/rte_pmd_octeontx_version.map | 8 +- dpdk/drivers/net/pcap/Makefile | 35 +- dpdk/drivers/net/pcap/meson.build | 12 + dpdk/drivers/net/pcap/rte_eth_pcap.c | 647 +- dpdk/drivers/net/qede/LICENSE.qede_pmd | 28 - dpdk/drivers/net/qede/Makefile | 12 +- dpdk/drivers/net/qede/base/bcm_osal.c | 17 +- dpdk/drivers/net/qede/base/bcm_osal.h | 11 +- dpdk/drivers/net/qede/base/common_hsi.h | 27 +- dpdk/drivers/net/qede/base/ecore.h | 92 +- .../drivers/net/qede/base/ecore_attn_values.h | 8 +- dpdk/drivers/net/qede/base/ecore_chain.h | 59 +- dpdk/drivers/net/qede/base/ecore_cxt.c | 53 +- dpdk/drivers/net/qede/base/ecore_cxt.h | 12 +- dpdk/drivers/net/qede/base/ecore_cxt_api.h | 8 +- dpdk/drivers/net/qede/base/ecore_dcbx.c | 109 +- dpdk/drivers/net/qede/base/ecore_dcbx.h | 8 +- dpdk/drivers/net/qede/base/ecore_dcbx_api.h | 18 +- dpdk/drivers/net/qede/base/ecore_dev.c | 2224 +- dpdk/drivers/net/qede/base/ecore_dev_api.h | 201 +- .../net/qede/base/ecore_gtt_reg_addr.h | 8 +- dpdk/drivers/net/qede/base/ecore_gtt_values.h | 8 +- dpdk/drivers/net/qede/base/ecore_hsi_common.h | 166 +- .../net/qede/base/ecore_hsi_debug_tools.h | 131 +- dpdk/drivers/net/qede/base/ecore_hsi_eth.h | 92 +- .../net/qede/base/ecore_hsi_init_func.h | 44 +- .../net/qede/base/ecore_hsi_init_tool.h | 115 +- dpdk/drivers/net/qede/base/ecore_hw.c | 251 +- dpdk/drivers/net/qede/base/ecore_hw.h | 52 +- dpdk/drivers/net/qede/base/ecore_hw_defs.h | 8 +- .../net/qede/base/ecore_init_fw_funcs.c | 225 +- .../net/qede/base/ecore_init_fw_funcs.h | 55 +- dpdk/drivers/net/qede/base/ecore_init_ops.c | 66 +- dpdk/drivers/net/qede/base/ecore_init_ops.h | 8 +- dpdk/drivers/net/qede/base/ecore_int.c | 141 +- dpdk/drivers/net/qede/base/ecore_int.h | 12 +- dpdk/drivers/net/qede/base/ecore_int_api.h | 22 +- dpdk/drivers/net/qede/base/ecore_iov_api.h | 36 +- dpdk/drivers/net/qede/base/ecore_iro.h | 172 +- dpdk/drivers/net/qede/base/ecore_iro_values.h | 92 +- dpdk/drivers/net/qede/base/ecore_l2.c | 169 +- dpdk/drivers/net/qede/base/ecore_l2.h | 8 +- dpdk/drivers/net/qede/base/ecore_l2_api.h | 49 +- dpdk/drivers/net/qede/base/ecore_mcp.c | 478 +- dpdk/drivers/net/qede/base/ecore_mcp.h | 29 +- dpdk/drivers/net/qede/base/ecore_mcp_api.h | 96 +- dpdk/drivers/net/qede/base/ecore_mng_tlv.c | 6 + dpdk/drivers/net/qede/base/ecore_proto_if.h | 13 +- dpdk/drivers/net/qede/base/ecore_rt_defs.h | 598 +- dpdk/drivers/net/qede/base/ecore_sp_api.h | 8 +- .../drivers/net/qede/base/ecore_sp_commands.c | 16 +- .../drivers/net/qede/base/ecore_sp_commands.h | 11 +- dpdk/drivers/net/qede/base/ecore_spq.c | 100 +- dpdk/drivers/net/qede/base/ecore_spq.h | 21 +- dpdk/drivers/net/qede/base/ecore_sriov.c | 133 +- dpdk/drivers/net/qede/base/ecore_sriov.h | 8 +- dpdk/drivers/net/qede/base/ecore_status.h | 8 +- dpdk/drivers/net/qede/base/ecore_utils.h | 8 +- dpdk/drivers/net/qede/base/ecore_vf.c | 33 +- dpdk/drivers/net/qede/base/ecore_vf.h | 8 +- dpdk/drivers/net/qede/base/ecore_vf_api.h | 8 +- dpdk/drivers/net/qede/base/ecore_vfpf_if.h | 10 +- dpdk/drivers/net/qede/base/eth_common.h | 16 +- dpdk/drivers/net/qede/base/mcp_public.h | 55 +- dpdk/drivers/net/qede/base/meson.build | 60 + dpdk/drivers/net/qede/base/nvm_cfg.h | 8 +- dpdk/drivers/net/qede/base/reg_addr.h | 94 +- dpdk/drivers/net/qede/meson.build | 12 + dpdk/drivers/net/qede/qede_ethdev.c | 759 +- dpdk/drivers/net/qede/qede_ethdev.h | 88 +- dpdk/drivers/net/qede/qede_fdir.c | 472 - dpdk/drivers/net/qede/qede_filter.c | 1546 + dpdk/drivers/net/qede/qede_if.h | 8 +- dpdk/drivers/net/qede/qede_logs.h | 65 +- dpdk/drivers/net/qede/qede_main.c | 35 +- dpdk/drivers/net/qede/qede_rxtx.c | 194 +- dpdk/drivers/net/qede/qede_rxtx.h | 38 +- dpdk/drivers/net/ring/Makefile | 32 +- dpdk/drivers/net/ring/meson.build | 6 + dpdk/drivers/net/ring/rte_eth_ring.c | 103 +- dpdk/drivers/net/ring/rte_eth_ring.h | 33 +- dpdk/drivers/net/sfc/Makefile | 37 +- dpdk/drivers/net/sfc/base/README | 24 +- dpdk/drivers/net/sfc/base/ef10_ev.c | 219 +- dpdk/drivers/net/sfc/base/ef10_filter.c | 265 +- dpdk/drivers/net/sfc/base/ef10_image.c | 886 + dpdk/drivers/net/sfc/base/ef10_impl.h | 257 +- dpdk/drivers/net/sfc/base/ef10_intr.c | 48 +- dpdk/drivers/net/sfc/base/ef10_mac.c | 253 +- dpdk/drivers/net/sfc/base/ef10_mcdi.c | 57 +- dpdk/drivers/net/sfc/base/ef10_nic.c | 1198 +- dpdk/drivers/net/sfc/base/ef10_nvram.c | 394 +- dpdk/drivers/net/sfc/base/ef10_phy.c | 263 +- dpdk/drivers/net/sfc/base/ef10_rx.c | 408 +- .../net/sfc/base/ef10_signed_image_layout.h | 70 + dpdk/drivers/net/sfc/base/ef10_tlv_layout.h | 130 +- dpdk/drivers/net/sfc/base/ef10_tx.c | 240 +- dpdk/drivers/net/sfc/base/ef10_vpd.c | 67 +- dpdk/drivers/net/sfc/base/efx.h | 1098 +- dpdk/drivers/net/sfc/base/efx_annote.h | 103 + dpdk/drivers/net/sfc/base/efx_bootcfg.c | 768 +- dpdk/drivers/net/sfc/base/efx_check.h | 159 +- dpdk/drivers/net/sfc/base/efx_crc32.c | 30 +- dpdk/drivers/net/sfc/base/efx_ev.c | 61 +- dpdk/drivers/net/sfc/base/efx_filter.c | 169 +- dpdk/drivers/net/sfc/base/efx_hash.c | 30 +- dpdk/drivers/net/sfc/base/efx_impl.h | 257 +- dpdk/drivers/net/sfc/base/efx_intr.c | 51 +- dpdk/drivers/net/sfc/base/efx_lic.c | 333 +- dpdk/drivers/net/sfc/base/efx_mac.c | 87 +- dpdk/drivers/net/sfc/base/efx_mcdi.c | 299 +- dpdk/drivers/net/sfc/base/efx_mcdi.h | 59 +- dpdk/drivers/net/sfc/base/efx_mon.c | 769 +- dpdk/drivers/net/sfc/base/efx_nic.c | 598 +- dpdk/drivers/net/sfc/base/efx_nvram.c | 273 +- dpdk/drivers/net/sfc/base/efx_phy.c | 106 +- dpdk/drivers/net/sfc/base/efx_phy_ids.h | 30 +- dpdk/drivers/net/sfc/base/efx_port.c | 36 +- dpdk/drivers/net/sfc/base/efx_regs.h | 30 +- dpdk/drivers/net/sfc/base/efx_regs_ef10.h | 260 +- dpdk/drivers/net/sfc/base/efx_regs_mcdi.h | 9398 +++-- dpdk/drivers/net/sfc/base/efx_regs_mcdi_aoe.h | 2914 ++ dpdk/drivers/net/sfc/base/efx_regs_pci.h | 30 +- dpdk/drivers/net/sfc/base/efx_rx.c | 443 +- dpdk/drivers/net/sfc/base/efx_sram.c | 44 +- dpdk/drivers/net/sfc/base/efx_tunnel.c | 469 + dpdk/drivers/net/sfc/base/efx_tx.c | 218 +- dpdk/drivers/net/sfc/base/efx_types.h | 71 +- dpdk/drivers/net/sfc/base/efx_vpd.c | 42 +- dpdk/drivers/net/sfc/base/hunt_impl.h | 30 +- dpdk/drivers/net/sfc/base/hunt_nic.c | 219 +- .../net/sfc/base/mc_driver_pcol_strs.h | 102 + dpdk/drivers/net/sfc/base/mcdi_mon.c | 430 +- dpdk/drivers/net/sfc/base/mcdi_mon.h | 35 +- dpdk/drivers/net/sfc/base/medford2_impl.h | 35 + dpdk/drivers/net/sfc/base/medford2_nic.c | 155 + dpdk/drivers/net/sfc/base/medford_impl.h | 38 +- dpdk/drivers/net/sfc/base/medford_nic.c | 284 +- dpdk/drivers/net/sfc/base/meson.build | 79 + dpdk/drivers/net/sfc/base/siena_flash.h | 49 +- dpdk/drivers/net/sfc/base/siena_impl.h | 54 +- dpdk/drivers/net/sfc/base/siena_mac.c | 70 +- dpdk/drivers/net/sfc/base/siena_mcdi.c | 42 +- dpdk/drivers/net/sfc/base/siena_nic.c | 286 +- dpdk/drivers/net/sfc/base/siena_nvram.c | 65 +- dpdk/drivers/net/sfc/base/siena_phy.c | 69 +- dpdk/drivers/net/sfc/base/siena_sram.c | 30 +- dpdk/drivers/net/sfc/base/siena_vpd.c | 59 +- dpdk/drivers/net/sfc/efsys.h | 81 +- dpdk/drivers/net/sfc/meson.build | 61 + ...fx_version.map => rte_pmd_sfc_version.map} | 0 dpdk/drivers/net/sfc/sfc.c | 513 +- dpdk/drivers/net/sfc/sfc.h | 73 +- dpdk/drivers/net/sfc/sfc_debug.h | 28 +- dpdk/drivers/net/sfc/sfc_dp.c | 31 +- dpdk/drivers/net/sfc/sfc_dp.h | 35 +- dpdk/drivers/net/sfc/sfc_dp_rx.h | 87 +- dpdk/drivers/net/sfc/sfc_dp_tx.h | 66 +- dpdk/drivers/net/sfc/sfc_ef10.h | 60 +- dpdk/drivers/net/sfc/sfc_ef10_essb_rx.c | 731 + dpdk/drivers/net/sfc/sfc_ef10_rx.c | 469 +- dpdk/drivers/net/sfc/sfc_ef10_rx_ev.h | 175 + dpdk/drivers/net/sfc/sfc_ef10_tx.c | 450 +- dpdk/drivers/net/sfc/sfc_ethdev.c | 449 +- dpdk/drivers/net/sfc/sfc_ev.c | 80 +- dpdk/drivers/net/sfc/sfc_ev.h | 28 +- dpdk/drivers/net/sfc/sfc_filter.c | 40 +- dpdk/drivers/net/sfc/sfc_filter.h | 36 +- dpdk/drivers/net/sfc/sfc_flow.c | 1347 +- dpdk/drivers/net/sfc/sfc_flow.h | 49 +- dpdk/drivers/net/sfc/sfc_intr.c | 36 +- dpdk/drivers/net/sfc/sfc_kvargs.c | 30 +- dpdk/drivers/net/sfc/sfc_kvargs.h | 50 +- dpdk/drivers/net/sfc/sfc_log.h | 103 +- dpdk/drivers/net/sfc/sfc_mcdi.c | 53 +- dpdk/drivers/net/sfc/sfc_port.c | 99 +- dpdk/drivers/net/sfc/sfc_rx.c | 512 +- dpdk/drivers/net/sfc/sfc_rx.h | 44 +- dpdk/drivers/net/sfc/sfc_tso.c | 54 +- dpdk/drivers/net/sfc/sfc_tso.h | 23 + dpdk/drivers/net/sfc/sfc_tweak.h | 34 +- dpdk/drivers/net/sfc/sfc_tx.c | 216 +- dpdk/drivers/net/sfc/sfc_tx.h | 34 +- dpdk/drivers/net/softnic/Makefile | 59 +- dpdk/drivers/net/softnic/conn.c | 331 + dpdk/drivers/net/softnic/conn.h | 49 + dpdk/drivers/net/softnic/firmware.cli | 21 + dpdk/drivers/net/softnic/meson.build | 24 + dpdk/drivers/net/softnic/parser.c | 703 + dpdk/drivers/net/softnic/parser.h | 68 + dpdk/drivers/net/softnic/rte_eth_softnic.c | 842 +- dpdk/drivers/net/softnic/rte_eth_softnic.h | 82 +- .../net/softnic/rte_eth_softnic_action.c | 422 + .../drivers/net/softnic/rte_eth_softnic_cli.c | 6100 +++ .../net/softnic/rte_eth_softnic_cryptodev.c | 125 + .../net/softnic/rte_eth_softnic_flow.c | 2287 ++ .../net/softnic/rte_eth_softnic_internals.h | 1057 +- .../net/softnic/rte_eth_softnic_link.c | 98 + .../net/softnic/rte_eth_softnic_mempool.c | 103 + .../net/softnic/rte_eth_softnic_meter.c | 749 + .../net/softnic/rte_eth_softnic_pipeline.c | 1103 + .../drivers/net/softnic/rte_eth_softnic_swq.c | 114 + .../drivers/net/softnic/rte_eth_softnic_tap.c | 118 + .../net/softnic/rte_eth_softnic_thread.c | 3005 ++ dpdk/drivers/net/softnic/rte_eth_softnic_tm.c | 394 +- .../softnic/rte_pmd_eth_softnic_version.map | 7 - .../net/softnic/rte_pmd_softnic_version.map | 13 + dpdk/drivers/net/szedata2/Makefile | 33 +- dpdk/drivers/net/szedata2/meson.build | 7 + dpdk/drivers/net/szedata2/rte_eth_szedata2.c | 910 +- dpdk/drivers/net/szedata2/rte_eth_szedata2.h | 37 +- dpdk/drivers/net/szedata2/szedata2_iobuf.c | 203 - dpdk/drivers/net/szedata2/szedata2_iobuf.h | 356 - dpdk/drivers/net/szedata2/szedata2_logs.h | 22 + dpdk/drivers/net/tap/Makefile | 64 +- dpdk/drivers/net/tap/meson.build | 45 + dpdk/drivers/net/tap/rte_eth_tap.c | 1319 +- dpdk/drivers/net/tap/rte_eth_tap.h | 74 +- dpdk/drivers/net/tap/tap_bpf.h | 117 + dpdk/drivers/net/tap/tap_bpf_api.c | 190 + dpdk/drivers/net/tap/tap_bpf_insns.h | 1696 + dpdk/drivers/net/tap/tap_bpf_program.c | 224 + dpdk/drivers/net/tap/tap_flow.c | 922 +- dpdk/drivers/net/tap/tap_flow.h | 47 +- dpdk/drivers/net/tap/tap_intr.c | 111 + dpdk/drivers/net/tap/tap_log.h | 10 + dpdk/drivers/net/tap/tap_netlink.c | 96 +- dpdk/drivers/net/tap/tap_netlink.h | 59 +- dpdk/drivers/net/tap/tap_rss.h | 40 + dpdk/drivers/net/tap/tap_tcmsgs.c | 71 +- dpdk/drivers/net/tap/tap_tcmsgs.h | 38 +- dpdk/drivers/net/thunderx/Makefile | 34 +- dpdk/drivers/net/thunderx/base/meson.build | 19 + dpdk/drivers/net/thunderx/base/nicvf_bsvf.c | 32 +- dpdk/drivers/net/thunderx/base/nicvf_bsvf.h | 32 +- dpdk/drivers/net/thunderx/base/nicvf_hw.c | 45 +- dpdk/drivers/net/thunderx/base/nicvf_hw.h | 33 +- .../drivers/net/thunderx/base/nicvf_hw_defs.h | 37 +- dpdk/drivers/net/thunderx/base/nicvf_mbox.c | 32 +- dpdk/drivers/net/thunderx/base/nicvf_mbox.h | 32 +- dpdk/drivers/net/thunderx/base/nicvf_plat.h | 32 +- dpdk/drivers/net/thunderx/meson.build | 20 + dpdk/drivers/net/thunderx/nicvf_ethdev.c | 337 +- dpdk/drivers/net/thunderx/nicvf_ethdev.h | 49 +- dpdk/drivers/net/thunderx/nicvf_logs.h | 79 +- dpdk/drivers/net/thunderx/nicvf_rxtx.c | 164 +- dpdk/drivers/net/thunderx/nicvf_rxtx.h | 58 +- dpdk/drivers/net/thunderx/nicvf_struct.h | 65 +- dpdk/drivers/net/thunderx/nicvf_svf.c | 32 +- dpdk/drivers/net/thunderx/nicvf_svf.h | 32 +- ...rsion.map => rte_pmd_thunderx_version.map} | 0 dpdk/drivers/net/vdev_netvsc/Makefile | 32 + dpdk/drivers/net/vdev_netvsc/meson.build | 22 + .../rte_pmd_vdev_netvsc_version.map | 4 + dpdk/drivers/net/vdev_netvsc/vdev_netvsc.c | 835 + dpdk/drivers/net/vhost/Makefile | 32 +- dpdk/drivers/net/vhost/meson.build | 8 + dpdk/drivers/net/vhost/rte_eth_vhost.c | 349 +- dpdk/drivers/net/vhost/rte_eth_vhost.h | 35 +- dpdk/drivers/net/virtio/Makefile | 33 +- dpdk/drivers/net/virtio/meson.build | 27 + dpdk/drivers/net/virtio/virtio_ethdev.c | 528 +- dpdk/drivers/net/virtio/virtio_ethdev.h | 60 +- dpdk/drivers/net/virtio/virtio_logs.h | 52 +- dpdk/drivers/net/virtio/virtio_pci.c | 123 +- dpdk/drivers/net/virtio/virtio_pci.h | 58 +- dpdk/drivers/net/virtio/virtio_ring.h | 33 +- dpdk/drivers/net/virtio/virtio_rxtx.c | 820 +- dpdk/drivers/net/virtio/virtio_rxtx.h | 37 +- dpdk/drivers/net/virtio/virtio_rxtx_simple.c | 132 +- dpdk/drivers/net/virtio/virtio_rxtx_simple.h | 82 +- .../net/virtio/virtio_rxtx_simple_neon.c | 38 +- .../net/virtio/virtio_rxtx_simple_sse.c | 37 +- dpdk/drivers/net/virtio/virtio_user/vhost.h | 37 +- .../net/virtio/virtio_user/vhost_kernel.c | 150 +- .../net/virtio/virtio_user/vhost_kernel_tap.c | 101 +- .../net/virtio/virtio_user/vhost_kernel_tap.h | 36 +- .../net/virtio/virtio_user/vhost_user.c | 254 +- .../net/virtio/virtio_user/virtio_user_dev.c | 293 +- .../net/virtio/virtio_user/virtio_user_dev.h | 46 +- dpdk/drivers/net/virtio/virtio_user_ethdev.c | 204 +- dpdk/drivers/net/virtio/virtqueue.c | 43 +- dpdk/drivers/net/virtio/virtqueue.h | 50 +- dpdk/drivers/net/vmxnet3/Makefile | 36 +- dpdk/drivers/net/vmxnet3/base/upt1_defs.h | 27 +- .../net/vmxnet3/base/vmware_pack_begin.h | 33 +- .../net/vmxnet3/base/vmware_pack_end.h | 33 +- dpdk/drivers/net/vmxnet3/base/vmxnet3_defs.h | 54 +- dpdk/drivers/net/vmxnet3/base/vmxnet3_osdep.h | 33 +- dpdk/drivers/net/vmxnet3/meson.build | 18 + dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.c | 323 +- dpdk/drivers/net/vmxnet3/vmxnet3_ethdev.h | 37 +- dpdk/drivers/net/vmxnet3/vmxnet3_logs.h | 50 +- dpdk/drivers/net/vmxnet3/vmxnet3_ring.h | 35 +- dpdk/drivers/net/vmxnet3/vmxnet3_rxtx.c | 241 +- dpdk/drivers/raw/Makefile | 14 + dpdk/drivers/raw/dpaa2_cmdif/Makefile | 37 + dpdk/drivers/raw/dpaa2_cmdif/dpaa2_cmdif.c | 297 + .../raw/dpaa2_cmdif/dpaa2_cmdif_logs.h | 46 + dpdk/drivers/raw/dpaa2_cmdif/meson.build | 12 + .../raw/dpaa2_cmdif/rte_pmd_dpaa2_cmdif.h | 35 + .../rte_pmd_dpaa2_cmdif_version.map | 4 + dpdk/drivers/raw/dpaa2_qdma/Makefile | 38 + dpdk/drivers/raw/dpaa2_qdma/dpaa2_qdma.c | 1001 + dpdk/drivers/raw/dpaa2_qdma/dpaa2_qdma.h | 152 + dpdk/drivers/raw/dpaa2_qdma/dpaa2_qdma_logs.h | 46 + dpdk/drivers/raw/dpaa2_qdma/meson.build | 12 + .../raw/dpaa2_qdma/rte_pmd_dpaa2_qdma.h | 286 + .../dpaa2_qdma/rte_pmd_dpaa2_qdma_version.map | 20 + dpdk/drivers/raw/ifpga_rawdev/Makefile | 36 + dpdk/drivers/raw/ifpga_rawdev/base/Makefile | 26 + dpdk/drivers/raw/ifpga_rawdev/base/README | 31 + .../drivers/raw/ifpga_rawdev/base/ifpga_api.c | 294 + .../drivers/raw/ifpga_rawdev/base/ifpga_api.h | 28 + .../raw/ifpga_rawdev/base/ifpga_compat.h | 58 + .../raw/ifpga_rawdev/base/ifpga_defines.h | 1663 + .../raw/ifpga_rawdev/base/ifpga_enumerate.c | 821 + .../raw/ifpga_rawdev/base/ifpga_enumerate.h | 11 + .../raw/ifpga_rawdev/base/ifpga_feature_dev.c | 253 + .../raw/ifpga_rawdev/base/ifpga_feature_dev.h | 168 + .../drivers/raw/ifpga_rawdev/base/ifpga_fme.c | 734 + .../raw/ifpga_rawdev/base/ifpga_fme_dperf.c | 301 + .../raw/ifpga_rawdev/base/ifpga_fme_error.c | 381 + .../raw/ifpga_rawdev/base/ifpga_fme_iperf.c | 715 + .../raw/ifpga_rawdev/base/ifpga_fme_pr.c | 352 + dpdk/drivers/raw/ifpga_rawdev/base/ifpga_hw.h | 127 + .../raw/ifpga_rawdev/base/ifpga_port.c | 388 + .../raw/ifpga_rawdev/base/ifpga_port_error.c | 144 + .../drivers/raw/ifpga_rawdev/base/meson.build | 34 + .../raw/ifpga_rawdev/base/opae_debug.c | 99 + .../raw/ifpga_rawdev/base/opae_debug.h | 19 + .../raw/ifpga_rawdev/base/opae_hw_api.c | 381 + .../raw/ifpga_rawdev/base/opae_hw_api.h | 253 + .../raw/ifpga_rawdev/base/opae_ifpga_hw_api.c | 145 + .../raw/ifpga_rawdev/base/opae_ifpga_hw_api.h | 279 + .../raw/ifpga_rawdev/base/opae_osdep.h | 79 + .../base/osdep_raw/osdep_generic.h | 75 + .../base/osdep_rte/osdep_generic.h | 45 + dpdk/drivers/raw/ifpga_rawdev/ifpga_rawdev.c | 618 + dpdk/drivers/raw/ifpga_rawdev/ifpga_rawdev.h | 37 + dpdk/drivers/raw/ifpga_rawdev/meson.build | 15 + .../rte_pmd_ifpga_rawdev_version.map | 4 + dpdk/drivers/raw/meson.build | 7 + dpdk/drivers/raw/skeleton_rawdev/Makefile | 28 + dpdk/drivers/raw/skeleton_rawdev/meson.build | 6 + .../rte_pmd_skeleton_rawdev_version.map | 4 + .../raw/skeleton_rawdev/skeleton_rawdev.c | 765 + .../raw/skeleton_rawdev/skeleton_rawdev.h | 136 + .../skeleton_rawdev/skeleton_rawdev_test.c | 463 + dpdk/examples/Makefile | 40 +- dpdk/examples/bbdev_app/Makefile | 59 + dpdk/examples/bbdev_app/main.c | 1155 + dpdk/examples/bbdev_app/meson.build | 13 + dpdk/examples/bond/Makefile | 83 +- dpdk/examples/bond/main.c | 103 +- dpdk/examples/bond/main.h | 33 +- dpdk/examples/bond/meson.build | 13 + dpdk/examples/cmdline/Makefile | 73 +- dpdk/examples/cmdline/commands.c | 58 +- dpdk/examples/cmdline/commands.h | 33 +- dpdk/examples/cmdline/main.c | 58 +- dpdk/examples/cmdline/meson.build | 11 + dpdk/examples/cmdline/parse_obj_list.c | 58 +- dpdk/examples/cmdline/parse_obj_list.h | 58 +- dpdk/examples/distributor/Makefile | 78 +- dpdk/examples/distributor/main.c | 87 +- dpdk/examples/distributor/meson.build | 12 + dpdk/examples/ethtool/Makefile | 34 +- dpdk/examples/ethtool/ethtool-app/Makefile | 34 +- dpdk/examples/ethtool/ethtool-app/ethapp.c | 107 +- dpdk/examples/ethtool/ethtool-app/ethapp.h | 33 +- dpdk/examples/ethtool/ethtool-app/main.c | 44 +- dpdk/examples/ethtool/lib/Makefile | 35 +- dpdk/examples/ethtool/lib/rte_ethtool.c | 79 +- dpdk/examples/ethtool/lib/rte_ethtool.h | 67 +- dpdk/examples/ethtool/meson.build | 10 + dpdk/examples/eventdev_pipeline/Makefile | 61 + dpdk/examples/eventdev_pipeline/main.c | 580 + dpdk/examples/eventdev_pipeline/meson.build | 15 + .../eventdev_pipeline/pipeline_common.h | 144 + .../pipeline_worker_generic.c | 434 + .../eventdev_pipeline/pipeline_worker_tx.c | 854 + .../eventdev_pipeline_sw_pmd/Makefile | 49 - dpdk/examples/eventdev_pipeline_sw_pmd/main.c | 1023 - dpdk/examples/exception_path/Makefile | 78 +- dpdk/examples/exception_path/main.c | 76 +- dpdk/examples/exception_path/meson.build | 11 + dpdk/examples/fips_validation/Makefile | 75 + .../fips_validation/fips_validation.c | 595 + .../fips_validation/fips_validation.h | 233 + .../fips_validation/fips_validation_aes.c | 188 + .../fips_validation/fips_validation_ccm.c | 272 + .../fips_validation/fips_validation_cmac.c | 116 + .../fips_validation/fips_validation_gcm.c | 125 + .../fips_validation/fips_validation_hmac.c | 105 + .../fips_validation/fips_validation_tdes.c | 264 + dpdk/examples/fips_validation/main.c | 1225 + dpdk/examples/fips_validation/meson.build | 20 + dpdk/examples/flow_classify/Makefile | 82 +- dpdk/examples/flow_classify/flow_classify.c | 112 +- dpdk/examples/flow_classify/meson.build | 13 + dpdk/examples/flow_filtering/Makefile | 75 +- dpdk/examples/flow_filtering/flow_blocks.c | 57 +- dpdk/examples/flow_filtering/main.c | 107 +- dpdk/examples/flow_filtering/meson.build | 11 + dpdk/examples/helloworld/Makefile | 78 +- dpdk/examples/helloworld/main.c | 33 +- dpdk/examples/helloworld/meson.build | 11 + dpdk/examples/ip_fragmentation/Makefile | 78 +- dpdk/examples/ip_fragmentation/main.c | 79 +- dpdk/examples/ip_fragmentation/meson.build | 12 + dpdk/examples/ip_pipeline/Makefile | 136 +- dpdk/examples/ip_pipeline/action.c | 391 + dpdk/examples/ip_pipeline/action.h | 78 + dpdk/examples/ip_pipeline/app.h | 1430 - dpdk/examples/ip_pipeline/cli.c | 6871 ++++ dpdk/examples/ip_pipeline/cli.h | 18 + dpdk/examples/ip_pipeline/common.h | 12 + dpdk/examples/ip_pipeline/config/action.cfg | 68 - dpdk/examples/ip_pipeline/config/action.sh | 119 - dpdk/examples/ip_pipeline/config/action.txt | 8 - .../ip_pipeline/config/diagram-generator.py | 346 - .../config/edge_router_downstream.cfg | 97 - .../config/edge_router_downstream.sh | 13 - .../config/edge_router_upstream.cfg | 124 - .../config/edge_router_upstream.sh | 33 - dpdk/examples/ip_pipeline/config/firewall.cfg | 68 - dpdk/examples/ip_pipeline/config/firewall.sh | 13 - dpdk/examples/ip_pipeline/config/firewall.txt | 9 - dpdk/examples/ip_pipeline/config/flow.cfg | 72 - dpdk/examples/ip_pipeline/config/flow.sh | 25 - dpdk/examples/ip_pipeline/config/flow.txt | 17 - .../ip_pipeline/config/ip_pipeline.cfg | 9 - .../ip_pipeline/config/ip_pipeline.sh | 5 - dpdk/examples/ip_pipeline/config/kni.cfg | 67 - dpdk/examples/ip_pipeline/config/l2fwd.cfg | 58 - dpdk/examples/ip_pipeline/config/l3fwd.cfg | 68 - dpdk/examples/ip_pipeline/config/l3fwd.sh | 33 - .../examples/ip_pipeline/config/l3fwd_arp.cfg | 70 - dpdk/examples/ip_pipeline/config/l3fwd_arp.sh | 43 - .../ip_pipeline/config/network_layers.cfg | 227 - .../ip_pipeline/config/network_layers.sh | 79 - .../config/pipeline-to-core-mapping.py | 935 - dpdk/examples/ip_pipeline/config/tap.cfg | 64 - .../ip_pipeline/config/tm_profile.cfg | 105 - dpdk/examples/ip_pipeline/config_check.c | 517 - dpdk/examples/ip_pipeline/config_parse.c | 3431 -- dpdk/examples/ip_pipeline/config_parse_tm.c | 448 - dpdk/examples/ip_pipeline/conn.c | 328 + dpdk/examples/ip_pipeline/conn.h | 47 + dpdk/examples/ip_pipeline/cpu_core_map.c | 500 - dpdk/examples/ip_pipeline/cpu_core_map.h | 69 - dpdk/examples/ip_pipeline/cryptodev.c | 117 + dpdk/examples/ip_pipeline/cryptodev.h | 43 + .../ip_pipeline/examples/firewall.cli | 59 + dpdk/examples/ip_pipeline/examples/flow.cli | 60 + .../ip_pipeline/examples/flow_crypto.cli | 58 + dpdk/examples/ip_pipeline/examples/kni.cli | 69 + dpdk/examples/ip_pipeline/examples/l2fwd.cli | 51 + dpdk/examples/ip_pipeline/examples/route.cli | 60 + .../ip_pipeline/examples/route_ecmp.cli | 57 + dpdk/examples/ip_pipeline/examples/rss.cli | 112 + dpdk/examples/ip_pipeline/examples/tap.cli | 66 + dpdk/examples/ip_pipeline/init.c | 1949 - dpdk/examples/ip_pipeline/kni.c | 175 + dpdk/examples/ip_pipeline/kni.h | 46 + dpdk/examples/ip_pipeline/link.c | 266 + dpdk/examples/ip_pipeline/link.h | 66 + dpdk/examples/ip_pipeline/main.c | 299 +- dpdk/examples/ip_pipeline/mempool.c | 82 + dpdk/examples/ip_pipeline/mempool.h | 40 + dpdk/examples/ip_pipeline/meson.build | 26 + dpdk/examples/ip_pipeline/parser.c | 77 +- dpdk/examples/ip_pipeline/parser.h | 41 +- dpdk/examples/ip_pipeline/pipeline.c | 1135 + dpdk/examples/ip_pipeline/pipeline.h | 489 +- .../examples/ip_pipeline/pipeline/hash_func.h | 385 - .../ip_pipeline/pipeline/hash_func_arm64.h | 261 - .../pipeline/pipeline_actions_common.h | 231 - .../ip_pipeline/pipeline/pipeline_common_be.c | 205 - .../ip_pipeline/pipeline/pipeline_common_be.h | 163 - .../ip_pipeline/pipeline/pipeline_common_fe.c | 1484 - .../ip_pipeline/pipeline/pipeline_common_fe.h | 260 - .../ip_pipeline/pipeline/pipeline_firewall.c | 1450 - .../ip_pipeline/pipeline/pipeline_firewall.h | 89 - .../pipeline/pipeline_firewall_be.c | 885 - .../pipeline/pipeline_firewall_be.h | 176 - .../pipeline/pipeline_flow_actions.c | 1315 - .../pipeline/pipeline_flow_actions.h | 89 - .../pipeline/pipeline_flow_actions_be.c | 989 - .../pipeline/pipeline_flow_actions_be.h | 168 - .../pipeline/pipeline_flow_classification.c | 1907 - .../pipeline/pipeline_flow_classification.h | 135 - .../pipeline_flow_classification_be.c | 752 - .../pipeline_flow_classification_be.h | 142 - .../ip_pipeline/pipeline/pipeline_master.c | 49 - .../ip_pipeline/pipeline/pipeline_master.h | 41 - .../ip_pipeline/pipeline/pipeline_master_be.c | 170 - .../ip_pipeline/pipeline/pipeline_master_be.h | 41 - .../pipeline/pipeline_passthrough.c | 74 - .../pipeline/pipeline_passthrough.h | 41 - .../pipeline/pipeline_passthrough_be.c | 958 - .../pipeline/pipeline_passthrough_be.h | 73 - .../ip_pipeline/pipeline/pipeline_routing.c | 1642 - .../ip_pipeline/pipeline/pipeline_routing.h | 100 - .../pipeline/pipeline_routing_be.c | 1995 - .../pipeline/pipeline_routing_be.h | 312 - dpdk/examples/ip_pipeline/pipeline_be.h | 351 - dpdk/examples/ip_pipeline/swq.c | 76 + dpdk/examples/ip_pipeline/swq.h | 37 + dpdk/examples/ip_pipeline/tap.c | 102 + dpdk/examples/ip_pipeline/tap.h | 29 + dpdk/examples/ip_pipeline/thread.c | 3358 +- dpdk/examples/ip_pipeline/thread.h | 106 +- dpdk/examples/ip_pipeline/thread_fe.c | 457 - dpdk/examples/ip_pipeline/thread_fe.h | 101 - dpdk/examples/ip_pipeline/tmgr.c | 229 + dpdk/examples/ip_pipeline/tmgr.h | 70 + dpdk/examples/ip_reassembly/Makefile | 78 +- dpdk/examples/ip_reassembly/main.c | 89 +- dpdk/examples/ip_reassembly/meson.build | 12 + dpdk/examples/ipsec-secgw/Makefile | 95 +- dpdk/examples/ipsec-secgw/esp.c | 42 +- dpdk/examples/ipsec-secgw/esp.h | 33 +- dpdk/examples/ipsec-secgw/ipip.h | 59 +- dpdk/examples/ipsec-secgw/ipsec-secgw.c | 414 +- dpdk/examples/ipsec-secgw/ipsec.c | 252 +- dpdk/examples/ipsec-secgw/ipsec.h | 44 +- dpdk/examples/ipsec-secgw/meson.build | 14 + dpdk/examples/ipsec-secgw/parser.c | 56 +- dpdk/examples/ipsec-secgw/parser.h | 32 +- dpdk/examples/ipsec-secgw/rt.c | 33 +- dpdk/examples/ipsec-secgw/sa.c | 56 +- dpdk/examples/ipsec-secgw/sp4.c | 33 +- dpdk/examples/ipsec-secgw/sp6.c | 33 +- dpdk/examples/ipv4_multicast/Makefile | 78 +- dpdk/examples/ipv4_multicast/main.c | 75 +- dpdk/examples/ipv4_multicast/meson.build | 12 + dpdk/examples/kni/Makefile | 81 +- dpdk/examples/kni/main.c | 249 +- dpdk/examples/kni/meson.build | 15 + dpdk/examples/l2fwd-cat/Makefile | 80 +- dpdk/examples/l2fwd-cat/cat.c | 33 +- dpdk/examples/l2fwd-cat/cat.h | 33 +- dpdk/examples/l2fwd-cat/l2fwd-cat.c | 44 +- dpdk/examples/l2fwd-cat/meson.build | 15 + dpdk/examples/l2fwd-crypto/Makefile | 83 +- dpdk/examples/l2fwd-crypto/main.c | 654 +- dpdk/examples/l2fwd-crypto/meson.build | 12 + dpdk/examples/l2fwd-jobstats/Makefile | 77 +- dpdk/examples/l2fwd-jobstats/main.c | 87 +- dpdk/examples/l2fwd-jobstats/meson.build | 12 + dpdk/examples/l2fwd-keepalive/Makefile | 79 +- .../l2fwd-keepalive/ka-agent/Makefile | 32 +- dpdk/examples/l2fwd-keepalive/ka-agent/main.c | 33 +- dpdk/examples/l2fwd-keepalive/main.c | 87 +- dpdk/examples/l2fwd-keepalive/meson.build | 13 + dpdk/examples/l2fwd-keepalive/shm.c | 33 +- dpdk/examples/l2fwd-keepalive/shm.h | 33 +- dpdk/examples/l2fwd/Makefile | 77 +- dpdk/examples/l2fwd/main.c | 117 +- dpdk/examples/l2fwd/meson.build | 11 + dpdk/examples/l3fwd-acl/Makefile | 77 +- dpdk/examples/l3fwd-acl/main.c | 102 +- dpdk/examples/l3fwd-acl/meson.build | 12 + dpdk/examples/l3fwd-power/Makefile | 82 +- dpdk/examples/l3fwd-power/main.c | 534 +- dpdk/examples/l3fwd-power/main.h | 20 + dpdk/examples/l3fwd-power/meson.build | 16 + dpdk/examples/l3fwd-power/perf_core.c | 230 + dpdk/examples/l3fwd-power/perf_core.h | 12 + dpdk/examples/l3fwd-vf/Makefile | 77 +- dpdk/examples/l3fwd-vf/main.c | 96 +- dpdk/examples/l3fwd-vf/meson.build | 12 + dpdk/examples/l3fwd/Makefile | 77 +- dpdk/examples/l3fwd/l3fwd.h | 33 +- dpdk/examples/l3fwd/l3fwd_altivec.h | 37 +- dpdk/examples/l3fwd/l3fwd_common.h | 35 +- dpdk/examples/l3fwd/l3fwd_em.c | 33 +- dpdk/examples/l3fwd/l3fwd_em.h | 33 +- dpdk/examples/l3fwd/l3fwd_em_hlm.h | 35 +- dpdk/examples/l3fwd/l3fwd_em_hlm_neon.h | 35 +- dpdk/examples/l3fwd/l3fwd_em_hlm_sse.h | 33 +- dpdk/examples/l3fwd/l3fwd_em_sequential.h | 33 +- dpdk/examples/l3fwd/l3fwd_lpm.c | 33 +- dpdk/examples/l3fwd/l3fwd_lpm.h | 33 +- dpdk/examples/l3fwd/l3fwd_lpm_altivec.h | 36 +- dpdk/examples/l3fwd/l3fwd_lpm_neon.h | 35 +- dpdk/examples/l3fwd/l3fwd_lpm_sse.h | 33 +- dpdk/examples/l3fwd/l3fwd_neon.h | 36 +- dpdk/examples/l3fwd/l3fwd_sse.h | 33 +- dpdk/examples/l3fwd/main.c | 146 +- dpdk/examples/l3fwd/meson.build | 12 + dpdk/examples/link_status_interrupt/Makefile | 77 +- dpdk/examples/link_status_interrupt/main.c | 68 +- .../link_status_interrupt/meson.build | 11 + dpdk/examples/load_balancer/Makefile | 78 +- dpdk/examples/load_balancer/config.c | 37 +- dpdk/examples/load_balancer/init.c | 69 +- dpdk/examples/load_balancer/main.c | 33 +- dpdk/examples/load_balancer/main.h | 33 +- dpdk/examples/load_balancer/meson.build | 12 + dpdk/examples/load_balancer/runtime.c | 33 +- dpdk/examples/meson.build | 64 + dpdk/examples/multi_process/Makefile | 33 +- .../multi_process/client_server_mp/Makefile | 32 +- .../client_server_mp/mp_client/Makefile | 32 +- .../client_server_mp/mp_client/client.c | 35 +- .../client_server_mp/mp_server/Makefile | 35 +- .../client_server_mp/mp_server/args.c | 33 +- .../client_server_mp/mp_server/args.h | 33 +- .../client_server_mp/mp_server/init.c | 39 +- .../client_server_mp/mp_server/init.h | 33 +- .../client_server_mp/mp_server/main.c | 48 +- .../client_server_mp/shared/common.h | 33 +- .../multi_process/hotplug_mp/Makefile | 23 + .../multi_process/hotplug_mp/commands.c | 214 + .../multi_process/hotplug_mp/commands.h | 10 + dpdk/examples/multi_process/hotplug_mp/main.c | 41 + .../multi_process/l2fwd_fork/Makefile | 50 - dpdk/examples/multi_process/l2fwd_fork/flib.c | 309 - dpdk/examples/multi_process/l2fwd_fork/flib.h | 149 - dpdk/examples/multi_process/l2fwd_fork/main.c | 1283 - dpdk/examples/multi_process/meson.build | 10 + .../examples/multi_process/simple_mp/Makefile | 32 +- dpdk/examples/multi_process/simple_mp/main.c | 33 +- .../multi_process/simple_mp/mp_commands.c | 33 +- .../multi_process/simple_mp/mp_commands.h | 33 +- .../multi_process/symmetric_mp/Makefile | 32 +- .../multi_process/symmetric_mp/main.c | 76 +- dpdk/examples/netmap_compat/Makefile | 32 +- dpdk/examples/netmap_compat/bridge/Makefile | 34 +- dpdk/examples/netmap_compat/bridge/bridge.c | 40 +- .../netmap_compat/lib/compat_netmap.c | 48 +- .../netmap_compat/lib/compat_netmap.h | 33 +- dpdk/examples/netmap_compat/meson.build | 10 + dpdk/examples/packet_ordering/Makefile | 77 +- dpdk/examples/packet_ordering/main.c | 75 +- dpdk/examples/packet_ordering/meson.build | 12 + dpdk/examples/performance-thread/Makefile | 32 +- .../common/arch/arm64/ctx.c | 32 +- .../common/arch/arm64/ctx.h | 32 +- .../common/arch/arm64/stack.h | 32 +- .../performance-thread/common/arch/x86/ctx.c | 62 +- .../performance-thread/common/arch/x86/ctx.h | 33 +- .../common/arch/x86/stack.h | 66 +- .../performance-thread/common/common.mk | 33 +- .../performance-thread/common/lthread.c | 61 +- .../performance-thread/common/lthread.h | 62 +- .../performance-thread/common/lthread_api.h | 62 +- .../performance-thread/common/lthread_cond.c | 61 +- .../performance-thread/common/lthread_cond.h | 61 +- .../performance-thread/common/lthread_diag.c | 33 +- .../performance-thread/common/lthread_diag.h | 33 +- .../common/lthread_diag_api.h | 33 +- .../performance-thread/common/lthread_int.h | 61 +- .../performance-thread/common/lthread_mutex.c | 33 +- .../performance-thread/common/lthread_mutex.h | 33 +- .../common/lthread_objcache.h | 33 +- .../performance-thread/common/lthread_pool.h | 68 +- .../performance-thread/common/lthread_queue.h | 68 +- .../performance-thread/common/lthread_sched.c | 62 +- .../performance-thread/common/lthread_sched.h | 61 +- .../performance-thread/common/lthread_timer.h | 33 +- .../performance-thread/common/lthread_tls.c | 35 +- .../performance-thread/common/lthread_tls.h | 33 +- .../performance-thread/l3fwd-thread/Makefile | 32 +- .../performance-thread/l3fwd-thread/main.c | 103 +- dpdk/examples/performance-thread/meson.build | 10 + .../performance-thread/pthread_shim/Makefile | 32 +- .../performance-thread/pthread_shim/main.c | 35 +- .../pthread_shim/pthread_shim.c | 34 +- .../pthread_shim/pthread_shim.h | 33 +- dpdk/examples/ptpclient/Makefile | 77 +- dpdk/examples/ptpclient/meson.build | 11 + dpdk/examples/ptpclient/ptpclient.c | 58 +- dpdk/examples/qos_meter/Makefile | 79 +- dpdk/examples/qos_meter/main.c | 132 +- dpdk/examples/qos_meter/main.h | 65 +- dpdk/examples/qos_meter/meson.build | 12 + dpdk/examples/qos_meter/rte_policer.c | 33 +- dpdk/examples/qos_meter/rte_policer.h | 33 +- dpdk/examples/qos_sched/Makefile | 81 +- dpdk/examples/qos_sched/app_thread.c | 33 +- dpdk/examples/qos_sched/args.c | 33 +- dpdk/examples/qos_sched/cfg_file.c | 33 +- dpdk/examples/qos_sched/cfg_file.h | 33 +- dpdk/examples/qos_sched/cmdline.c | 33 +- dpdk/examples/qos_sched/init.c | 53 +- dpdk/examples/qos_sched/main.c | 33 +- dpdk/examples/qos_sched/main.h | 37 +- dpdk/examples/qos_sched/meson.build | 13 + dpdk/examples/qos_sched/stats.c | 35 +- dpdk/examples/quota_watermark/Makefile | 32 +- dpdk/examples/quota_watermark/include/conf.h | 37 +- dpdk/examples/quota_watermark/meson.build | 10 + dpdk/examples/quota_watermark/qw/Makefile | 32 +- dpdk/examples/quota_watermark/qw/args.c | 33 +- dpdk/examples/quota_watermark/qw/args.h | 33 +- dpdk/examples/quota_watermark/qw/init.c | 60 +- dpdk/examples/quota_watermark/qw/init.h | 33 +- dpdk/examples/quota_watermark/qw/main.c | 33 +- dpdk/examples/quota_watermark/qw/main.h | 33 +- dpdk/examples/quota_watermark/qwctl/Makefile | 32 +- .../examples/quota_watermark/qwctl/commands.c | 33 +- .../examples/quota_watermark/qwctl/commands.h | 33 +- dpdk/examples/quota_watermark/qwctl/qwctl.c | 33 +- dpdk/examples/quota_watermark/qwctl/qwctl.h | 33 +- dpdk/examples/rxtx_callbacks/Makefile | 77 +- dpdk/examples/rxtx_callbacks/main.c | 65 +- dpdk/examples/rxtx_callbacks/meson.build | 11 + dpdk/examples/server_node_efd/Makefile | 32 +- dpdk/examples/server_node_efd/meson.build | 10 + dpdk/examples/server_node_efd/node/Makefile | 32 +- dpdk/examples/server_node_efd/node/node.c | 35 +- dpdk/examples/server_node_efd/server/Makefile | 34 +- dpdk/examples/server_node_efd/server/args.c | 33 +- dpdk/examples/server_node_efd/server/args.h | 33 +- dpdk/examples/server_node_efd/server/init.c | 52 +- dpdk/examples/server_node_efd/server/init.h | 33 +- dpdk/examples/server_node_efd/server/main.c | 33 +- dpdk/examples/server_node_efd/shared/common.h | 33 +- dpdk/examples/service_cores/Makefile | 77 +- dpdk/examples/service_cores/main.c | 38 +- dpdk/examples/service_cores/meson.build | 11 + dpdk/examples/skeleton/Makefile | 77 +- dpdk/examples/skeleton/basicfwd.c | 63 +- dpdk/examples/skeleton/meson.build | 11 + dpdk/examples/tep_termination/Makefile | 85 +- dpdk/examples/tep_termination/main.c | 60 +- dpdk/examples/tep_termination/main.h | 33 +- dpdk/examples/tep_termination/meson.build | 16 + dpdk/examples/tep_termination/vxlan.c | 34 +- dpdk/examples/tep_termination/vxlan.h | 33 +- dpdk/examples/tep_termination/vxlan_setup.c | 61 +- dpdk/examples/tep_termination/vxlan_setup.h | 33 +- dpdk/examples/timer/Makefile | 77 +- dpdk/examples/timer/main.c | 33 +- dpdk/examples/timer/meson.build | 12 + dpdk/examples/vdpa/Makefile | 32 + dpdk/examples/vdpa/main.c | 454 + dpdk/examples/vdpa/meson.build | 16 + dpdk/examples/vhost/Makefile | 85 +- dpdk/examples/vhost/main.c | 95 +- dpdk/examples/vhost/main.h | 33 +- dpdk/examples/vhost/meson.build | 16 + dpdk/examples/vhost/virtio_net.c | 46 +- dpdk/examples/vhost_crypto/Makefile | 31 + dpdk/examples/vhost_crypto/main.c | 613 + dpdk/examples/vhost_crypto/meson.build | 14 + dpdk/examples/vhost_scsi/Makefile | 86 +- dpdk/examples/vhost_scsi/meson.build | 16 + dpdk/examples/vhost_scsi/scsi.c | 47 +- dpdk/examples/vhost_scsi/scsi_spec.h | 35 +- dpdk/examples/vhost_scsi/vhost_scsi.c | 40 +- dpdk/examples/vhost_scsi/vhost_scsi.h | 33 +- dpdk/examples/vm_power_manager/Makefile | 45 +- .../vm_power_manager/channel_manager.c | 215 +- .../vm_power_manager/channel_manager.h | 54 +- .../vm_power_manager/channel_monitor.c | 559 +- .../vm_power_manager/channel_monitor.h | 33 +- .../vm_power_manager/guest_cli/Makefile | 34 +- .../vm_power_manager/guest_cli/main.c | 184 +- .../vm_power_manager/guest_cli/meson.build | 21 + .../vm_power_manager/guest_cli/parse.c | 82 + .../vm_power_manager/guest_cli/parse.h | 19 + .../guest_cli/vm_power_cli_guest.c | 147 +- .../guest_cli/vm_power_cli_guest.h | 39 +- dpdk/examples/vm_power_manager/main.c | 224 +- dpdk/examples/vm_power_manager/meson.build | 43 + dpdk/examples/vm_power_manager/oob_monitor.h | 68 + .../vm_power_manager/oob_monitor_nop.c | 38 + .../vm_power_manager/oob_monitor_x86.c | 258 + dpdk/examples/vm_power_manager/parse.c | 81 + dpdk/examples/vm_power_manager/parse.h | 20 + .../examples/vm_power_manager/power_manager.c | 184 +- .../examples/vm_power_manager/power_manager.h | 56 +- dpdk/examples/vm_power_manager/vm_power_cli.c | 33 +- dpdk/examples/vm_power_manager/vm_power_cli.h | 33 +- dpdk/examples/vmdq/Makefile | 77 +- dpdk/examples/vmdq/main.c | 64 +- dpdk/examples/vmdq/meson.build | 11 + dpdk/examples/vmdq_dcb/Makefile | 77 +- dpdk/examples/vmdq_dcb/main.c | 75 +- dpdk/examples/vmdq_dcb/meson.build | 11 + dpdk/kernel/Makefile | 9 + dpdk/kernel/freebsd/BSDmakefile.meson | 18 + dpdk/kernel/freebsd/Makefile | 9 + dpdk/kernel/freebsd/contigmem/BSDmakefile | 8 + dpdk/kernel/freebsd/contigmem/Makefile | 24 + dpdk/kernel/freebsd/contigmem/contigmem.c | 353 + dpdk/kernel/freebsd/contigmem/meson.build | 4 + dpdk/kernel/freebsd/meson.build | 32 + dpdk/kernel/freebsd/nic_uio/BSDmakefile | 8 + dpdk/kernel/freebsd/nic_uio/Makefile | 24 + dpdk/kernel/freebsd/nic_uio/meson.build | 4 + dpdk/kernel/freebsd/nic_uio/nic_uio.c | 350 + dpdk/kernel/linux/Makefile | 9 + dpdk/kernel/linux/igb_uio/Kbuild | 2 + dpdk/kernel/linux/igb_uio/Makefile | 25 + dpdk/kernel/linux/igb_uio/compat.h | 154 + dpdk/kernel/linux/igb_uio/igb_uio.c | 660 + dpdk/kernel/linux/igb_uio/meson.build | 20 + dpdk/kernel/linux/kni/Kbuild | 8 + dpdk/kernel/linux/kni/Makefile | 58 + dpdk/kernel/linux/kni/compat.h | 112 + dpdk/kernel/linux/kni/ethtool/README | 71 + .../linux/kni/ethtool/igb/e1000_82575.c | 3650 ++ .../linux/kni/ethtool/igb/e1000_82575.h | 494 + dpdk/kernel/linux/kni/ethtool/igb/e1000_api.c | 1144 + dpdk/kernel/linux/kni/ethtool/igb/e1000_api.h | 142 + .../linux/kni/ethtool/igb/e1000_defines.h | 1365 + dpdk/kernel/linux/kni/ethtool/igb/e1000_hw.h | 778 + .../kernel/linux/kni/ethtool/igb/e1000_i210.c | 894 + .../kernel/linux/kni/ethtool/igb/e1000_i210.h | 76 + dpdk/kernel/linux/kni/ethtool/igb/e1000_mac.c | 2081 + dpdk/kernel/linux/kni/ethtool/igb/e1000_mac.h | 65 + .../linux/kni/ethtool/igb/e1000_manage.c | 539 + .../linux/kni/ethtool/igb/e1000_manage.h | 74 + dpdk/kernel/linux/kni/ethtool/igb/e1000_mbx.c | 510 + dpdk/kernel/linux/kni/ethtool/igb/e1000_mbx.h | 72 + dpdk/kernel/linux/kni/ethtool/igb/e1000_nvm.c | 950 + dpdk/kernel/linux/kni/ethtool/igb/e1000_nvm.h | 60 + .../linux/kni/ethtool/igb/e1000_osdep.h | 121 + dpdk/kernel/linux/kni/ethtool/igb/e1000_phy.c | 3392 ++ dpdk/kernel/linux/kni/ethtool/igb/e1000_phy.h | 241 + .../kernel/linux/kni/ethtool/igb/e1000_regs.h | 631 + dpdk/kernel/linux/kni/ethtool/igb/igb.h | 844 + .../linux/kni/ethtool/igb/igb_ethtool.c | 2851 ++ dpdk/kernel/linux/kni/ethtool/igb/igb_main.c | 10344 +++++ dpdk/kernel/linux/kni/ethtool/igb/igb_param.c | 832 + .../linux/kni/ethtool/igb/igb_regtest.h | 234 + dpdk/kernel/linux/kni/ethtool/igb/igb_vmdq.c | 421 + dpdk/kernel/linux/kni/ethtool/igb/igb_vmdq.h | 31 + dpdk/kernel/linux/kni/ethtool/igb/kcompat.h | 3945 ++ dpdk/kernel/linux/kni/ethtool/igb/meson.build | 16 + dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe.h | 912 + .../linux/kni/ethtool/ixgbe/ixgbe_82598.c | 1281 + .../linux/kni/ethtool/ixgbe/ixgbe_82598.h | 29 + .../linux/kni/ethtool/ixgbe/ixgbe_82599.c | 2299 ++ .../linux/kni/ethtool/ixgbe/ixgbe_82599.h | 43 + .../linux/kni/ethtool/ixgbe/ixgbe_api.c | 1142 + .../linux/kni/ethtool/ixgbe/ixgbe_api.h | 153 + .../linux/kni/ethtool/ixgbe/ixgbe_common.c | 4067 ++ .../linux/kni/ethtool/ixgbe/ixgbe_common.h | 125 + .../linux/kni/ethtool/ixgbe/ixgbe_dcb.h | 153 + .../linux/kni/ethtool/ixgbe/ixgbe_ethtool.c | 2894 ++ .../linux/kni/ethtool/ixgbe/ixgbe_fcoe.h | 76 + .../linux/kni/ethtool/ixgbe/ixgbe_main.c | 2951 ++ .../linux/kni/ethtool/ixgbe/ixgbe_mbx.h | 90 + .../linux/kni/ethtool/ixgbe/ixgbe_osdep.h | 117 + .../linux/kni/ethtool/ixgbe/ixgbe_phy.c | 1832 + .../linux/kni/ethtool/ixgbe/ixgbe_phy.h | 122 + .../linux/kni/ethtool/ixgbe/ixgbe_type.h | 3239 ++ .../linux/kni/ethtool/ixgbe/ixgbe_x540.c | 922 + .../linux/kni/ethtool/ixgbe/ixgbe_x540.h | 43 + dpdk/kernel/linux/kni/ethtool/ixgbe/kcompat.c | 1231 + dpdk/kernel/linux/kni/ethtool/ixgbe/kcompat.h | 3140 ++ .../linux/kni/ethtool/ixgbe/meson.build | 13 + dpdk/kernel/linux/kni/ethtool/meson.build | 5 + dpdk/kernel/linux/kni/kni_dev.h | 110 + dpdk/kernel/linux/kni/kni_ethtool.c | 229 + dpdk/kernel/linux/kni/kni_fifo.h | 87 + dpdk/kernel/linux/kni/kni_misc.c | 707 + dpdk/kernel/linux/kni/kni_net.c | 802 + dpdk/kernel/linux/kni/meson.build | 34 + dpdk/kernel/linux/meson.build | 42 + dpdk/kernel/meson.build | 4 + dpdk/lib/Makefile | 86 +- dpdk/lib/librte_acl/Makefile | 32 +- dpdk/lib/librte_acl/acl.h | 33 +- dpdk/lib/librte_acl/acl_bld.c | 33 +- dpdk/lib/librte_acl/acl_gen.c | 35 +- dpdk/lib/librte_acl/acl_run.h | 33 +- dpdk/lib/librte_acl/acl_run_avx2.c | 33 +- dpdk/lib/librte_acl/acl_run_avx2.h | 33 +- dpdk/lib/librte_acl/acl_run_neon.c | 34 +- dpdk/lib/librte_acl/acl_run_neon.h | 34 +- dpdk/lib/librte_acl/acl_run_scalar.c | 33 +- dpdk/lib/librte_acl/acl_run_sse.c | 33 +- dpdk/lib/librte_acl/acl_run_sse.h | 33 +- dpdk/lib/librte_acl/acl_vect.h | 33 +- dpdk/lib/librte_acl/meson.build | 31 + dpdk/lib/librte_acl/rte_acl.c | 41 +- dpdk/lib/librte_acl/rte_acl.h | 35 +- dpdk/lib/librte_acl/rte_acl_osdep.h | 33 +- dpdk/lib/librte_acl/tb_mem.c | 33 +- dpdk/lib/librte_acl/tb_mem.h | 33 +- dpdk/lib/librte_bbdev/Makefile | 29 + dpdk/lib/librte_bbdev/meson.build | 9 + dpdk/lib/librte_bbdev/rte_bbdev.c | 1133 + dpdk/lib/librte_bbdev/rte_bbdev.h | 700 + dpdk/lib/librte_bbdev/rte_bbdev_op.h | 598 + dpdk/lib/librte_bbdev/rte_bbdev_pmd.h | 198 + dpdk/lib/librte_bbdev/rte_bbdev_version.map | 37 + dpdk/lib/librte_bitratestats/Makefile | 32 +- dpdk/lib/librte_bitratestats/meson.build | 7 + dpdk/lib/librte_bitratestats/rte_bitrate.c | 33 +- dpdk/lib/librte_bitratestats/rte_bitrate.h | 33 +- dpdk/lib/librte_bpf/Makefile | 41 + dpdk/lib/librte_bpf/bpf.c | 61 + dpdk/lib/librte_bpf/bpf_def.h | 143 + dpdk/lib/librte_bpf/bpf_exec.c | 453 + dpdk/lib/librte_bpf/bpf_impl.h | 55 + dpdk/lib/librte_bpf/bpf_jit_x86.c | 1368 + dpdk/lib/librte_bpf/bpf_load.c | 148 + dpdk/lib/librte_bpf/bpf_load_elf.c | 322 + dpdk/lib/librte_bpf/bpf_pkt.c | 605 + dpdk/lib/librte_bpf/bpf_validate.c | 2248 ++ dpdk/lib/librte_bpf/meson.build | 25 + dpdk/lib/librte_bpf/rte_bpf.h | 203 + dpdk/lib/librte_bpf/rte_bpf_ethdev.h | 117 + dpdk/lib/librte_bpf/rte_bpf_version.map | 16 + dpdk/lib/librte_cfgfile/Makefile | 32 +- dpdk/lib/librte_cfgfile/meson.build | 6 + dpdk/lib/librte_cfgfile/rte_cfgfile.c | 33 +- dpdk/lib/librte_cfgfile/rte_cfgfile.h | 33 +- .../librte_cfgfile/rte_cfgfile_version.map | 4 +- dpdk/lib/librte_cmdline/Makefile | 33 +- dpdk/lib/librte_cmdline/cmdline.c | 82 +- dpdk/lib/librte_cmdline/cmdline.h | 58 +- dpdk/lib/librte_cmdline/cmdline_cirbuf.c | 58 +- dpdk/lib/librte_cmdline/cmdline_cirbuf.h | 58 +- dpdk/lib/librte_cmdline/cmdline_parse.c | 71 +- dpdk/lib/librte_cmdline/cmdline_parse.h | 58 +- .../librte_cmdline/cmdline_parse_etheraddr.c | 60 +- .../librte_cmdline/cmdline_parse_etheraddr.h | 58 +- .../lib/librte_cmdline/cmdline_parse_ipaddr.c | 281 +- .../lib/librte_cmdline/cmdline_parse_ipaddr.h | 58 +- dpdk/lib/librte_cmdline/cmdline_parse_num.c | 58 +- dpdk/lib/librte_cmdline/cmdline_parse_num.h | 58 +- .../librte_cmdline/cmdline_parse_portlist.c | 60 +- .../librte_cmdline/cmdline_parse_portlist.h | 58 +- .../lib/librte_cmdline/cmdline_parse_string.c | 62 +- .../lib/librte_cmdline/cmdline_parse_string.h | 58 +- dpdk/lib/librte_cmdline/cmdline_rdline.c | 59 +- dpdk/lib/librte_cmdline/cmdline_rdline.h | 58 +- dpdk/lib/librte_cmdline/cmdline_socket.c | 58 +- dpdk/lib/librte_cmdline/cmdline_socket.h | 58 +- dpdk/lib/librte_cmdline/cmdline_vt100.c | 59 +- dpdk/lib/librte_cmdline/cmdline_vt100.h | 59 +- dpdk/lib/librte_cmdline/meson.build | 31 + dpdk/lib/librte_compat/Makefile | 33 +- dpdk/lib/librte_compat/meson.build | 8 + dpdk/lib/librte_compat/rte_compat.h | 43 +- dpdk/lib/librte_compressdev/Makefile | 31 + dpdk/lib/librte_compressdev/meson.build | 12 + dpdk/lib/librte_compressdev/rte_comp.c | 215 + dpdk/lib/librte_compressdev/rte_comp.h | 485 + dpdk/lib/librte_compressdev/rte_compressdev.c | 768 + dpdk/lib/librte_compressdev/rte_compressdev.h | 547 + .../rte_compressdev_internal.h | 114 + .../librte_compressdev/rte_compressdev_pmd.c | 155 + .../librte_compressdev/rte_compressdev_pmd.h | 385 + .../rte_compressdev_version.map | 39 + dpdk/lib/librte_cryptodev/Makefile | 34 +- dpdk/lib/librte_cryptodev/meson.build | 11 + dpdk/lib/librte_cryptodev/rte_crypto.h | 114 +- dpdk/lib/librte_cryptodev/rte_crypto_asym.h | 496 + dpdk/lib/librte_cryptodev/rte_crypto_sym.h | 49 +- dpdk/lib/librte_cryptodev/rte_cryptodev.c | 472 +- dpdk/lib/librte_cryptodev/rte_cryptodev.h | 416 +- dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.c | 65 +- dpdk/lib/librte_cryptodev/rte_cryptodev_pmd.h | 160 +- .../rte_cryptodev_version.map | 33 +- dpdk/lib/librte_distributor/Makefile | 32 +- dpdk/lib/librte_distributor/meson.build | 11 + dpdk/lib/librte_distributor/rte_distributor.c | 34 +- dpdk/lib/librte_distributor/rte_distributor.h | 32 +- .../rte_distributor_match_generic.c | 32 +- .../rte_distributor_match_sse.c | 32 +- .../rte_distributor_private.h | 32 +- .../rte_distributor_v1705.h | 32 +- .../librte_distributor/rte_distributor_v20.c | 33 +- .../librte_distributor/rte_distributor_v20.h | 33 +- dpdk/lib/librte_eal/Makefile | 32 +- dpdk/lib/librte_eal/bsdapp/BSDmakefile.meson | 43 + dpdk/lib/librte_eal/bsdapp/Makefile | 34 +- .../librte_eal/bsdapp/contigmem/BSDmakefile | 36 - dpdk/lib/librte_eal/bsdapp/contigmem/Makefile | 52 - .../librte_eal/bsdapp/contigmem/contigmem.c | 381 - dpdk/lib/librte_eal/bsdapp/eal/Makefile | 54 +- dpdk/lib/librte_eal/bsdapp/eal/eal.c | 381 +- dpdk/lib/librte_eal/bsdapp/eal/eal_alarm.c | 334 +- .../librte_eal/bsdapp/eal/eal_alarm_private.h | 19 + dpdk/lib/librte_eal/bsdapp/eal/eal_cpuflags.c | 21 + dpdk/lib/librte_eal/bsdapp/eal/eal_debug.c | 37 +- dpdk/lib/librte_eal/bsdapp/eal/eal_dev.c | 35 + .../librte_eal/bsdapp/eal/eal_hugepage_info.c | 102 +- .../librte_eal/bsdapp/eal/eal_interrupts.c | 495 +- dpdk/lib/librte_eal/bsdapp/eal/eal_lcore.c | 33 +- dpdk/lib/librte_eal/bsdapp/eal/eal_memalloc.c | 75 + dpdk/lib/librte_eal/bsdapp/eal/eal_memory.c | 509 +- dpdk/lib/librte_eal/bsdapp/eal/eal_thread.c | 35 +- dpdk/lib/librte_eal/bsdapp/eal/eal_timer.c | 33 +- dpdk/lib/librte_eal/bsdapp/eal/meson.build | 20 + .../lib/librte_eal/bsdapp/nic_uio/BSDmakefile | 36 - dpdk/lib/librte_eal/bsdapp/nic_uio/Makefile | 52 - dpdk/lib/librte_eal/bsdapp/nic_uio/nic_uio.c | 379 - dpdk/lib/librte_eal/common/Makefile | 38 +- .../librte_eal/common/arch/arm/meson.build | 5 + .../librte_eal/common/arch/arm/rte_cpuflags.c | 54 +- .../librte_eal/common/arch/arm/rte_cycles.c | 32 +- .../common/arch/arm/rte_hypervisor.c | 11 + .../librte_eal/common/arch/ppc_64/meson.build | 5 + .../common/arch/ppc_64/rte_cpuflags.c | 15 +- .../common/arch/ppc_64/rte_hypervisor.c | 11 + .../librte_eal/common/arch/x86/meson.build | 5 + .../librte_eal/common/arch/x86/rte_cpuflags.c | 44 +- .../librte_eal/common/arch/x86/rte_cpuid.h | 19 + .../librte_eal/common/arch/x86/rte_cycles.c | 33 +- .../common/arch/x86/rte_hypervisor.c | 40 + .../librte_eal/common/arch/x86/rte_memcpy.c | 58 - .../librte_eal/common/arch/x86/rte_spinlock.c | 33 +- dpdk/lib/librte_eal/common/eal_common_bus.c | 48 +- dpdk/lib/librte_eal/common/eal_common_class.c | 64 + .../librte_eal/common/eal_common_cpuflags.c | 33 +- dpdk/lib/librte_eal/common/eal_common_dev.c | 820 +- .../librte_eal/common/eal_common_devargs.c | 319 +- dpdk/lib/librte_eal/common/eal_common_errno.c | 36 +- .../librte_eal/common/eal_common_fbarray.c | 1244 + .../librte_eal/common/eal_common_hexdump.c | 33 +- .../librte_eal/common/eal_common_hypervisor.c | 22 + .../lib/librte_eal/common/eal_common_launch.c | 33 +- dpdk/lib/librte_eal/common/eal_common_lcore.c | 108 +- dpdk/lib/librte_eal/common/eal_common_log.c | 201 +- .../librte_eal/common/eal_common_memalloc.c | 364 + .../lib/librte_eal/common/eal_common_memory.c | 795 +- .../librte_eal/common/eal_common_memzone.c | 331 +- .../librte_eal/common/eal_common_options.c | 306 +- dpdk/lib/librte_eal/common/eal_common_proc.c | 1185 +- .../librte_eal/common/eal_common_string_fns.c | 59 +- .../lib/librte_eal/common/eal_common_tailqs.c | 33 +- .../lib/librte_eal/common/eal_common_thread.c | 131 +- dpdk/lib/librte_eal/common/eal_common_timer.c | 57 +- dpdk/lib/librte_eal/common/eal_common_uuid.c | 193 + dpdk/lib/librte_eal/common/eal_filesystem.h | 103 +- dpdk/lib/librte_eal/common/eal_hugepages.h | 44 +- dpdk/lib/librte_eal/common/eal_internal_cfg.h | 57 +- dpdk/lib/librte_eal/common/eal_memalloc.h | 93 + dpdk/lib/librte_eal/common/eal_options.h | 42 +- dpdk/lib/librte_eal/common/eal_private.h | 208 +- dpdk/lib/librte_eal/common/eal_thread.h | 33 +- dpdk/lib/librte_eal/common/hotplug_mp.c | 440 + dpdk/lib/librte_eal/common/hotplug_mp.h | 46 + .../common/include/arch/arm/meson.build | 29 + .../common/include/arch/arm/rte_atomic.h | 32 +- .../common/include/arch/arm/rte_atomic_32.h | 36 +- .../common/include/arch/arm/rte_atomic_64.h | 38 +- .../common/include/arch/arm/rte_byteorder.h | 34 +- .../common/include/arch/arm/rte_cpuflags.h | 32 +- .../common/include/arch/arm/rte_cpuflags_32.h | 32 +- .../common/include/arch/arm/rte_cpuflags_64.h | 32 +- .../common/include/arch/arm/rte_cycles.h | 32 +- .../common/include/arch/arm/rte_cycles_32.h | 36 +- .../common/include/arch/arm/rte_cycles_64.h | 34 +- .../common/include/arch/arm/rte_io.h | 33 +- .../common/include/arch/arm/rte_io_64.h | 32 +- .../common/include/arch/arm/rte_memcpy.h | 32 +- .../common/include/arch/arm/rte_memcpy_32.h | 32 +- .../common/include/arch/arm/rte_memcpy_64.h | 347 +- .../common/include/arch/arm/rte_pause.h | 32 +- .../common/include/arch/arm/rte_pause_32.h | 32 +- .../common/include/arch/arm/rte_pause_64.h | 32 +- .../common/include/arch/arm/rte_prefetch.h | 32 +- .../common/include/arch/arm/rte_prefetch_32.h | 32 +- .../common/include/arch/arm/rte_prefetch_64.h | 32 +- .../common/include/arch/arm/rte_rwlock.h | 2 + .../common/include/arch/arm/rte_spinlock.h | 32 +- .../common/include/arch/arm/rte_vect.h | 38 +- .../common/include/arch/ppc_64/meson.build | 16 + .../common/include/arch/ppc_64/rte_atomic.h | 25 +- .../common/include/arch/ppc_64/rte_io.h | 33 +- .../common/include/arch/ppc_64/rte_pause.h | 40 +- .../common/include/arch/ppc_64/rte_rwlock.h | 2 + .../common/include/arch/x86/meson.build | 21 + .../common/include/arch/x86/rte_atomic.h | 105 +- .../common/include/arch/x86/rte_atomic_32.h | 12 + .../common/include/arch/x86/rte_atomic_64.h | 12 + .../common/include/arch/x86/rte_byteorder.h | 37 +- .../include/arch/x86/rte_byteorder_32.h | 33 +- .../include/arch/x86/rte_byteorder_64.h | 35 +- .../common/include/arch/x86/rte_cpuflags.h | 34 +- .../common/include/arch/x86/rte_cycles.h | 35 +- .../common/include/arch/x86/rte_io.h | 33 +- .../common/include/arch/x86/rte_memcpy.h | 57 +- .../common/include/arch/x86/rte_pause.h | 33 +- .../common/include/arch/x86/rte_prefetch.h | 33 +- .../common/include/arch/x86/rte_rtm.h | 19 +- .../common/include/arch/x86/rte_rwlock.h | 33 +- .../common/include/arch/x86/rte_spinlock.h | 56 +- .../common/include/arch/x86/rte_vect.h | 33 +- .../common/include/generic/rte_atomic.h | 180 +- .../common/include/generic/rte_byteorder.h | 39 +- .../common/include/generic/rte_cpuflags.h | 54 +- .../common/include/generic/rte_cycles.h | 46 +- .../common/include/generic/rte_io.h | 33 +- .../common/include/generic/rte_memcpy.h | 33 +- .../common/include/generic/rte_pause.h | 33 +- .../common/include/generic/rte_prefetch.h | 33 +- .../common/include/generic/rte_rwlock.h | 37 +- .../common/include/generic/rte_spinlock.h | 33 +- .../common/include/generic/rte_vect.h | 32 +- .../lib/librte_eal/common/include/rte_alarm.h | 33 +- .../librte_eal/common/include/rte_bitmap.h | 92 +- .../common/include/rte_branch_prediction.h | 37 +- dpdk/lib/librte_eal/common/include/rte_bus.h | 38 +- .../lib/librte_eal/common/include/rte_class.h | 134 + .../librte_eal/common/include/rte_common.h | 265 +- .../lib/librte_eal/common/include/rte_debug.h | 35 +- dpdk/lib/librte_eal/common/include/rte_dev.h | 363 +- .../librte_eal/common/include/rte_devargs.h | 206 +- dpdk/lib/librte_eal/common/include/rte_eal.h | 248 +- .../common/include/rte_eal_interrupts.h | 35 +- .../common/include/rte_eal_memconfig.h | 77 +- .../lib/librte_eal/common/include/rte_errno.h | 33 +- .../librte_eal/common/include/rte_fbarray.h | 470 + .../librte_eal/common/include/rte_hexdump.h | 33 +- .../common/include/rte_hypervisor.h | 33 + .../common/include/rte_interrupts.h | 33 +- .../librte_eal/common/include/rte_launch.h | 33 +- .../lib/librte_eal/common/include/rte_lcore.h | 109 +- dpdk/lib/librte_eal/common/include/rte_log.h | 73 +- .../librte_eal/common/include/rte_malloc.h | 235 +- .../common/include/rte_malloc_heap.h | 42 +- .../librte_eal/common/include/rte_memory.h | 510 +- .../librte_eal/common/include/rte_memzone.h | 78 +- .../librte_eal/common/include/rte_option.h | 63 + .../common/include/rte_pci_dev_feature_defs.h | 58 +- .../common/include/rte_pci_dev_features.h | 58 +- .../librte_eal/common/include/rte_per_lcore.h | 33 +- .../librte_eal/common/include/rte_random.h | 39 +- .../common/include/rte_reciprocal.h | 90 + .../librte_eal/common/include/rte_service.h | 174 +- .../common/include/rte_service_component.h | 65 +- .../common/include/rte_string_fns.h | 86 +- .../lib/librte_eal/common/include/rte_tailq.h | 36 +- dpdk/lib/librte_eal/common/include/rte_test.h | 46 + dpdk/lib/librte_eal/common/include/rte_time.h | 33 +- dpdk/lib/librte_eal/common/include/rte_uuid.h | 129 + .../librte_eal/common/include/rte_version.h | 37 +- dpdk/lib/librte_eal/common/include/rte_vfio.h | 307 +- dpdk/lib/librte_eal/common/malloc_elem.c | 520 +- dpdk/lib/librte_eal/common/malloc_elem.h | 84 +- dpdk/lib/librte_eal/common/malloc_heap.c | 1208 +- dpdk/lib/librte_eal/common/malloc_heap.h | 69 +- dpdk/lib/librte_eal/common/malloc_mp.c | 741 + dpdk/lib/librte_eal/common/malloc_mp.h | 86 + dpdk/lib/librte_eal/common/meson.build | 105 + dpdk/lib/librte_eal/common/rte_keepalive.c | 32 +- dpdk/lib/librte_eal/common/rte_malloc.c | 541 +- dpdk/lib/librte_eal/common/rte_option.c | 54 + dpdk/lib/librte_eal/common/rte_reciprocal.c | 149 + dpdk/lib/librte_eal/common/rte_service.c | 187 +- dpdk/lib/librte_eal/linuxapp/Makefile | 36 +- dpdk/lib/librte_eal/linuxapp/eal/Makefile | 65 +- dpdk/lib/librte_eal/linuxapp/eal/eal.c | 411 +- dpdk/lib/librte_eal/linuxapp/eal/eal_alarm.c | 44 +- .../librte_eal/linuxapp/eal/eal_cpuflags.c | 84 + dpdk/lib/librte_eal/linuxapp/eal/eal_debug.c | 37 +- dpdk/lib/librte_eal/linuxapp/eal/eal_dev.c | 396 + .../linuxapp/eal/eal_hugepage_info.c | 287 +- .../librte_eal/linuxapp/eal/eal_interrupts.c | 173 +- dpdk/lib/librte_eal/linuxapp/eal/eal_lcore.c | 33 +- dpdk/lib/librte_eal/linuxapp/eal/eal_log.c | 46 +- .../librte_eal/linuxapp/eal/eal_memalloc.c | 1650 + dpdk/lib/librte_eal/linuxapp/eal/eal_memory.c | 1752 +- dpdk/lib/librte_eal/linuxapp/eal/eal_thread.c | 39 +- dpdk/lib/librte_eal/linuxapp/eal/eal_timer.c | 52 +- dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.c | 1718 +- dpdk/lib/librte_eal/linuxapp/eal/eal_vfio.h | 103 +- .../linuxapp/eal/eal_vfio_mp_sync.c | 450 +- .../eal/include/exec-env/rte_kni_common.h | 71 +- dpdk/lib/librte_eal/linuxapp/eal/meson.build | 29 + dpdk/lib/librte_eal/linuxapp/igb_uio/Makefile | 53 - dpdk/lib/librte_eal/linuxapp/igb_uio/compat.h | 134 - .../lib/librte_eal/linuxapp/igb_uio/igb_uio.c | 662 - dpdk/lib/librte_eal/linuxapp/kni/Makefile | 86 - dpdk/lib/librte_eal/linuxapp/kni/compat.h | 111 - .../librte_eal/linuxapp/kni/ethtool/README | 100 - .../linuxapp/kni/ethtool/igb/e1000_82575.c | 3665 -- .../linuxapp/kni/ethtool/igb/e1000_82575.h | 509 - .../linuxapp/kni/ethtool/igb/e1000_api.c | 1159 - .../linuxapp/kni/ethtool/igb/e1000_api.h | 157 - .../linuxapp/kni/ethtool/igb/e1000_defines.h | 1380 - .../linuxapp/kni/ethtool/igb/e1000_hw.h | 793 - .../linuxapp/kni/ethtool/igb/e1000_i210.c | 909 - .../linuxapp/kni/ethtool/igb/e1000_i210.h | 91 - .../linuxapp/kni/ethtool/igb/e1000_mac.c | 2096 - .../linuxapp/kni/ethtool/igb/e1000_mac.h | 80 - .../linuxapp/kni/ethtool/igb/e1000_manage.c | 554 - .../linuxapp/kni/ethtool/igb/e1000_manage.h | 89 - .../linuxapp/kni/ethtool/igb/e1000_mbx.c | 525 - .../linuxapp/kni/ethtool/igb/e1000_mbx.h | 87 - .../linuxapp/kni/ethtool/igb/e1000_nvm.c | 965 - .../linuxapp/kni/ethtool/igb/e1000_nvm.h | 75 - .../linuxapp/kni/ethtool/igb/e1000_osdep.h | 136 - .../linuxapp/kni/ethtool/igb/e1000_phy.c | 3407 -- .../linuxapp/kni/ethtool/igb/e1000_phy.h | 256 - .../linuxapp/kni/ethtool/igb/e1000_regs.h | 646 - .../librte_eal/linuxapp/kni/ethtool/igb/igb.h | 859 - .../linuxapp/kni/ethtool/igb/igb_ethtool.c | 2858 -- .../linuxapp/kni/ethtool/igb/igb_main.c | 10359 ----- .../linuxapp/kni/ethtool/igb/igb_param.c | 847 - .../linuxapp/kni/ethtool/igb/igb_regtest.h | 249 - .../linuxapp/kni/ethtool/igb/igb_vmdq.c | 436 - .../linuxapp/kni/ethtool/igb/igb_vmdq.h | 46 - .../linuxapp/kni/ethtool/igb/kcompat.h | 3960 -- .../linuxapp/kni/ethtool/ixgbe/ixgbe.h | 925 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_82598.c | 1296 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_82598.h | 44 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_82599.c | 2314 -- .../linuxapp/kni/ethtool/ixgbe/ixgbe_82599.h | 58 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_api.c | 1157 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_api.h | 168 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_common.c | 4082 -- .../linuxapp/kni/ethtool/ixgbe/ixgbe_common.h | 140 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_dcb.h | 168 - .../kni/ethtool/ixgbe/ixgbe_ethtool.c | 2901 -- .../linuxapp/kni/ethtool/ixgbe/ixgbe_fcoe.h | 91 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_main.c | 2966 -- .../linuxapp/kni/ethtool/ixgbe/ixgbe_mbx.h | 105 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_osdep.h | 132 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_phy.c | 1847 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_phy.h | 137 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_type.h | 3254 -- .../linuxapp/kni/ethtool/ixgbe/ixgbe_x540.c | 937 - .../linuxapp/kni/ethtool/ixgbe/ixgbe_x540.h | 58 - .../linuxapp/kni/ethtool/ixgbe/kcompat.c | 1246 - .../linuxapp/kni/ethtool/ixgbe/kcompat.h | 3155 -- dpdk/lib/librte_eal/linuxapp/kni/kni_dev.h | 125 - .../lib/librte_eal/linuxapp/kni/kni_ethtool.c | 238 - dpdk/lib/librte_eal/linuxapp/kni/kni_fifo.h | 94 - dpdk/lib/librte_eal/linuxapp/kni/kni_misc.c | 674 - dpdk/lib/librte_eal/linuxapp/kni/kni_net.c | 747 - dpdk/lib/librte_eal/meson.build | 31 + dpdk/lib/librte_eal/rte_eal_version.map | 168 +- dpdk/lib/librte_efd/Makefile | 32 +- dpdk/lib/librte_efd/meson.build | 6 + dpdk/lib/librte_efd/rte_efd.c | 44 +- dpdk/lib/librte_efd/rte_efd.h | 33 +- dpdk/lib/librte_efd/rte_efd_arm64.h | 32 +- dpdk/lib/librte_efd/rte_efd_x86.h | 33 +- dpdk/lib/librte_ethdev/Makefile | 46 + dpdk/lib/librte_ethdev/ethdev_private.c | 121 + dpdk/lib/librte_ethdev/ethdev_private.h | 38 + dpdk/lib/librte_ethdev/ethdev_profile.c | 62 + dpdk/lib/librte_ethdev/ethdev_profile.h | 27 + dpdk/lib/librte_ethdev/meson.build | 29 + dpdk/lib/librte_ethdev/rte_class_eth.c | 173 + dpdk/lib/librte_ethdev/rte_dev_info.h | 49 + dpdk/lib/librte_ethdev/rte_eth_ctrl.h | 828 + dpdk/lib/librte_ethdev/rte_ethdev.c | 4482 +++ dpdk/lib/librte_ethdev/rte_ethdev.h | 4361 ++ dpdk/lib/librte_ethdev/rte_ethdev_core.h | 641 + dpdk/lib/librte_ethdev/rte_ethdev_driver.h | 338 + dpdk/lib/librte_ethdev/rte_ethdev_pci.h | 199 + dpdk/lib/librte_ethdev/rte_ethdev_vdev.h | 84 + dpdk/lib/librte_ethdev/rte_ethdev_version.map | 264 + dpdk/lib/librte_ethdev/rte_flow.c | 1003 + dpdk/lib/librte_ethdev/rte_flow.h | 2684 ++ dpdk/lib/librte_ethdev/rte_flow_driver.h | 184 + dpdk/lib/librte_ethdev/rte_mtr.c | 201 + dpdk/lib/librte_ethdev/rte_mtr.h | 730 + dpdk/lib/librte_ethdev/rte_mtr_driver.h | 192 + dpdk/lib/librte_ethdev/rte_tm.c | 409 + dpdk/lib/librte_ethdev/rte_tm.h | 1965 + dpdk/lib/librte_ethdev/rte_tm_driver.h | 337 + dpdk/lib/librte_ether/Makefile | 69 - dpdk/lib/librte_ether/ethdev_profile.c | 164 - dpdk/lib/librte_ether/ethdev_profile.h | 56 - dpdk/lib/librte_ether/rte_dev_info.h | 60 - dpdk/lib/librte_ether/rte_eth_ctrl.h | 856 - dpdk/lib/librte_ether/rte_ethdev.c | 3610 -- dpdk/lib/librte_ether/rte_ethdev.h | 4616 --- dpdk/lib/librte_ether/rte_ethdev_pci.h | 196 - dpdk/lib/librte_ether/rte_ethdev_vdev.h | 84 - dpdk/lib/librte_ether/rte_ethdev_version.map | 217 - dpdk/lib/librte_ether/rte_flow.c | 427 - dpdk/lib/librte_ether/rte_flow.h | 1479 - dpdk/lib/librte_ether/rte_flow_driver.h | 149 - dpdk/lib/librte_ether/rte_mtr.c | 229 - dpdk/lib/librte_ether/rte_mtr.h | 730 - dpdk/lib/librte_ether/rte_mtr_driver.h | 221 - dpdk/lib/librte_ether/rte_tm.c | 438 - dpdk/lib/librte_ether/rte_tm.h | 1912 - dpdk/lib/librte_ether/rte_tm_driver.h | 366 - dpdk/lib/librte_eventdev/Makefile | 48 +- dpdk/lib/librte_eventdev/meson.build | 29 + .../rte_event_crypto_adapter.c | 1128 + .../rte_event_crypto_adapter.h | 575 + .../rte_event_eth_rx_adapter.c | 1656 +- .../rte_event_eth_rx_adapter.h | 131 +- .../rte_event_eth_tx_adapter.c | 1138 + .../rte_event_eth_tx_adapter.h | 462 + dpdk/lib/librte_eventdev/rte_event_ring.c | 32 +- dpdk/lib/librte_eventdev/rte_event_ring.h | 38 +- .../librte_eventdev/rte_event_timer_adapter.c | 1299 + .../librte_eventdev/rte_event_timer_adapter.h | 766 + .../rte_event_timer_adapter_pmd.h | 114 + dpdk/lib/librte_eventdev/rte_eventdev.c | 225 +- dpdk/lib/librte_eventdev/rte_eventdev.h | 298 +- dpdk/lib/librte_eventdev/rte_eventdev_pmd.h | 491 +- .../librte_eventdev/rte_eventdev_pmd_pci.h | 31 +- .../librte_eventdev/rte_eventdev_pmd_vdev.h | 31 +- .../librte_eventdev/rte_eventdev_version.map | 58 +- dpdk/lib/librte_flow_classify/Makefile | 33 +- dpdk/lib/librte_flow_classify/meson.build | 7 + .../librte_flow_classify/rte_flow_classify.c | 376 +- .../librte_flow_classify/rte_flow_classify.h | 130 +- .../rte_flow_classify_parse.c | 167 +- .../rte_flow_classify_parse.h | 49 +- .../rte_flow_classify_version.map | 1 + dpdk/lib/librte_gro/Makefile | 33 +- dpdk/lib/librte_gro/gro_tcp4.c | 357 +- dpdk/lib/librte_gro/gro_tcp4.h | 283 +- dpdk/lib/librte_gro/gro_vxlan_tcp4.c | 494 + dpdk/lib/librte_gro/gro_vxlan_tcp4.h | 156 + dpdk/lib/librte_gro/meson.build | 6 + dpdk/lib/librte_gro/rte_gro.c | 231 +- dpdk/lib/librte_gro/rte_gro.h | 129 +- dpdk/lib/librte_gso/Makefile | 33 +- dpdk/lib/librte_gso/gso_common.c | 33 +- dpdk/lib/librte_gso/gso_common.h | 36 +- dpdk/lib/librte_gso/gso_tcp4.c | 33 +- dpdk/lib/librte_gso/gso_tcp4.h | 33 +- dpdk/lib/librte_gso/gso_tunnel_tcp4.c | 33 +- dpdk/lib/librte_gso/gso_tunnel_tcp4.h | 33 +- dpdk/lib/librte_gso/gso_udp4.c | 81 + dpdk/lib/librte_gso/gso_udp4.h | 42 + dpdk/lib/librte_gso/meson.build | 7 + dpdk/lib/librte_gso/rte_gso.c | 57 +- dpdk/lib/librte_gso/rte_gso.h | 39 +- dpdk/lib/librte_hash/Makefile | 32 +- dpdk/lib/librte_hash/meson.build | 16 + dpdk/lib/librte_hash/rte_cmp_arm64.h | 33 +- dpdk/lib/librte_hash/rte_cmp_x86.h | 35 +- dpdk/lib/librte_hash/rte_crc_arm64.h | 33 +- dpdk/lib/librte_hash/rte_cuckoo_hash.c | 1748 +- dpdk/lib/librte_hash/rte_cuckoo_hash.h | 91 +- dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h | 196 - dpdk/lib/librte_hash/rte_fbk_hash.c | 46 +- dpdk/lib/librte_hash/rte_fbk_hash.h | 41 +- dpdk/lib/librte_hash/rte_hash.h | 186 +- dpdk/lib/librte_hash/rte_hash_crc.h | 44 +- dpdk/lib/librte_hash/rte_hash_version.map | 15 + dpdk/lib/librte_ip_frag/Makefile | 32 +- dpdk/lib/librte_ip_frag/ip_frag_common.h | 70 +- dpdk/lib/librte_ip_frag/ip_frag_internal.c | 59 +- dpdk/lib/librte_ip_frag/meson.build | 11 + dpdk/lib/librte_ip_frag/rte_ip_frag.h | 66 +- dpdk/lib/librte_ip_frag/rte_ip_frag_common.c | 54 +- .../librte_ip_frag/rte_ip_frag_version.map | 10 +- .../librte_ip_frag/rte_ipv4_fragmentation.c | 33 +- dpdk/lib/librte_ip_frag/rte_ipv4_reassembly.c | 55 +- .../librte_ip_frag/rte_ipv6_fragmentation.c | 33 +- dpdk/lib/librte_ip_frag/rte_ipv6_reassembly.c | 61 +- dpdk/lib/librte_jobstats/Makefile | 32 +- dpdk/lib/librte_jobstats/meson.build | 5 + dpdk/lib/librte_jobstats/rte_jobstats.c | 33 +- dpdk/lib/librte_jobstats/rte_jobstats.h | 33 +- dpdk/lib/librte_kni/Makefile | 32 +- dpdk/lib/librte_kni/meson.build | 10 + dpdk/lib/librte_kni/rte_kni.c | 661 +- dpdk/lib/librte_kni/rte_kni.h | 68 +- dpdk/lib/librte_kni/rte_kni_fifo.h | 76 +- dpdk/lib/librte_kni/rte_kni_version.map | 6 + dpdk/lib/librte_kvargs/Makefile | 34 +- dpdk/lib/librte_kvargs/meson.build | 11 + dpdk/lib/librte_kvargs/rte_kvargs.c | 112 +- dpdk/lib/librte_kvargs/rte_kvargs.h | 100 +- dpdk/lib/librte_kvargs/rte_kvargs_version.map | 8 + dpdk/lib/librte_latencystats/Makefile | 31 +- dpdk/lib/librte_latencystats/meson.build | 6 + .../librte_latencystats/rte_latencystats.c | 53 +- .../librte_latencystats/rte_latencystats.h | 32 +- dpdk/lib/librte_lpm/Makefile | 34 +- dpdk/lib/librte_lpm/meson.build | 10 + dpdk/lib/librte_lpm/rte_lpm.c | 59 +- dpdk/lib/librte_lpm/rte_lpm.h | 33 +- dpdk/lib/librte_lpm/rte_lpm6.c | 1085 +- dpdk/lib/librte_lpm/rte_lpm6.h | 33 +- dpdk/lib/librte_lpm/rte_lpm_sse.h | 33 +- dpdk/lib/librte_mbuf/Makefile | 38 +- dpdk/lib/librte_mbuf/meson.build | 7 + dpdk/lib/librte_mbuf/rte_mbuf.c | 148 +- dpdk/lib/librte_mbuf/rte_mbuf.h | 755 +- dpdk/lib/librte_mbuf/rte_mbuf_pool_ops.c | 97 + dpdk/lib/librte_mbuf/rte_mbuf_pool_ops.h | 95 + dpdk/lib/librte_mbuf/rte_mbuf_ptype.c | 41 +- dpdk/lib/librte_mbuf/rte_mbuf_ptype.h | 136 +- dpdk/lib/librte_mbuf/rte_mbuf_version.map | 12 +- dpdk/lib/librte_member/Makefile | 32 +- dpdk/lib/librte_member/meson.build | 6 + dpdk/lib/librte_member/rte_member.c | 44 +- dpdk/lib/librte_member/rte_member.h | 33 +- dpdk/lib/librte_member/rte_member_ht.c | 33 +- dpdk/lib/librte_member/rte_member_ht.h | 33 +- dpdk/lib/librte_member/rte_member_vbf.c | 33 +- dpdk/lib/librte_member/rte_member_vbf.h | 33 +- dpdk/lib/librte_member/rte_member_x86.h | 33 +- dpdk/lib/librte_mempool/Makefile | 39 +- dpdk/lib/librte_mempool/meson.build | 21 + dpdk/lib/librte_mempool/rte_mempool.c | 609 +- dpdk/lib/librte_mempool/rte_mempool.h | 625 +- dpdk/lib/librte_mempool/rte_mempool_ops.c | 102 +- .../librte_mempool/rte_mempool_ops_default.c | 70 + .../librte_mempool/rte_mempool_version.map | 23 +- dpdk/lib/librte_meter/Makefile | 34 +- dpdk/lib/librte_meter/meson.build | 6 + dpdk/lib/librte_meter/rte_meter.c | 138 +- dpdk/lib/librte_meter/rte_meter.h | 226 +- dpdk/lib/librte_meter/rte_meter_version.map | 7 + dpdk/lib/librte_metrics/Makefile | 32 +- dpdk/lib/librte_metrics/meson.build | 5 + dpdk/lib/librte_metrics/rte_metrics.c | 41 +- dpdk/lib/librte_metrics/rte_metrics.h | 33 +- dpdk/lib/librte_net/Makefile | 38 +- dpdk/lib/librte_net/meson.build | 20 + dpdk/lib/librte_net/net_crc_neon.h | 32 +- dpdk/lib/librte_net/net_crc_sse.h | 37 +- dpdk/lib/librte_net/rte_arp.c | 50 + dpdk/lib/librte_net/rte_arp.h | 49 +- dpdk/lib/librte_net/rte_esp.h | 32 +- dpdk/lib/librte_net/rte_ether.h | 66 +- dpdk/lib/librte_net/rte_gre.h | 33 +- dpdk/lib/librte_net/rte_icmp.h | 69 +- dpdk/lib/librte_net/rte_ip.h | 114 +- dpdk/lib/librte_net/rte_mpls.h | 42 + dpdk/lib/librte_net/rte_net.c | 75 +- dpdk/lib/librte_net/rte_net.h | 80 +- dpdk/lib/librte_net/rte_net_crc.c | 37 +- dpdk/lib/librte_net/rte_net_crc.h | 33 +- dpdk/lib/librte_net/rte_net_version.map | 7 + dpdk/lib/librte_net/rte_sctp.h | 72 +- dpdk/lib/librte_net/rte_tcp.h | 72 +- dpdk/lib/librte_net/rte_udp.h | 72 +- dpdk/lib/librte_pci/Makefile | 32 +- dpdk/lib/librte_pci/meson.build | 5 + dpdk/lib/librte_pci/rte_pci.c | 50 +- dpdk/lib/librte_pci/rte_pci.h | 35 +- dpdk/lib/librte_pdump/Makefile | 35 +- dpdk/lib/librte_pdump/meson.build | 8 + dpdk/lib/librte_pdump/rte_pdump.c | 451 +- dpdk/lib/librte_pdump/rte_pdump.h | 42 +- dpdk/lib/librte_pipeline/Makefile | 39 +- dpdk/lib/librte_pipeline/meson.build | 8 + dpdk/lib/librte_pipeline/rte_pipeline.c | 38 +- dpdk/lib/librte_pipeline/rte_pipeline.h | 33 +- .../librte_pipeline/rte_pipeline_version.map | 29 + dpdk/lib/librte_pipeline/rte_port_in_action.c | 531 + dpdk/lib/librte_pipeline/rte_port_in_action.h | 301 + dpdk/lib/librte_pipeline/rte_table_action.c | 3431 ++ dpdk/lib/librte_pipeline/rte_table_action.h | 1108 + dpdk/lib/librte_port/Makefile | 36 +- dpdk/lib/librte_port/meson.build | 30 + dpdk/lib/librte_port/rte_port.h | 33 +- dpdk/lib/librte_port/rte_port_ethdev.c | 33 +- dpdk/lib/librte_port/rte_port_ethdev.h | 33 +- dpdk/lib/librte_port/rte_port_fd.c | 33 +- dpdk/lib/librte_port/rte_port_fd.h | 33 +- dpdk/lib/librte_port/rte_port_frag.c | 33 +- dpdk/lib/librte_port/rte_port_frag.h | 33 +- dpdk/lib/librte_port/rte_port_ras.c | 33 +- dpdk/lib/librte_port/rte_port_ras.h | 33 +- dpdk/lib/librte_port/rte_port_ring.c | 33 +- dpdk/lib/librte_port/rte_port_ring.h | 33 +- dpdk/lib/librte_port/rte_port_sched.c | 33 +- dpdk/lib/librte_port/rte_port_sched.h | 33 +- dpdk/lib/librte_port/rte_port_source_sink.c | 33 +- dpdk/lib/librte_port/rte_port_source_sink.h | 33 +- dpdk/lib/librte_port/rte_port_sym_crypto.c | 552 + dpdk/lib/librte_port/rte_port_sym_crypto.h | 93 + dpdk/lib/librte_port/rte_port_version.map | 9 + dpdk/lib/librte_power/Makefile | 41 +- dpdk/lib/librte_power/channel_commands.h | 41 +- dpdk/lib/librte_power/guest_channel.c | 39 +- dpdk/lib/librte_power/guest_channel.h | 39 +- dpdk/lib/librte_power/meson.build | 11 + dpdk/lib/librte_power/power_acpi_cpufreq.c | 646 + dpdk/lib/librte_power/power_acpi_cpufreq.h | 219 + dpdk/lib/librte_power/power_common.h | 10 + dpdk/lib/librte_power/power_kvm_vm.c | 134 + dpdk/lib/librte_power/power_kvm_vm.h | 200 + dpdk/lib/librte_power/rte_power.c | 98 +- dpdk/lib/librte_power/rte_power.h | 77 +- .../lib/librte_power/rte_power_acpi_cpufreq.c | 654 - .../lib/librte_power/rte_power_acpi_cpufreq.h | 232 - dpdk/lib/librte_power/rte_power_common.h | 39 - dpdk/lib/librte_power/rte_power_empty_poll.c | 545 + dpdk/lib/librte_power/rte_power_empty_poll.h | 223 + dpdk/lib/librte_power/rte_power_kvm_vm.c | 155 - dpdk/lib/librte_power/rte_power_kvm_vm.h | 212 - dpdk/lib/librte_power/rte_power_version.map | 21 +- dpdk/lib/librte_rawdev/Makefile | 27 + dpdk/lib/librte_rawdev/meson.build | 5 + dpdk/lib/librte_rawdev/rte_rawdev.c | 550 + dpdk/lib/librte_rawdev/rte_rawdev.h | 610 + dpdk/lib/librte_rawdev/rte_rawdev_pmd.h | 625 + dpdk/lib/librte_rawdev/rte_rawdev_version.map | 35 + dpdk/lib/librte_reorder/Makefile | 32 +- dpdk/lib/librte_reorder/meson.build | 6 + dpdk/lib/librte_reorder/rte_reorder.c | 33 +- dpdk/lib/librte_reorder/rte_reorder.h | 33 +- dpdk/lib/librte_ring/Makefile | 38 +- dpdk/lib/librte_ring/meson.build | 8 + dpdk/lib/librte_ring/rte_ring.c | 66 +- dpdk/lib/librte_ring/rte_ring.h | 262 +- dpdk/lib/librte_ring/rte_ring_c11_mem.h | 181 + dpdk/lib/librte_ring/rte_ring_generic.h | 170 + dpdk/lib/librte_ring/rte_ring_version.map | 1 - dpdk/lib/librte_sched/Makefile | 36 +- dpdk/lib/librte_sched/meson.build | 7 + dpdk/lib/librte_sched/rte_approx.c | 33 +- dpdk/lib/librte_sched/rte_approx.h | 33 +- dpdk/lib/librte_sched/rte_reciprocal.c | 72 - dpdk/lib/librte_sched/rte_reciprocal.h | 41 - dpdk/lib/librte_sched/rte_red.c | 33 +- dpdk/lib/librte_sched/rte_red.h | 33 +- dpdk/lib/librte_sched/rte_sched.c | 287 +- dpdk/lib/librte_sched/rte_sched.h | 54 +- dpdk/lib/librte_sched/rte_sched_common.h | 33 +- dpdk/lib/librte_sched/rte_sched_version.map | 6 + dpdk/lib/librte_security/Makefile | 31 +- dpdk/lib/librte_security/meson.build | 6 + dpdk/lib/librte_security/rte_security.c | 63 +- dpdk/lib/librte_security/rte_security.h | 166 +- .../lib/librte_security/rte_security_driver.h | 67 +- .../librte_security/rte_security_version.map | 12 +- dpdk/lib/librte_table/Makefile | 35 +- dpdk/lib/librte_table/meson.build | 31 + dpdk/lib/librte_table/rte_lru.h | 33 +- dpdk/lib/librte_table/rte_lru_arm64.h | 32 +- dpdk/lib/librte_table/rte_lru_x86.h | 33 +- dpdk/lib/librte_table/rte_table.h | 33 +- dpdk/lib/librte_table/rte_table_acl.c | 39 +- dpdk/lib/librte_table/rte_table_acl.h | 33 +- dpdk/lib/librte_table/rte_table_array.c | 33 +- dpdk/lib/librte_table/rte_table_array.h | 33 +- dpdk/lib/librte_table/rte_table_hash.h | 36 +- dpdk/lib/librte_table/rte_table_hash_cuckoo.c | 44 +- dpdk/lib/librte_table/rte_table_hash_cuckoo.h | 57 + dpdk/lib/librte_table/rte_table_hash_ext.c | 33 +- dpdk/lib/librte_table/rte_table_hash_func.h | 245 + .../librte_table/rte_table_hash_func_arm64.h | 21 + dpdk/lib/librte_table/rte_table_hash_key16.c | 33 +- dpdk/lib/librte_table/rte_table_hash_key32.c | 33 +- dpdk/lib/librte_table/rte_table_hash_key8.c | 33 +- dpdk/lib/librte_table/rte_table_hash_lru.c | 33 +- dpdk/lib/librte_table/rte_table_lpm.c | 33 +- dpdk/lib/librte_table/rte_table_lpm.h | 33 +- dpdk/lib/librte_table/rte_table_lpm_ipv6.c | 33 +- dpdk/lib/librte_table/rte_table_lpm_ipv6.h | 33 +- dpdk/lib/librte_table/rte_table_stub.c | 33 +- dpdk/lib/librte_table/rte_table_stub.h | 33 +- dpdk/lib/librte_telemetry/Makefile | 30 + dpdk/lib/librte_telemetry/meson.build | 15 + dpdk/lib/librte_telemetry/rte_telemetry.c | 1813 + dpdk/lib/librte_telemetry/rte_telemetry.h | 66 + .../librte_telemetry/rte_telemetry_internal.h | 81 + .../librte_telemetry/rte_telemetry_parser.c | 586 + .../librte_telemetry/rte_telemetry_parser.h | 14 + .../rte_telemetry_parser_test.c | 534 + .../rte_telemetry_parser_test.h | 39 + .../rte_telemetry_socket_tests.h | 36 + .../rte_telemetry_version.map | 10 + dpdk/lib/librte_timer/Makefile | 32 +- dpdk/lib/librte_timer/meson.build | 5 + dpdk/lib/librte_timer/rte_timer.c | 35 +- dpdk/lib/librte_timer/rte_timer.h | 33 +- dpdk/lib/librte_vhost/Makefile | 47 +- dpdk/lib/librte_vhost/fd_man.c | 99 +- dpdk/lib/librte_vhost/fd_man.h | 49 +- dpdk/lib/librte_vhost/meson.build | 19 + dpdk/lib/librte_vhost/rte_vdpa.h | 158 + dpdk/lib/librte_vhost/rte_vhost.h | 219 +- dpdk/lib/librte_vhost/rte_vhost_crypto.h | 109 + dpdk/lib/librte_vhost/rte_vhost_version.map | 32 +- dpdk/lib/librte_vhost/socket.c | 352 +- dpdk/lib/librte_vhost/vdpa.c | 124 + dpdk/lib/librte_vhost/vhost.c | 374 +- dpdk/lib/librte_vhost/vhost.h | 328 +- dpdk/lib/librte_vhost/vhost_crypto.c | 1607 + dpdk/lib/librte_vhost/vhost_user.c | 1174 +- dpdk/lib/librte_vhost/vhost_user.h | 95 +- dpdk/lib/librte_vhost/virtio_crypto.h | 422 + dpdk/lib/librte_vhost/virtio_net.c | 1769 +- dpdk/lib/meson.build | 139 + dpdk/license/README | 77 + dpdk/license/bsd-3-clause.txt | 9 + dpdk/license/exceptions.txt | 18 + dpdk/license/gpl-2.0.txt | 339 + dpdk/license/lgpl-2.1.txt | 502 + dpdk/meson.build | 125 + dpdk/meson_options.txt | 30 + dpdk/mk/arch/arm/rte.vars.mk | 32 +- dpdk/mk/arch/arm64/rte.vars.mk | 30 +- dpdk/mk/arch/i686/rte.vars.mk | 32 +- dpdk/mk/arch/x86_64/rte.vars.mk | 32 +- dpdk/mk/arch/x86_x32/rte.vars.mk | 32 +- dpdk/mk/exec-env/bsdapp/rte.app.mk | 32 +- dpdk/mk/exec-env/bsdapp/rte.vars.mk | 32 +- dpdk/mk/exec-env/linuxapp/rte.app.mk | 32 +- dpdk/mk/exec-env/linuxapp/rte.vars.mk | 32 +- dpdk/mk/internal/rte.build-post.mk | 32 +- dpdk/mk/internal/rte.build-pre.mk | 32 +- dpdk/mk/internal/rte.clean-post.mk | 32 +- dpdk/mk/internal/rte.clean-pre.mk | 32 +- dpdk/mk/internal/rte.compile-post.mk | 32 +- dpdk/mk/internal/rte.compile-pre.mk | 38 +- dpdk/mk/internal/rte.extvars.mk | 32 +- dpdk/mk/internal/rte.install-post.mk | 32 +- dpdk/mk/internal/rte.install-pre.mk | 32 +- dpdk/mk/machine/armv7a/rte.vars.mk | 31 +- dpdk/mk/machine/armv8a/rte.vars.mk | 31 +- dpdk/mk/machine/default/rte.vars.mk | 32 +- dpdk/mk/machine/dpaa/rte.vars.mk | 36 +- dpdk/mk/machine/dpaa2/rte.vars.mk | 36 +- dpdk/mk/machine/hsw/rte.vars.mk | 32 +- dpdk/mk/machine/ivb/rte.vars.mk | 32 +- dpdk/mk/machine/native/rte.vars.mk | 32 +- dpdk/mk/machine/nhm/rte.vars.mk | 32 +- dpdk/mk/machine/silvermont/rte.vars.mk | 32 +- dpdk/mk/machine/snb/rte.vars.mk | 32 +- dpdk/mk/machine/thunderx/rte.vars.mk | 31 +- dpdk/mk/machine/wsm/rte.vars.mk | 32 +- dpdk/mk/machine/xgene1/rte.vars.mk | 31 +- dpdk/mk/rte.app.mk | 192 +- dpdk/mk/rte.bsdmodule.mk | 32 +- dpdk/mk/rte.combinedlib.mk | 32 +- dpdk/mk/rte.cpuflags.mk | 39 +- dpdk/mk/rte.extapp.mk | 32 +- dpdk/mk/rte.extlib.mk | 32 +- dpdk/mk/rte.extobj.mk | 32 +- dpdk/mk/rte.extshared.mk | 29 +- dpdk/mk/rte.extsubdir.mk | 31 +- dpdk/mk/rte.gnuconfigure.mk | 32 +- dpdk/mk/rte.hostapp.mk | 32 +- dpdk/mk/rte.hostlib.mk | 32 +- dpdk/mk/rte.install.mk | 32 +- dpdk/mk/rte.lib.mk | 32 +- dpdk/mk/rte.module.mk | 32 +- dpdk/mk/rte.obj.mk | 32 +- dpdk/mk/rte.sdkbuild.mk | 33 +- dpdk/mk/rte.sdkconfig.mk | 35 +- dpdk/mk/rte.sdkdepdirs.mk | 32 +- dpdk/mk/rte.sdkdoc.mk | 63 +- dpdk/mk/rte.sdkexamples.mk | 31 +- dpdk/mk/rte.sdkgcov.mk | 32 +- dpdk/mk/rte.sdkinstall.mk | 34 +- dpdk/mk/rte.sdkroot.mk | 32 +- dpdk/mk/rte.sdktest.mk | 32 +- dpdk/mk/rte.shared.mk | 29 +- dpdk/mk/rte.subdir.mk | 32 +- dpdk/mk/rte.vars.mk | 32 +- dpdk/mk/target/generic/rte.app.mk | 32 +- dpdk/mk/target/generic/rte.vars.mk | 35 +- .../toolchain/clang/rte.toolchain-compat.mk | 32 +- dpdk/mk/toolchain/clang/rte.vars.mk | 34 +- dpdk/mk/toolchain/gcc/rte.toolchain-compat.mk | 48 +- dpdk/mk/toolchain/gcc/rte.vars.mk | 34 +- dpdk/mk/toolchain/icc/rte.toolchain-compat.mk | 38 +- dpdk/mk/toolchain/icc/rte.vars.mk | 34 +- dpdk/pkg/dpdk.spec | 32 +- dpdk/test/Makefile | 32 +- dpdk/test/bpf/dummy.c | 20 + dpdk/test/bpf/mbuf.h | 578 + dpdk/test/bpf/t1.c | 56 + dpdk/test/bpf/t2.c | 31 + dpdk/test/bpf/t3.c | 43 + dpdk/test/cmdline_test/Makefile | 32 +- dpdk/test/cmdline_test/cmdline_test.c | 33 +- dpdk/test/cmdline_test/cmdline_test.h | 33 +- dpdk/test/cmdline_test/cmdline_test.py | 33 +- dpdk/test/cmdline_test/cmdline_test_data.py | 32 +- dpdk/test/cmdline_test/commands.c | 33 +- dpdk/test/meson.build | 4 + dpdk/test/test-acl/Makefile | 32 +- dpdk/test/test-acl/main.c | 33 +- dpdk/test/test-pipeline/Makefile | 32 +- dpdk/test/test-pipeline/config.c | 33 +- dpdk/test/test-pipeline/init.c | 39 +- dpdk/test/test-pipeline/main.c | 33 +- dpdk/test/test-pipeline/main.h | 37 +- dpdk/test/test-pipeline/pipeline_acl.c | 33 +- dpdk/test/test-pipeline/pipeline_hash.c | 59 +- dpdk/test/test-pipeline/pipeline_lpm.c | 33 +- dpdk/test/test-pipeline/pipeline_lpm_ipv6.c | 33 +- dpdk/test/test-pipeline/pipeline_stub.c | 33 +- dpdk/test/test-pipeline/runtime.c | 33 +- dpdk/test/test/Makefile | 70 +- dpdk/test/test/autotest.py | 46 +- dpdk/test/test/autotest_data.py | 1087 +- dpdk/test/test/autotest_runner.py | 536 +- dpdk/test/test/autotest_test_funcs.py | 32 +- dpdk/test/test/commands.c | 57 +- dpdk/test/test/meson.build | 304 + dpdk/test/test/packet_burst_generator.c | 33 +- dpdk/test/test/packet_burst_generator.h | 33 +- dpdk/test/test/process.h | 62 +- dpdk/test/test/resource.c | 33 +- dpdk/test/test/resource.h | 33 +- dpdk/test/test/test.c | 116 +- dpdk/test/test/test.h | 121 +- dpdk/test/test/test_acl.c | 33 +- dpdk/test/test/test_acl.h | 51 +- dpdk/test/test/test_alarm.c | 38 +- dpdk/test/test/test_atomic.c | 33 +- dpdk/test/test/test_barrier.c | 286 + dpdk/test/test/test_bitmap.c | 50 +- dpdk/test/test/test_bpf.c | 2034 + dpdk/test/test/test_byteorder.c | 33 +- dpdk/test/test/test_cmdline.c | 33 +- dpdk/test/test/test_cmdline.h | 33 +- dpdk/test/test/test_cmdline_cirbuf.c | 35 +- dpdk/test/test/test_cmdline_etheraddr.c | 33 +- dpdk/test/test/test_cmdline_ipaddr.c | 35 +- dpdk/test/test/test_cmdline_lib.c | 33 +- dpdk/test/test/test_cmdline_num.c | 33 +- dpdk/test/test/test_cmdline_portlist.c | 33 +- dpdk/test/test/test_cmdline_string.c | 33 +- dpdk/test/test/test_common.c | 103 +- dpdk/test/test/test_compressdev.c | 1500 + dpdk/test/test/test_compressdev_test_buffer.h | 295 + dpdk/test/test/test_cpuflags.c | 33 +- dpdk/test/test/test_crc.c | 35 +- dpdk/test/test/test_cryptodev.c | 1338 +- dpdk/test/test/test_cryptodev.h | 39 +- .../test/test_cryptodev_aead_test_vectors.h | 32 +- .../test/test_cryptodev_aes_test_vectors.h | 253 +- dpdk/test/test/test_cryptodev_asym.c | 1372 + dpdk/test/test/test_cryptodev_asym_util.h | 42 + dpdk/test/test/test_cryptodev_blockcipher.c | 157 +- dpdk/test/test/test_cryptodev_blockcipher.h | 38 +- .../test/test_cryptodev_des_test_vectors.h | 140 +- .../test/test_cryptodev_dh_test_vectors.h | 80 + .../test/test_cryptodev_dsa_test_vectors.h | 117 + .../test/test_cryptodev_hash_test_vectors.h | 252 +- .../test/test_cryptodev_hmac_test_vectors.h | 32 +- .../test_cryptodev_kasumi_hash_test_vectors.h | 32 +- .../test/test_cryptodev_kasumi_test_vectors.h | 32 +- .../test/test_cryptodev_mod_test_vectors.h | 103 + .../test/test_cryptodev_rsa_test_vectors.h | 88 + .../test_cryptodev_snow3g_hash_test_vectors.h | 32 +- .../test/test_cryptodev_snow3g_test_vectors.h | 32 +- .../test/test_cryptodev_zuc_test_vectors.h | 32 +- dpdk/test/test/test_cycles.c | 86 +- dpdk/test/test/test_debug.c | 39 +- dpdk/test/test/test_devargs.c | 131 - dpdk/test/test/test_distributor.c | 33 +- dpdk/test/test/test_distributor_perf.c | 33 +- dpdk/test/test/test_eal_flags.c | 180 +- dpdk/test/test/test_eal_fs.c | 33 +- dpdk/test/test/test_efd.c | 33 +- dpdk/test/test/test_efd_perf.c | 33 +- dpdk/test/test/test_errno.c | 33 +- dpdk/test/test/test_event_crypto_adapter.c | 928 + dpdk/test/test/test_event_eth_rx_adapter.c | 372 +- dpdk/test/test/test_event_eth_tx_adapter.c | 699 + dpdk/test/test/test_event_ring.c | 33 +- dpdk/test/test/test_event_timer_adapter.c | 1830 + dpdk/test/test/test_eventdev.c | 84 +- dpdk/test/test/test_eventdev_octeontx.c | 1474 - dpdk/test/test/test_eventdev_sw.c | 3261 -- dpdk/test/test/test_external_mem.c | 390 + dpdk/test/test/test_fbarray.c | 576 + dpdk/test/test/test_flow_classify.c | 400 +- dpdk/test/test/test_flow_classify.h | 212 +- dpdk/test/test/test_func_reentrancy.c | 129 +- dpdk/test/test/test_hash.c | 366 +- dpdk/test/test/test_hash_functions.c | 33 +- dpdk/test/test/test_hash_multiwriter.c | 49 +- dpdk/test/test/test_hash_perf.c | 186 +- dpdk/test/test/test_hash_readwrite.c | 707 + dpdk/test/test/test_hash_readwrite_lf.c | 1220 + dpdk/test/test/test_hash_scaling.c | 220 - dpdk/test/test/test_interrupts.c | 72 +- dpdk/test/test/test_kni.c | 226 +- dpdk/test/test/test_kvargs.c | 52 +- dpdk/test/test/test_link_bonding.c | 69 +- dpdk/test/test/test_link_bonding_mode4.c | 44 +- dpdk/test/test/test_link_bonding_rssconf.c | 50 +- dpdk/test/test/test_logs.c | 87 +- dpdk/test/test/test_lpm.c | 33 +- dpdk/test/test/test_lpm6.c | 33 +- dpdk/test/test/test_lpm6_data.h | 33 +- dpdk/test/test/test_lpm6_perf.c | 33 +- dpdk/test/test/test_lpm_perf.c | 33 +- dpdk/test/test/test_malloc.c | 68 +- dpdk/test/test/test_mbuf.c | 33 +- dpdk/test/test/test_member.c | 33 +- dpdk/test/test/test_member_perf.c | 33 +- dpdk/test/test/test_memcpy.c | 33 +- dpdk/test/test/test_memcpy_perf.c | 33 +- dpdk/test/test/test_memory.c | 60 +- dpdk/test/test/test_mempool.c | 77 +- dpdk/test/test/test_mempool_perf.c | 49 +- dpdk/test/test/test_memzone.c | 500 +- dpdk/test/test/test_meter.c | 227 +- dpdk/test/test/test_metrics.c | 313 + dpdk/test/test/test_mp_secondary.c | 59 +- dpdk/test/test/test_per_lcore.c | 33 +- dpdk/test/test/test_pmd_perf.c | 81 +- dpdk/test/test/test_pmd_ring.c | 363 +- dpdk/test/test/test_pmd_ring_perf.c | 41 +- dpdk/test/test/test_power.c | 45 +- dpdk/test/test/test_power_acpi_cpufreq.c | 92 +- dpdk/test/test/test_power_kvm_vm.c | 65 +- dpdk/test/test/test_prefetch.c | 33 +- dpdk/test/test/test_rawdev.c | 27 + dpdk/test/test/test_reciprocal_division.c | 167 + .../test/test/test_reciprocal_division_perf.c | 201 + dpdk/test/test/test_red.c | 33 +- dpdk/test/test/test_reorder.c | 35 +- dpdk/test/test/test_resource.c | 33 +- dpdk/test/test/test_ring.c | 33 +- dpdk/test/test/test_ring_perf.c | 33 +- dpdk/test/test/test_rwlock.c | 33 +- dpdk/test/test/test_sched.c | 33 +- dpdk/test/test/test_service_cores.c | 236 +- dpdk/test/test/test_spinlock.c | 33 +- dpdk/test/test/test_string_fns.c | 33 +- dpdk/test/test/test_table.c | 44 +- dpdk/test/test/test_table.h | 39 +- dpdk/test/test/test_table_acl.c | 33 +- dpdk/test/test/test_table_acl.h | 33 +- dpdk/test/test/test_table_combined.c | 37 +- dpdk/test/test/test_table_combined.h | 33 +- dpdk/test/test/test_table_pipeline.c | 33 +- dpdk/test/test/test_table_pipeline.h | 33 +- dpdk/test/test/test_table_ports.c | 33 +- dpdk/test/test/test_table_ports.h | 33 +- dpdk/test/test/test_table_tables.c | 39 +- dpdk/test/test/test_table_tables.h | 33 +- dpdk/test/test/test_tailq.c | 33 +- dpdk/test/test/test_timer.c | 33 +- dpdk/test/test/test_timer_perf.c | 33 +- dpdk/test/test/test_timer_racecond.c | 2 +- dpdk/test/test/test_version.c | 33 +- dpdk/test/test/test_xmmt_ops.h | 33 +- dpdk/test/test/virtual_pmd.c | 46 +- dpdk/test/test/virtual_pmd.h | 33 +- dpdk/usertools/cpu_layout.py | 36 +- dpdk/usertools/dpdk-devbind.py | 119 +- dpdk/usertools/dpdk-pmdinfo.py | 2 + dpdk/usertools/dpdk-setup.sh | 33 +- dpdk/usertools/dpdk-telemetry-client.py | 116 + dpdk/usertools/meson.build | 4 + lib/ff_dpdk_if.c | 81 +- lib/ff_dpdk_kni.c | 13 +- 3509 files changed, 579725 insertions(+), 267772 deletions(-) delete mode 100644 dpdk/LICENSE.GPL delete mode 100644 dpdk/LICENSE.LGPL create mode 100644 dpdk/app/meson.build create mode 100644 dpdk/app/pdump/meson.build create mode 100644 dpdk/app/proc-info/Makefile create mode 100644 dpdk/app/proc-info/main.c create mode 100644 dpdk/app/proc-info/meson.build delete mode 100644 dpdk/app/proc_info/Makefile delete mode 100644 dpdk/app/proc_info/main.c create mode 100644 dpdk/app/test-bbdev/Makefile create mode 100644 dpdk/app/test-bbdev/main.c create mode 100644 dpdk/app/test-bbdev/main.h create mode 100644 dpdk/app/test-bbdev/meson.build create mode 100755 dpdk/app/test-bbdev/test-bbdev.py create mode 100644 dpdk/app/test-bbdev/test_bbdev.c create mode 100644 dpdk/app/test-bbdev/test_bbdev_perf.c create mode 100644 dpdk/app/test-bbdev/test_bbdev_vector.c create mode 100644 dpdk/app/test-bbdev/test_bbdev_vector.h create mode 100644 dpdk/app/test-bbdev/test_vectors/bbdev_null.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k40_r0_e17280_sbd_negllr.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e10376_crc24b_sbd_negllr_high_snr.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e10376_crc24b_sbd_negllr_low_snr.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_negllr.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_posllr.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_sbd_negllr.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_sbd_posllr.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_dec_c2_k3136_r0_e4920_sbd_negllr.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_dec_c2_k3136_r0_e4920_sbd_negllr_crc24b.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1190_rm.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1194_rm.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1196_rm.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e272_rm.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e120_rm_rvidx.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e18444.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e18448_crc24a.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e32256_crc24b_rm.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_enc_c2_k5952_r0_e17868_crc24b.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_enc_c3_k4800_r2_e14412_crc24b.data create mode 100644 dpdk/app/test-bbdev/test_vectors/turbo_enc_c4_k4800_r2_e14412_crc24b.data create mode 120000 dpdk/app/test-bbdev/turbo_dec_default.data create mode 120000 dpdk/app/test-bbdev/turbo_enc_default.data create mode 100644 dpdk/app/test-crypto-perf/meson.build create mode 100644 dpdk/app/test-eventdev/meson.build create mode 100644 dpdk/app/test-eventdev/test_pipeline_atq.c create mode 100644 dpdk/app/test-eventdev/test_pipeline_common.c create mode 100644 dpdk/app/test-eventdev/test_pipeline_common.h create mode 100644 dpdk/app/test-eventdev/test_pipeline_queue.c create mode 100644 dpdk/app/test-pmd/bpf_cmd.c create mode 100644 dpdk/app/test-pmd/bpf_cmd.h create mode 100644 dpdk/app/test-pmd/meson.build create mode 100644 dpdk/app/test-pmd/noisy_vnf.c create mode 100644 dpdk/app/test-pmd/softnicfwd.c delete mode 100644 dpdk/app/test-pmd/tm.c create mode 100644 dpdk/app/test-pmd/util.c create mode 100755 dpdk/buildtools/check-experimental-syms.sh create mode 100755 dpdk/buildtools/gen-pmdinfo-cfile.sh create mode 100644 dpdk/buildtools/meson.build create mode 100644 dpdk/buildtools/pmdinfogen/meson.build create mode 100644 dpdk/buildtools/symlink-drivers-solibs.sh create mode 100644 dpdk/config/arm/arm64_armv8_linuxapp_gcc create mode 100644 dpdk/config/arm/arm64_dpaa2_linuxapp_gcc create mode 100644 dpdk/config/arm/arm64_dpaa_linuxapp_gcc create mode 100644 dpdk/config/arm/arm64_thunderx_linuxapp_gcc create mode 100755 dpdk/config/arm/armv8_machine.py create mode 100644 dpdk/config/arm/meson.build create mode 100644 dpdk/config/defconfig_arm64-stingray-linuxapp-gcc create mode 100644 dpdk/config/meson.build create mode 100644 dpdk/config/ppc_64/meson.build create mode 100644 dpdk/config/rte_config.h create mode 100644 dpdk/config/x86/meson.build create mode 100755 dpdk/devtools/check-forbidden-tokens.awk create mode 100755 dpdk/devtools/check-symbol-change.sh create mode 100755 dpdk/devtools/check-symbol-maps.sh create mode 100644 dpdk/devtools/cocci/strlcpy.cocci create mode 100755 dpdk/devtools/test-meson-builds.sh delete mode 100644 dpdk/doc/api/doxy-api.conf create mode 100644 dpdk/doc/api/doxy-api.conf.in create mode 100755 dpdk/doc/api/generate_doxygen.sh create mode 100755 dpdk/doc/api/generate_examples.sh create mode 100644 dpdk/doc/api/meson.build create mode 100644 dpdk/doc/build-sdk-meson.txt create mode 100644 dpdk/doc/guides/bbdevs/index.rst create mode 100644 dpdk/doc/guides/bbdevs/null.rst create mode 100644 dpdk/doc/guides/bbdevs/turbo_sw.rst create mode 100644 dpdk/doc/guides/compressdevs/features/default.ini create mode 100644 dpdk/doc/guides/compressdevs/features/isal.ini create mode 100644 dpdk/doc/guides/compressdevs/features/octeontx.ini create mode 100644 dpdk/doc/guides/compressdevs/features/qat.ini create mode 100644 dpdk/doc/guides/compressdevs/features/zlib.ini create mode 100644 dpdk/doc/guides/compressdevs/index.rst create mode 100644 dpdk/doc/guides/compressdevs/isal.rst create mode 100644 dpdk/doc/guides/compressdevs/octeontx.rst create mode 100644 dpdk/doc/guides/compressdevs/overview.rst create mode 100644 dpdk/doc/guides/compressdevs/qat_comp.rst create mode 100644 dpdk/doc/guides/compressdevs/zlib.rst create mode 100644 dpdk/doc/guides/cryptodevs/caam_jr.rst create mode 100644 dpdk/doc/guides/cryptodevs/ccp.rst create mode 100644 dpdk/doc/guides/cryptodevs/features/caam_jr.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/ccp.ini delete mode 100644 dpdk/doc/guides/cryptodevs/features/mrvl.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/mvsam.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/octeontx.ini create mode 100644 dpdk/doc/guides/cryptodevs/features/virtio.ini delete mode 100644 dpdk/doc/guides/cryptodevs/mrvl.rst create mode 100644 dpdk/doc/guides/cryptodevs/mvsam.rst create mode 100644 dpdk/doc/guides/cryptodevs/octeontx.rst create mode 100644 dpdk/doc/guides/cryptodevs/virtio.rst create mode 100644 dpdk/doc/guides/custom.css create mode 100644 dpdk/doc/guides/eventdevs/dpaa.rst create mode 100644 dpdk/doc/guides/eventdevs/dsw.rst create mode 100644 dpdk/doc/guides/eventdevs/opdl.rst create mode 100644 dpdk/doc/guides/freebsd_gsg/freebsd_eal_parameters.rst create mode 100644 dpdk/doc/guides/howto/telemetry.rst create mode 100644 dpdk/doc/guides/linux_gsg/cross_build_dpdk_for_arm64.rst create mode 100644 dpdk/doc/guides/linux_gsg/eal_args.include.rst create mode 100644 dpdk/doc/guides/linux_gsg/linux_eal_parameters.rst create mode 100644 dpdk/doc/guides/meson.build create mode 100644 dpdk/doc/guides/nics/atlantic.rst create mode 100644 dpdk/doc/guides/nics/axgbe.rst create mode 100644 dpdk/doc/guides/nics/enetc.rst create mode 100644 dpdk/doc/guides/nics/features/atlantic.ini create mode 100644 dpdk/doc/guides/nics/features/avf.ini create mode 100644 dpdk/doc/guides/nics/features/avf_vec.ini create mode 100644 dpdk/doc/guides/nics/features/axgbe.ini create mode 100644 dpdk/doc/guides/nics/features/cxgbevf.ini create mode 100644 dpdk/doc/guides/nics/features/enetc.ini create mode 100644 dpdk/doc/guides/nics/features/ifcvf.ini delete mode 100644 dpdk/doc/guides/nics/features/mrvl.ini create mode 100644 dpdk/doc/guides/nics/features/mvneta.ini create mode 100644 dpdk/doc/guides/nics/features/mvpp2.ini create mode 100644 dpdk/doc/guides/nics/features/netvsc.ini rename dpdk/doc/guides/nics/features/{nfp_pf.ini => nfp.ini} (100%) create mode 100644 dpdk/doc/guides/nics/features/softnic.ini create mode 100644 dpdk/doc/guides/nics/features/vdev_netvsc.ini create mode 100644 dpdk/doc/guides/nics/ifc.rst create mode 100644 dpdk/doc/guides/nics/igb.rst create mode 100644 dpdk/doc/guides/nics/img/mvpp2_tm.svg create mode 100644 dpdk/doc/guides/nics/img/szedata2_nfb200g_architecture.svg delete mode 100644 dpdk/doc/guides/nics/mrvl.rst create mode 100644 dpdk/doc/guides/nics/mvneta.rst create mode 100644 dpdk/doc/guides/nics/mvpp2.rst create mode 100644 dpdk/doc/guides/nics/netvsc.rst create mode 100644 dpdk/doc/guides/nics/softnic.rst create mode 100644 dpdk/doc/guides/nics/vdev_netvsc.rst create mode 100644 dpdk/doc/guides/platform/dpaa.rst create mode 100644 dpdk/doc/guides/platform/dpaa2.rst create mode 100644 dpdk/doc/guides/prog_guide/bbdev.rst create mode 100644 dpdk/doc/guides/prog_guide/bpf_lib.rst create mode 100644 dpdk/doc/guides/prog_guide/compressdev.rst create mode 100644 dpdk/doc/guides/prog_guide/event_crypto_adapter.rst create mode 100644 dpdk/doc/guides/prog_guide/event_ethernet_tx_adapter.rst create mode 100644 dpdk/doc/guides/prog_guide/event_timer_adapter.rst create mode 100644 dpdk/doc/guides/prog_guide/img/event_crypto_adapter_op_forward.svg create mode 100644 dpdk/doc/guides/prog_guide/img/event_crypto_adapter_op_new.svg create mode 100644 dpdk/doc/guides/prog_guide/img/gro-key-algorithm.svg create mode 100644 dpdk/doc/guides/prog_guide/img/stateful-op.svg create mode 100644 dpdk/doc/guides/prog_guide/img/stateless-op-shared.svg create mode 100644 dpdk/doc/guides/prog_guide/img/stateless-op.svg create mode 100644 dpdk/doc/guides/prog_guide/img/turbo_tb_decode.svg create mode 100644 dpdk/doc/guides/prog_guide/img/turbo_tb_encode.svg delete mode 100644 dpdk/doc/guides/prog_guide/port_hotplug_framework.rst create mode 100644 dpdk/doc/guides/prog_guide/rawdev.rst create mode 100644 dpdk/doc/guides/prog_guide/switch_representation.rst create mode 100644 dpdk/doc/guides/rawdevs/dpaa2_cmdif.rst create mode 100644 dpdk/doc/guides/rawdevs/dpaa2_qdma.rst create mode 100644 dpdk/doc/guides/rawdevs/ifpga_rawdev.rst create mode 100644 dpdk/doc/guides/rawdevs/index.rst delete mode 100644 dpdk/doc/guides/rel_notes/rel_description.rst create mode 100644 dpdk/doc/guides/rel_notes/release_18_02.rst create mode 100644 dpdk/doc/guides/rel_notes/release_18_05.rst create mode 100644 dpdk/doc/guides/rel_notes/release_18_08.rst create mode 100644 dpdk/doc/guides/rel_notes/release_18_11.rst create mode 100644 dpdk/doc/guides/sample_app_ug/bbdev_app.rst create mode 100644 dpdk/doc/guides/sample_app_ug/eventdev_pipeline.rst delete mode 100644 dpdk/doc/guides/sample_app_ug/eventdev_pipeline_sw_pmd.rst create mode 100644 dpdk/doc/guides/sample_app_ug/fips_validation.rst delete mode 100644 dpdk/doc/guides/sample_app_ug/img/ip_pipelines_1.svg delete mode 100644 dpdk/doc/guides/sample_app_ug/img/ip_pipelines_2.svg delete mode 100644 dpdk/doc/guides/sample_app_ug/img/ip_pipelines_3.svg delete mode 100644 dpdk/doc/guides/sample_app_ug/img/master_slave_proc.png delete mode 100644 dpdk/doc/guides/sample_app_ug/img/slave_proc_recov.png create mode 100644 dpdk/doc/guides/sample_app_ug/vdpa.rst create mode 100644 dpdk/doc/guides/sample_app_ug/vhost_crypto.rst create mode 100644 dpdk/doc/guides/tools/img/eventdev_pipeline_atq_test_generic.svg create mode 100644 dpdk/doc/guides/tools/img/eventdev_pipeline_atq_test_internal_port.svg create mode 100644 dpdk/doc/guides/tools/img/eventdev_pipeline_queue_test_generic.svg create mode 100644 dpdk/doc/guides/tools/img/eventdev_pipeline_queue_test_internal_port.svg create mode 100644 dpdk/doc/guides/tools/testbbdev.rst create mode 100644 dpdk/doc/meson.build create mode 100644 dpdk/drivers/baseband/Makefile create mode 100644 dpdk/drivers/baseband/meson.build create mode 100644 dpdk/drivers/baseband/null/Makefile create mode 100644 dpdk/drivers/baseband/null/bbdev_null.c create mode 100644 dpdk/drivers/baseband/null/meson.build create mode 100644 dpdk/drivers/baseband/null/rte_pmd_bbdev_null_version.map create mode 100644 dpdk/drivers/baseband/turbo_sw/Makefile create mode 100644 dpdk/drivers/baseband/turbo_sw/bbdev_turbo_software.c create mode 100644 dpdk/drivers/baseband/turbo_sw/rte_pmd_bbdev_turbo_sw_version.map create mode 100644 dpdk/drivers/bus/dpaa/meson.build create mode 100644 dpdk/drivers/bus/fslmc/mc/dpdmai.c create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpdmai.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpdmai_cmd.h create mode 100644 dpdk/drivers/bus/fslmc/mc/fsl_dpopr.h create mode 100644 dpdk/drivers/bus/fslmc/meson.build create mode 100644 dpdk/drivers/bus/fslmc/qbman/include/fsl_qbman_debug.h create mode 100644 dpdk/drivers/bus/fslmc/qbman/qbman_debug.c create mode 100644 dpdk/drivers/bus/ifpga/Makefile create mode 100644 dpdk/drivers/bus/ifpga/ifpga_bus.c create mode 100644 dpdk/drivers/bus/ifpga/ifpga_common.c create mode 100644 dpdk/drivers/bus/ifpga/ifpga_common.h create mode 100644 dpdk/drivers/bus/ifpga/ifpga_logs.h create mode 100644 dpdk/drivers/bus/ifpga/meson.build create mode 100644 dpdk/drivers/bus/ifpga/rte_bus_ifpga.h create mode 100644 dpdk/drivers/bus/ifpga/rte_bus_ifpga_version.map create mode 100644 dpdk/drivers/bus/meson.build create mode 100644 dpdk/drivers/bus/pci/meson.build create mode 100644 dpdk/drivers/bus/pci/pci_params.c create mode 100644 dpdk/drivers/bus/vdev/meson.build create mode 100644 dpdk/drivers/bus/vdev/vdev_params.c create mode 100644 dpdk/drivers/bus/vdev/vdev_private.h create mode 100644 dpdk/drivers/bus/vmbus/Makefile create mode 100644 dpdk/drivers/bus/vmbus/linux/Makefile create mode 100644 dpdk/drivers/bus/vmbus/linux/vmbus_bus.c create mode 100644 dpdk/drivers/bus/vmbus/linux/vmbus_uio.c create mode 100644 dpdk/drivers/bus/vmbus/meson.build create mode 100644 dpdk/drivers/bus/vmbus/private.h create mode 100644 dpdk/drivers/bus/vmbus/rte_bus_vmbus.h create mode 100644 dpdk/drivers/bus/vmbus/rte_bus_vmbus_version.map create mode 100644 dpdk/drivers/bus/vmbus/rte_vmbus_reg.h create mode 100644 dpdk/drivers/bus/vmbus/vmbus_bufring.c create mode 100644 dpdk/drivers/bus/vmbus/vmbus_channel.c create mode 100644 dpdk/drivers/bus/vmbus/vmbus_common.c create mode 100644 dpdk/drivers/bus/vmbus/vmbus_common_uio.c create mode 100644 dpdk/drivers/common/Makefile create mode 100644 dpdk/drivers/common/cpt/Makefile create mode 100644 dpdk/drivers/common/cpt/cpt_common.h create mode 100644 dpdk/drivers/common/cpt/cpt_hw_types.h create mode 100644 dpdk/drivers/common/cpt/cpt_mcode_defines.h create mode 100644 dpdk/drivers/common/cpt/cpt_pmd_logs.h create mode 100644 dpdk/drivers/common/cpt/cpt_pmd_ops_helper.c create mode 100644 dpdk/drivers/common/cpt/cpt_pmd_ops_helper.h create mode 100644 dpdk/drivers/common/cpt/cpt_request_mgr.h create mode 100644 dpdk/drivers/common/cpt/cpt_ucode.h create mode 100644 dpdk/drivers/common/cpt/meson.build create mode 100644 dpdk/drivers/common/cpt/rte_common_cpt_version.map create mode 100644 dpdk/drivers/common/dpaax/Makefile create mode 100644 dpdk/drivers/common/dpaax/dpaax_iova_table.c create mode 100644 dpdk/drivers/common/dpaax/dpaax_iova_table.h create mode 100644 dpdk/drivers/common/dpaax/dpaax_logs.h create mode 100644 dpdk/drivers/common/dpaax/meson.build create mode 100644 dpdk/drivers/common/dpaax/rte_common_dpaax_version.map create mode 100644 dpdk/drivers/common/meson.build create mode 100644 dpdk/drivers/common/mvep/Makefile create mode 100644 dpdk/drivers/common/mvep/meson.build create mode 100644 dpdk/drivers/common/mvep/mvep_common.c create mode 100644 dpdk/drivers/common/mvep/rte_common_mvep_version.map create mode 100644 dpdk/drivers/common/mvep/rte_mvep_common.h create mode 100644 dpdk/drivers/common/octeontx/Makefile create mode 100644 dpdk/drivers/common/octeontx/meson.build create mode 100644 dpdk/drivers/common/octeontx/octeontx_mbox.c create mode 100644 dpdk/drivers/common/octeontx/octeontx_mbox.h create mode 100644 dpdk/drivers/common/octeontx/rte_common_octeontx_version.map create mode 100644 dpdk/drivers/common/qat/Makefile create mode 100644 dpdk/drivers/common/qat/meson.build create mode 100644 dpdk/drivers/common/qat/qat_adf/adf_transport_access_macros.h create mode 100644 dpdk/drivers/common/qat/qat_adf/icp_qat_fw.h create mode 100644 dpdk/drivers/common/qat/qat_adf/icp_qat_fw_comp.h create mode 100644 dpdk/drivers/common/qat/qat_adf/icp_qat_fw_la.h create mode 100644 dpdk/drivers/common/qat/qat_adf/icp_qat_hw.h create mode 100644 dpdk/drivers/common/qat/qat_common.c create mode 100644 dpdk/drivers/common/qat/qat_common.h create mode 100644 dpdk/drivers/common/qat/qat_device.c create mode 100644 dpdk/drivers/common/qat/qat_device.h create mode 100644 dpdk/drivers/common/qat/qat_logs.c create mode 100644 dpdk/drivers/common/qat/qat_logs.h create mode 100644 dpdk/drivers/common/qat/qat_qp.c create mode 100644 dpdk/drivers/common/qat/qat_qp.h create mode 100644 dpdk/drivers/compress/Makefile create mode 100644 dpdk/drivers/compress/isal/Makefile create mode 100644 dpdk/drivers/compress/isal/isal_compress_pmd.c create mode 100644 dpdk/drivers/compress/isal/isal_compress_pmd_ops.c create mode 100644 dpdk/drivers/compress/isal/isal_compress_pmd_private.h create mode 100644 dpdk/drivers/compress/isal/meson.build create mode 100644 dpdk/drivers/compress/isal/rte_pmd_isal_version.map create mode 100644 dpdk/drivers/compress/meson.build create mode 100644 dpdk/drivers/compress/octeontx/Makefile create mode 100644 dpdk/drivers/compress/octeontx/include/zip_regs.h create mode 100644 dpdk/drivers/compress/octeontx/meson.build create mode 100644 dpdk/drivers/compress/octeontx/otx_zip.c create mode 100644 dpdk/drivers/compress/octeontx/otx_zip.h create mode 100644 dpdk/drivers/compress/octeontx/otx_zip_pmd.c create mode 100644 dpdk/drivers/compress/octeontx/rte_pmd_octeontx_compress_version.map create mode 100644 dpdk/drivers/compress/qat/meson.build create mode 100644 dpdk/drivers/compress/qat/qat_comp.c create mode 100644 dpdk/drivers/compress/qat/qat_comp.h create mode 100644 dpdk/drivers/compress/qat/qat_comp_pmd.c create mode 100644 dpdk/drivers/compress/qat/qat_comp_pmd.h create mode 100644 dpdk/drivers/compress/qat/rte_pmd_qat_version.map create mode 100644 dpdk/drivers/compress/zlib/Makefile create mode 100644 dpdk/drivers/compress/zlib/meson.build create mode 100644 dpdk/drivers/compress/zlib/rte_pmd_zlib_version.map create mode 100644 dpdk/drivers/compress/zlib/zlib_pmd.c create mode 100644 dpdk/drivers/compress/zlib/zlib_pmd_ops.c create mode 100644 dpdk/drivers/compress/zlib/zlib_pmd_private.h create mode 100644 dpdk/drivers/crypto/aesni_gcm/meson.build create mode 100644 dpdk/drivers/crypto/aesni_mb/meson.build create mode 100644 dpdk/drivers/crypto/caam_jr/Makefile create mode 100644 dpdk/drivers/crypto/caam_jr/caam_jr.c create mode 100644 dpdk/drivers/crypto/caam_jr/caam_jr_capabilities.c create mode 100644 dpdk/drivers/crypto/caam_jr/caam_jr_capabilities.h create mode 100644 dpdk/drivers/crypto/caam_jr/caam_jr_config.h create mode 100644 dpdk/drivers/crypto/caam_jr/caam_jr_desc.h create mode 100644 dpdk/drivers/crypto/caam_jr/caam_jr_hw.c create mode 100644 dpdk/drivers/crypto/caam_jr/caam_jr_hw_specific.h create mode 100644 dpdk/drivers/crypto/caam_jr/caam_jr_log.h create mode 100644 dpdk/drivers/crypto/caam_jr/caam_jr_pvt.h create mode 100644 dpdk/drivers/crypto/caam_jr/caam_jr_uio.c create mode 100644 dpdk/drivers/crypto/caam_jr/meson.build create mode 100644 dpdk/drivers/crypto/caam_jr/rte_pmd_caam_jr_version.map create mode 100644 dpdk/drivers/crypto/ccp/Makefile create mode 100644 dpdk/drivers/crypto/ccp/ccp_crypto.c create mode 100644 dpdk/drivers/crypto/ccp/ccp_crypto.h create mode 100644 dpdk/drivers/crypto/ccp/ccp_dev.c create mode 100644 dpdk/drivers/crypto/ccp/ccp_dev.h create mode 100644 dpdk/drivers/crypto/ccp/ccp_pci.c create mode 100644 dpdk/drivers/crypto/ccp/ccp_pci.h create mode 100644 dpdk/drivers/crypto/ccp/ccp_pmd_ops.c create mode 100644 dpdk/drivers/crypto/ccp/ccp_pmd_private.h create mode 100644 dpdk/drivers/crypto/ccp/meson.build create mode 100644 dpdk/drivers/crypto/ccp/rte_ccp_pmd.c create mode 100644 dpdk/drivers/crypto/ccp/rte_pmd_ccp_version.map create mode 100644 dpdk/drivers/crypto/dpaa2_sec/dpaa2_sec_event.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/hw/desc/pdcp.h create mode 100644 dpdk/drivers/crypto/dpaa2_sec/meson.build create mode 100644 dpdk/drivers/crypto/dpaa_sec/meson.build create mode 100644 dpdk/drivers/crypto/kasumi/meson.build create mode 100644 dpdk/drivers/crypto/meson.build delete mode 100644 dpdk/drivers/crypto/mrvl/Makefile delete mode 100644 dpdk/drivers/crypto/mrvl/rte_mrvl_compat.h delete mode 100644 dpdk/drivers/crypto/mrvl/rte_mrvl_pmd.c delete mode 100644 dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_ops.c delete mode 100644 dpdk/drivers/crypto/mrvl/rte_mrvl_pmd_private.h create mode 100644 dpdk/drivers/crypto/mvsam/Makefile create mode 100644 dpdk/drivers/crypto/mvsam/meson.build create mode 100644 dpdk/drivers/crypto/mvsam/rte_mrvl_compat.h create mode 100644 dpdk/drivers/crypto/mvsam/rte_mrvl_pmd.c create mode 100644 dpdk/drivers/crypto/mvsam/rte_mrvl_pmd_ops.c create mode 100644 dpdk/drivers/crypto/mvsam/rte_mrvl_pmd_private.h rename dpdk/drivers/crypto/{mrvl/rte_pmd_mrvl_version.map => mvsam/rte_pmd_mvsam_version.map} (100%) create mode 100644 dpdk/drivers/crypto/null/meson.build create mode 100644 dpdk/drivers/crypto/octeontx/Makefile create mode 100644 dpdk/drivers/crypto/octeontx/meson.build create mode 100644 dpdk/drivers/crypto/octeontx/otx_cryptodev.c create mode 100644 dpdk/drivers/crypto/octeontx/otx_cryptodev.h create mode 100644 dpdk/drivers/crypto/octeontx/otx_cryptodev_capabilities.c create mode 100644 dpdk/drivers/crypto/octeontx/otx_cryptodev_capabilities.h create mode 100644 dpdk/drivers/crypto/octeontx/otx_cryptodev_hw_access.c create mode 100644 dpdk/drivers/crypto/octeontx/otx_cryptodev_hw_access.h create mode 100644 dpdk/drivers/crypto/octeontx/otx_cryptodev_mbox.c create mode 100644 dpdk/drivers/crypto/octeontx/otx_cryptodev_mbox.h create mode 100644 dpdk/drivers/crypto/octeontx/otx_cryptodev_ops.c create mode 100644 dpdk/drivers/crypto/octeontx/otx_cryptodev_ops.h create mode 100644 dpdk/drivers/crypto/octeontx/rte_pmd_octeontx_crypto_version.map create mode 100644 dpdk/drivers/crypto/openssl/compat.h create mode 100644 dpdk/drivers/crypto/openssl/meson.build delete mode 100644 dpdk/drivers/crypto/qat/Makefile create mode 100644 dpdk/drivers/crypto/qat/README create mode 100644 dpdk/drivers/crypto/qat/meson.build delete mode 100644 dpdk/drivers/crypto/qat/qat_adf/adf_transport_access_macros.h delete mode 100644 dpdk/drivers/crypto/qat/qat_adf/icp_qat_fw.h delete mode 100644 dpdk/drivers/crypto/qat/qat_adf/icp_qat_fw_la.h delete mode 100644 dpdk/drivers/crypto/qat/qat_adf/icp_qat_hw.h delete mode 100644 dpdk/drivers/crypto/qat/qat_adf/qat_algs.h delete mode 100644 dpdk/drivers/crypto/qat/qat_adf/qat_algs_build_desc.c delete mode 100644 dpdk/drivers/crypto/qat/qat_crypto.c delete mode 100644 dpdk/drivers/crypto/qat/qat_crypto.h delete mode 100644 dpdk/drivers/crypto/qat/qat_crypto_capabilities.h delete mode 100644 dpdk/drivers/crypto/qat/qat_logs.h delete mode 100644 dpdk/drivers/crypto/qat/qat_qp.c create mode 100644 dpdk/drivers/crypto/qat/qat_sym.c create mode 100644 dpdk/drivers/crypto/qat/qat_sym.h create mode 100644 dpdk/drivers/crypto/qat/qat_sym_capabilities.h create mode 100644 dpdk/drivers/crypto/qat/qat_sym_pmd.c create mode 100644 dpdk/drivers/crypto/qat/qat_sym_pmd.h create mode 100644 dpdk/drivers/crypto/qat/qat_sym_session.c create mode 100644 dpdk/drivers/crypto/qat/qat_sym_session.h delete mode 100644 dpdk/drivers/crypto/qat/rte_pmd_qat_version.map delete mode 100644 dpdk/drivers/crypto/qat/rte_qat_cryptodev.c create mode 100644 dpdk/drivers/crypto/scheduler/meson.build create mode 100644 dpdk/drivers/crypto/virtio/Makefile create mode 100644 dpdk/drivers/crypto/virtio/meson.build create mode 100644 dpdk/drivers/crypto/virtio/rte_pmd_virtio_crypto_version.map create mode 100644 dpdk/drivers/crypto/virtio/virtio_crypto_algs.h create mode 100644 dpdk/drivers/crypto/virtio/virtio_crypto_capabilities.h create mode 100644 dpdk/drivers/crypto/virtio/virtio_cryptodev.c create mode 100644 dpdk/drivers/crypto/virtio/virtio_cryptodev.h create mode 100644 dpdk/drivers/crypto/virtio/virtio_logs.h create mode 100644 dpdk/drivers/crypto/virtio/virtio_pci.c create mode 100644 dpdk/drivers/crypto/virtio/virtio_pci.h create mode 100644 dpdk/drivers/crypto/virtio/virtio_ring.h create mode 100644 dpdk/drivers/crypto/virtio/virtio_rxtx.c create mode 100644 dpdk/drivers/crypto/virtio/virtqueue.c create mode 100644 dpdk/drivers/crypto/virtio/virtqueue.h create mode 100644 dpdk/drivers/crypto/zuc/meson.build create mode 100644 dpdk/drivers/event/dpaa/Makefile create mode 100644 dpdk/drivers/event/dpaa/dpaa_eventdev.c create mode 100644 dpdk/drivers/event/dpaa/dpaa_eventdev.h create mode 100644 dpdk/drivers/event/dpaa/meson.build create mode 100644 dpdk/drivers/event/dpaa/rte_pmd_dpaa_event_version.map create mode 100644 dpdk/drivers/event/dpaa2/dpaa2_eventdev_logs.h create mode 100644 dpdk/drivers/event/dpaa2/meson.build create mode 100644 dpdk/drivers/event/dsw/Makefile create mode 100644 dpdk/drivers/event/dsw/dsw_evdev.c create mode 100644 dpdk/drivers/event/dsw/dsw_evdev.h create mode 100644 dpdk/drivers/event/dsw/dsw_event.c create mode 100644 dpdk/drivers/event/dsw/dsw_sort.h create mode 100644 dpdk/drivers/event/dsw/dsw_xstats.c create mode 100644 dpdk/drivers/event/dsw/meson.build create mode 100644 dpdk/drivers/event/dsw/rte_pmd_dsw_event_version.map create mode 100644 dpdk/drivers/event/meson.build create mode 100644 dpdk/drivers/event/octeontx/meson.build rename dpdk/drivers/event/octeontx/{rte_pmd_octeontx_ssovf_version.map => rte_pmd_octeontx_event_version.map} (100%) create mode 100644 dpdk/drivers/event/octeontx/ssovf_evdev_selftest.c create mode 100644 dpdk/drivers/event/octeontx/ssovf_probe.c create mode 100644 dpdk/drivers/event/octeontx/timvf_evdev.c create mode 100644 dpdk/drivers/event/octeontx/timvf_evdev.h create mode 100644 dpdk/drivers/event/octeontx/timvf_probe.c create mode 100644 dpdk/drivers/event/octeontx/timvf_worker.c create mode 100644 dpdk/drivers/event/octeontx/timvf_worker.h create mode 100644 dpdk/drivers/event/opdl/Makefile create mode 100644 dpdk/drivers/event/opdl/meson.build create mode 100644 dpdk/drivers/event/opdl/opdl_evdev.c create mode 100644 dpdk/drivers/event/opdl/opdl_evdev.h create mode 100644 dpdk/drivers/event/opdl/opdl_evdev_init.c create mode 100644 dpdk/drivers/event/opdl/opdl_evdev_xstats.c create mode 100644 dpdk/drivers/event/opdl/opdl_log.h create mode 100644 dpdk/drivers/event/opdl/opdl_ring.c create mode 100644 dpdk/drivers/event/opdl/opdl_ring.h create mode 100644 dpdk/drivers/event/opdl/opdl_test.c create mode 100644 dpdk/drivers/event/opdl/rte_pmd_opdl_event_version.map create mode 100644 dpdk/drivers/event/skeleton/meson.build create mode 100644 dpdk/drivers/event/sw/iq_chunk.h delete mode 100644 dpdk/drivers/event/sw/iq_ring.h create mode 100644 dpdk/drivers/event/sw/meson.build create mode 100644 dpdk/drivers/event/sw/sw_evdev_log.h create mode 100644 dpdk/drivers/event/sw/sw_evdev_selftest.c create mode 100644 dpdk/drivers/mempool/bucket/Makefile create mode 100644 dpdk/drivers/mempool/bucket/meson.build create mode 100644 dpdk/drivers/mempool/bucket/rte_mempool_bucket.c create mode 100644 dpdk/drivers/mempool/bucket/rte_mempool_bucket_version.map create mode 100644 dpdk/drivers/mempool/dpaa/meson.build create mode 100644 dpdk/drivers/mempool/dpaa2/dpaa2_hw_mempool_logs.h create mode 100644 dpdk/drivers/mempool/dpaa2/meson.build create mode 100644 dpdk/drivers/mempool/dpaa2/rte_dpaa2_mempool.h create mode 100644 dpdk/drivers/mempool/meson.build create mode 100644 dpdk/drivers/mempool/octeontx/meson.build delete mode 100644 dpdk/drivers/mempool/octeontx/octeontx_mbox.c delete mode 100644 dpdk/drivers/mempool/octeontx/octeontx_mbox.h delete mode 100644 dpdk/drivers/mempool/octeontx/octeontx_ssovf.c create mode 100644 dpdk/drivers/mempool/ring/meson.build create mode 100644 dpdk/drivers/mempool/stack/meson.build create mode 100644 dpdk/drivers/meson.build create mode 100644 dpdk/drivers/net/af_packet/meson.build create mode 100644 dpdk/drivers/net/ark/meson.build create mode 100644 dpdk/drivers/net/atlantic/Makefile create mode 100644 dpdk/drivers/net/atlantic/atl_common.h create mode 100644 dpdk/drivers/net/atlantic/atl_ethdev.c create mode 100644 dpdk/drivers/net/atlantic/atl_ethdev.h create mode 100644 dpdk/drivers/net/atlantic/atl_hw_regs.c create mode 100644 dpdk/drivers/net/atlantic/atl_hw_regs.h create mode 100644 dpdk/drivers/net/atlantic/atl_logs.h create mode 100644 dpdk/drivers/net/atlantic/atl_rxtx.c create mode 100644 dpdk/drivers/net/atlantic/atl_types.h create mode 100644 dpdk/drivers/net/atlantic/hw_atl/hw_atl_b0.c create mode 100644 dpdk/drivers/net/atlantic/hw_atl/hw_atl_b0.h create mode 100644 dpdk/drivers/net/atlantic/hw_atl/hw_atl_b0_internal.h create mode 100644 dpdk/drivers/net/atlantic/hw_atl/hw_atl_llh.c create mode 100644 dpdk/drivers/net/atlantic/hw_atl/hw_atl_llh.h create mode 100644 dpdk/drivers/net/atlantic/hw_atl/hw_atl_llh_internal.h create mode 100644 dpdk/drivers/net/atlantic/hw_atl/hw_atl_utils.c create mode 100644 dpdk/drivers/net/atlantic/hw_atl/hw_atl_utils.h create mode 100644 dpdk/drivers/net/atlantic/hw_atl/hw_atl_utils_fw2x.c create mode 100644 dpdk/drivers/net/atlantic/meson.build create mode 100644 dpdk/drivers/net/atlantic/rte_pmd_atlantic_version.map create mode 100644 dpdk/drivers/net/avf/Makefile create mode 100644 dpdk/drivers/net/avf/avf.h create mode 100644 dpdk/drivers/net/avf/avf_ethdev.c create mode 100644 dpdk/drivers/net/avf/avf_log.h create mode 100644 dpdk/drivers/net/avf/avf_rxtx.c create mode 100644 dpdk/drivers/net/avf/avf_rxtx.h create mode 100644 dpdk/drivers/net/avf/avf_rxtx_vec_common.h create mode 100644 dpdk/drivers/net/avf/avf_rxtx_vec_sse.c create mode 100644 dpdk/drivers/net/avf/avf_vchnl.c create mode 100644 dpdk/drivers/net/avf/base/README create mode 100644 dpdk/drivers/net/avf/base/avf_adminq.c create mode 100644 dpdk/drivers/net/avf/base/avf_adminq.h create mode 100644 dpdk/drivers/net/avf/base/avf_adminq_cmd.h create mode 100644 dpdk/drivers/net/avf/base/avf_alloc.h create mode 100644 dpdk/drivers/net/avf/base/avf_common.c create mode 100644 dpdk/drivers/net/avf/base/avf_devids.h create mode 100644 dpdk/drivers/net/avf/base/avf_hmc.h create mode 100644 dpdk/drivers/net/avf/base/avf_lan_hmc.h create mode 100644 dpdk/drivers/net/avf/base/avf_osdep.h create mode 100644 dpdk/drivers/net/avf/base/avf_prototype.h create mode 100644 dpdk/drivers/net/avf/base/avf_register.h create mode 100644 dpdk/drivers/net/avf/base/avf_status.h create mode 100644 dpdk/drivers/net/avf/base/avf_type.h create mode 100644 dpdk/drivers/net/avf/base/meson.build create mode 100644 dpdk/drivers/net/avf/base/virtchnl.h create mode 100644 dpdk/drivers/net/avf/meson.build create mode 100644 dpdk/drivers/net/avf/rte_pmd_avf_version.map create mode 100644 dpdk/drivers/net/avp/meson.build create mode 100644 dpdk/drivers/net/axgbe/Makefile create mode 100644 dpdk/drivers/net/axgbe/axgbe_common.h create mode 100644 dpdk/drivers/net/axgbe/axgbe_dev.c create mode 100644 dpdk/drivers/net/axgbe/axgbe_ethdev.c create mode 100644 dpdk/drivers/net/axgbe/axgbe_ethdev.h create mode 100644 dpdk/drivers/net/axgbe/axgbe_i2c.c create mode 100644 dpdk/drivers/net/axgbe/axgbe_logs.h create mode 100644 dpdk/drivers/net/axgbe/axgbe_mdio.c create mode 100644 dpdk/drivers/net/axgbe/axgbe_phy.h create mode 100644 dpdk/drivers/net/axgbe/axgbe_phy_impl.c create mode 100644 dpdk/drivers/net/axgbe/axgbe_rxtx.c create mode 100644 dpdk/drivers/net/axgbe/axgbe_rxtx.h create mode 100644 dpdk/drivers/net/axgbe/axgbe_rxtx_vec_sse.c create mode 100644 dpdk/drivers/net/axgbe/meson.build create mode 100644 dpdk/drivers/net/axgbe/rte_pmd_axgbe_version.map delete mode 100644 dpdk/drivers/net/bnx2x/LICENSE.bnx2x_pmd create mode 100644 dpdk/drivers/net/bnx2x/meson.build create mode 100644 dpdk/drivers/net/bnxt/bnxt_flow.c create mode 100644 dpdk/drivers/net/bnxt/bnxt_util.c create mode 100644 dpdk/drivers/net/bnxt/bnxt_util.h create mode 100644 dpdk/drivers/net/bnxt/meson.build create mode 100644 dpdk/drivers/net/bonding/meson.build create mode 100644 dpdk/drivers/net/bonding/rte_eth_bond_flow.c create mode 100644 dpdk/drivers/net/cxgbe/base/t4_tcb.h create mode 100644 dpdk/drivers/net/cxgbe/base/t4vf_hw.c create mode 100644 dpdk/drivers/net/cxgbe/base/t4vf_hw.h create mode 100644 dpdk/drivers/net/cxgbe/clip_tbl.c create mode 100644 dpdk/drivers/net/cxgbe/clip_tbl.h create mode 100644 dpdk/drivers/net/cxgbe/cxgbe_filter.c create mode 100644 dpdk/drivers/net/cxgbe/cxgbe_filter.h create mode 100644 dpdk/drivers/net/cxgbe/cxgbe_flow.c create mode 100644 dpdk/drivers/net/cxgbe/cxgbe_flow.h create mode 100644 dpdk/drivers/net/cxgbe/cxgbe_ofld.h create mode 100644 dpdk/drivers/net/cxgbe/cxgbe_pfvf.h create mode 100644 dpdk/drivers/net/cxgbe/cxgbevf_ethdev.c create mode 100644 dpdk/drivers/net/cxgbe/cxgbevf_main.c create mode 100644 dpdk/drivers/net/cxgbe/l2t.c create mode 100644 dpdk/drivers/net/cxgbe/l2t.h create mode 100644 dpdk/drivers/net/cxgbe/meson.build create mode 100644 dpdk/drivers/net/cxgbe/mps_tcam.c create mode 100644 dpdk/drivers/net/cxgbe/mps_tcam.h create mode 100644 dpdk/drivers/net/dpaa/meson.build create mode 100644 dpdk/drivers/net/dpaa/rte_pmd_dpaa.h create mode 100644 dpdk/drivers/net/dpaa2/dpaa2_pmd_logs.h create mode 100644 dpdk/drivers/net/dpaa2/meson.build create mode 100644 dpdk/drivers/net/e1000/base/meson.build create mode 100644 dpdk/drivers/net/e1000/e1000_logs.c create mode 100644 dpdk/drivers/net/e1000/meson.build create mode 100644 dpdk/drivers/net/ena/meson.build create mode 100644 dpdk/drivers/net/enetc/Makefile create mode 100644 dpdk/drivers/net/enetc/base/enetc_hw.h create mode 100644 dpdk/drivers/net/enetc/enetc.h create mode 100644 dpdk/drivers/net/enetc/enetc_ethdev.c create mode 100644 dpdk/drivers/net/enetc/enetc_logs.h create mode 100644 dpdk/drivers/net/enetc/enetc_rxtx.c create mode 100644 dpdk/drivers/net/enetc/meson.build create mode 100644 dpdk/drivers/net/enetc/rte_pmd_enetc_version.map delete mode 100644 dpdk/drivers/net/enic/LICENSE create mode 100644 dpdk/drivers/net/enic/enic_rxtx_common.h create mode 100644 dpdk/drivers/net/enic/enic_rxtx_vec_avx2.c create mode 100644 dpdk/drivers/net/enic/meson.build create mode 100644 dpdk/drivers/net/failsafe/failsafe_intr.c create mode 100644 dpdk/drivers/net/failsafe/meson.build create mode 100644 dpdk/drivers/net/fm10k/base/meson.build create mode 100644 dpdk/drivers/net/fm10k/meson.build create mode 100644 dpdk/drivers/net/i40e/base/meson.build create mode 100644 dpdk/drivers/net/i40e/i40e_rxtx_vec_avx2.c create mode 100644 dpdk/drivers/net/i40e/i40e_vf_representor.c create mode 100644 dpdk/drivers/net/i40e/meson.build create mode 100644 dpdk/drivers/net/ifc/Makefile create mode 100644 dpdk/drivers/net/ifc/base/ifcvf.c create mode 100644 dpdk/drivers/net/ifc/base/ifcvf.h create mode 100644 dpdk/drivers/net/ifc/base/ifcvf_osdep.h create mode 100644 dpdk/drivers/net/ifc/ifcvf_vdpa.c create mode 100644 dpdk/drivers/net/ifc/meson.build create mode 100644 dpdk/drivers/net/ifc/rte_pmd_ifc_version.map create mode 100644 dpdk/drivers/net/ixgbe/base/meson.build create mode 100644 dpdk/drivers/net/ixgbe/ixgbe_vf_representor.c create mode 100644 dpdk/drivers/net/ixgbe/meson.build create mode 100644 dpdk/drivers/net/kni/meson.build create mode 100644 dpdk/drivers/net/liquidio/meson.build rename dpdk/drivers/net/liquidio/{rte_pmd_lio_version.map => rte_pmd_liquidio_version.map} (100%) create mode 100644 dpdk/drivers/net/meson.build create mode 100644 dpdk/drivers/net/mlx4/meson.build create mode 100644 dpdk/drivers/net/mlx4/mlx4_glue.c create mode 100644 dpdk/drivers/net/mlx4/mlx4_glue.h create mode 100644 dpdk/drivers/net/mlx4/mlx4_mr.h create mode 100644 dpdk/drivers/net/mlx5/meson.build create mode 100644 dpdk/drivers/net/mlx5/mlx5_flow.h create mode 100644 dpdk/drivers/net/mlx5/mlx5_flow_dv.c create mode 100644 dpdk/drivers/net/mlx5/mlx5_flow_tcf.c create mode 100644 dpdk/drivers/net/mlx5/mlx5_flow_verbs.c create mode 100644 dpdk/drivers/net/mlx5/mlx5_glue.c create mode 100644 dpdk/drivers/net/mlx5/mlx5_glue.h create mode 100644 dpdk/drivers/net/mlx5/mlx5_mr.h create mode 100644 dpdk/drivers/net/mlx5/mlx5_nl.c delete mode 100644 dpdk/drivers/net/mrvl/Makefile delete mode 100644 dpdk/drivers/net/mrvl/mrvl_ethdev.c delete mode 100644 dpdk/drivers/net/mrvl/mrvl_ethdev.h delete mode 100644 dpdk/drivers/net/mrvl/mrvl_qos.c delete mode 100644 dpdk/drivers/net/mrvl/mrvl_qos.h create mode 100644 dpdk/drivers/net/mvneta/Makefile create mode 100644 dpdk/drivers/net/mvneta/meson.build create mode 100644 dpdk/drivers/net/mvneta/mvneta_ethdev.c create mode 100644 dpdk/drivers/net/mvneta/mvneta_ethdev.h create mode 100644 dpdk/drivers/net/mvneta/mvneta_rxtx.c create mode 100644 dpdk/drivers/net/mvneta/mvneta_rxtx.h create mode 100644 dpdk/drivers/net/mvneta/rte_pmd_mvneta_version.map create mode 100644 dpdk/drivers/net/mvpp2/Makefile create mode 100644 dpdk/drivers/net/mvpp2/meson.build create mode 100644 dpdk/drivers/net/mvpp2/mrvl_ethdev.c create mode 100644 dpdk/drivers/net/mvpp2/mrvl_ethdev.h create mode 100644 dpdk/drivers/net/mvpp2/mrvl_flow.c create mode 100644 dpdk/drivers/net/mvpp2/mrvl_flow.h create mode 100644 dpdk/drivers/net/mvpp2/mrvl_mtr.c create mode 100644 dpdk/drivers/net/mvpp2/mrvl_mtr.h create mode 100644 dpdk/drivers/net/mvpp2/mrvl_qos.c create mode 100644 dpdk/drivers/net/mvpp2/mrvl_qos.h create mode 100644 dpdk/drivers/net/mvpp2/mrvl_tm.c create mode 100644 dpdk/drivers/net/mvpp2/mrvl_tm.h rename dpdk/drivers/net/{mrvl/rte_pmd_mrvl_version.map => mvpp2/rte_pmd_mvpp2_version.map} (100%) create mode 100644 dpdk/drivers/net/netvsc/Makefile create mode 100644 dpdk/drivers/net/netvsc/hn_ethdev.c create mode 100644 dpdk/drivers/net/netvsc/hn_logs.h create mode 100644 dpdk/drivers/net/netvsc/hn_nvs.c create mode 100644 dpdk/drivers/net/netvsc/hn_nvs.h create mode 100644 dpdk/drivers/net/netvsc/hn_rndis.c create mode 100644 dpdk/drivers/net/netvsc/hn_rndis.h create mode 100644 dpdk/drivers/net/netvsc/hn_rxtx.c create mode 100644 dpdk/drivers/net/netvsc/hn_var.h create mode 100644 dpdk/drivers/net/netvsc/hn_vf.c create mode 100644 dpdk/drivers/net/netvsc/meson.build create mode 100644 dpdk/drivers/net/netvsc/ndis.h create mode 100644 dpdk/drivers/net/netvsc/rndis.h create mode 100644 dpdk/drivers/net/netvsc/rte_pmd_netvsc_version.map create mode 100644 dpdk/drivers/net/nfp/meson.build delete mode 100644 dpdk/drivers/net/nfp/nfp_net_eth.h delete mode 100644 dpdk/drivers/net/nfp/nfp_nfpu.c delete mode 100644 dpdk/drivers/net/nfp/nfp_nfpu.h delete mode 100644 dpdk/drivers/net/nfp/nfp_nspu.c delete mode 100644 dpdk/drivers/net/nfp/nfp_nspu.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp-common/nfp_cppat.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp-common/nfp_platform.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp-common/nfp_resid.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp6000/nfp6000.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp6000/nfp_xpb.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_cpp.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_cpp_pcie_ops.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_cppcore.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_crc.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_crc.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_hwinfo.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_hwinfo.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_mip.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_mip.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_mutex.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_nffw.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_nffw.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_nsp.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_nsp.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_nsp_cmds.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_nsp_eth.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_resource.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_resource.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_rtsym.c create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_rtsym.h create mode 100644 dpdk/drivers/net/nfp/nfpcore/nfp_target.h create mode 100644 dpdk/drivers/net/null/meson.build create mode 100644 dpdk/drivers/net/octeontx/base/meson.build create mode 100644 dpdk/drivers/net/octeontx/meson.build create mode 100644 dpdk/drivers/net/pcap/meson.build delete mode 100644 dpdk/drivers/net/qede/LICENSE.qede_pmd create mode 100644 dpdk/drivers/net/qede/base/meson.build create mode 100644 dpdk/drivers/net/qede/meson.build delete mode 100644 dpdk/drivers/net/qede/qede_fdir.c create mode 100644 dpdk/drivers/net/qede/qede_filter.c create mode 100644 dpdk/drivers/net/ring/meson.build create mode 100644 dpdk/drivers/net/sfc/base/ef10_image.c create mode 100644 dpdk/drivers/net/sfc/base/ef10_signed_image_layout.h create mode 100644 dpdk/drivers/net/sfc/base/efx_annote.h create mode 100644 dpdk/drivers/net/sfc/base/efx_regs_mcdi_aoe.h create mode 100644 dpdk/drivers/net/sfc/base/efx_tunnel.c create mode 100644 dpdk/drivers/net/sfc/base/mc_driver_pcol_strs.h create mode 100644 dpdk/drivers/net/sfc/base/medford2_impl.h create mode 100644 dpdk/drivers/net/sfc/base/medford2_nic.c create mode 100644 dpdk/drivers/net/sfc/base/meson.build create mode 100644 dpdk/drivers/net/sfc/meson.build rename dpdk/drivers/net/sfc/{rte_pmd_sfc_efx_version.map => rte_pmd_sfc_version.map} (100%) create mode 100644 dpdk/drivers/net/sfc/sfc_ef10_essb_rx.c create mode 100644 dpdk/drivers/net/sfc/sfc_ef10_rx_ev.h create mode 100644 dpdk/drivers/net/sfc/sfc_tso.h create mode 100644 dpdk/drivers/net/softnic/conn.c create mode 100644 dpdk/drivers/net/softnic/conn.h create mode 100644 dpdk/drivers/net/softnic/firmware.cli create mode 100644 dpdk/drivers/net/softnic/meson.build create mode 100644 dpdk/drivers/net/softnic/parser.c create mode 100644 dpdk/drivers/net/softnic/parser.h create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_action.c create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_cli.c create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_cryptodev.c create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_flow.c create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_link.c create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_mempool.c create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_meter.c create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_pipeline.c create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_swq.c create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_tap.c create mode 100644 dpdk/drivers/net/softnic/rte_eth_softnic_thread.c delete mode 100644 dpdk/drivers/net/softnic/rte_pmd_eth_softnic_version.map create mode 100644 dpdk/drivers/net/softnic/rte_pmd_softnic_version.map create mode 100644 dpdk/drivers/net/szedata2/meson.build delete mode 100644 dpdk/drivers/net/szedata2/szedata2_iobuf.c delete mode 100644 dpdk/drivers/net/szedata2/szedata2_iobuf.h create mode 100644 dpdk/drivers/net/szedata2/szedata2_logs.h create mode 100644 dpdk/drivers/net/tap/meson.build create mode 100644 dpdk/drivers/net/tap/tap_bpf.h create mode 100644 dpdk/drivers/net/tap/tap_bpf_api.c create mode 100644 dpdk/drivers/net/tap/tap_bpf_insns.h create mode 100644 dpdk/drivers/net/tap/tap_bpf_program.c create mode 100644 dpdk/drivers/net/tap/tap_intr.c create mode 100644 dpdk/drivers/net/tap/tap_log.h create mode 100644 dpdk/drivers/net/tap/tap_rss.h create mode 100644 dpdk/drivers/net/thunderx/base/meson.build create mode 100644 dpdk/drivers/net/thunderx/meson.build rename dpdk/drivers/net/thunderx/{rte_pmd_thunderx_nicvf_version.map => rte_pmd_thunderx_version.map} (100%) create mode 100644 dpdk/drivers/net/vdev_netvsc/Makefile create mode 100644 dpdk/drivers/net/vdev_netvsc/meson.build create mode 100644 dpdk/drivers/net/vdev_netvsc/rte_pmd_vdev_netvsc_version.map create mode 100644 dpdk/drivers/net/vdev_netvsc/vdev_netvsc.c create mode 100644 dpdk/drivers/net/vhost/meson.build create mode 100644 dpdk/drivers/net/virtio/meson.build create mode 100644 dpdk/drivers/net/vmxnet3/meson.build create mode 100644 dpdk/drivers/raw/Makefile create mode 100644 dpdk/drivers/raw/dpaa2_cmdif/Makefile create mode 100644 dpdk/drivers/raw/dpaa2_cmdif/dpaa2_cmdif.c create mode 100644 dpdk/drivers/raw/dpaa2_cmdif/dpaa2_cmdif_logs.h create mode 100644 dpdk/drivers/raw/dpaa2_cmdif/meson.build create mode 100644 dpdk/drivers/raw/dpaa2_cmdif/rte_pmd_dpaa2_cmdif.h create mode 100644 dpdk/drivers/raw/dpaa2_cmdif/rte_pmd_dpaa2_cmdif_version.map create mode 100644 dpdk/drivers/raw/dpaa2_qdma/Makefile create mode 100644 dpdk/drivers/raw/dpaa2_qdma/dpaa2_qdma.c create mode 100644 dpdk/drivers/raw/dpaa2_qdma/dpaa2_qdma.h create mode 100644 dpdk/drivers/raw/dpaa2_qdma/dpaa2_qdma_logs.h create mode 100644 dpdk/drivers/raw/dpaa2_qdma/meson.build create mode 100644 dpdk/drivers/raw/dpaa2_qdma/rte_pmd_dpaa2_qdma.h create mode 100644 dpdk/drivers/raw/dpaa2_qdma/rte_pmd_dpaa2_qdma_version.map create mode 100644 dpdk/drivers/raw/ifpga_rawdev/Makefile create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/Makefile create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/README create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_api.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_api.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_compat.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_defines.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_enumerate.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_enumerate.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_feature_dev.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_fme.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_fme_dperf.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_fme_error.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_fme_iperf.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_fme_pr.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_hw.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_port.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/ifpga_port_error.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/meson.build create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/opae_debug.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/opae_debug.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/opae_hw_api.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/opae_hw_api.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/opae_ifpga_hw_api.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/opae_osdep.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/osdep_raw/osdep_generic.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/base/osdep_rte/osdep_generic.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/ifpga_rawdev.c create mode 100644 dpdk/drivers/raw/ifpga_rawdev/ifpga_rawdev.h create mode 100644 dpdk/drivers/raw/ifpga_rawdev/meson.build create mode 100644 dpdk/drivers/raw/ifpga_rawdev/rte_pmd_ifpga_rawdev_version.map create mode 100644 dpdk/drivers/raw/meson.build create mode 100644 dpdk/drivers/raw/skeleton_rawdev/Makefile create mode 100644 dpdk/drivers/raw/skeleton_rawdev/meson.build create mode 100644 dpdk/drivers/raw/skeleton_rawdev/rte_pmd_skeleton_rawdev_version.map create mode 100644 dpdk/drivers/raw/skeleton_rawdev/skeleton_rawdev.c create mode 100644 dpdk/drivers/raw/skeleton_rawdev/skeleton_rawdev.h create mode 100644 dpdk/drivers/raw/skeleton_rawdev/skeleton_rawdev_test.c create mode 100644 dpdk/examples/bbdev_app/Makefile create mode 100644 dpdk/examples/bbdev_app/main.c create mode 100644 dpdk/examples/bbdev_app/meson.build create mode 100644 dpdk/examples/bond/meson.build create mode 100644 dpdk/examples/cmdline/meson.build create mode 100644 dpdk/examples/distributor/meson.build create mode 100644 dpdk/examples/ethtool/meson.build create mode 100644 dpdk/examples/eventdev_pipeline/Makefile create mode 100644 dpdk/examples/eventdev_pipeline/main.c create mode 100644 dpdk/examples/eventdev_pipeline/meson.build create mode 100644 dpdk/examples/eventdev_pipeline/pipeline_common.h create mode 100644 dpdk/examples/eventdev_pipeline/pipeline_worker_generic.c create mode 100644 dpdk/examples/eventdev_pipeline/pipeline_worker_tx.c delete mode 100644 dpdk/examples/eventdev_pipeline_sw_pmd/Makefile delete mode 100644 dpdk/examples/eventdev_pipeline_sw_pmd/main.c create mode 100644 dpdk/examples/exception_path/meson.build create mode 100644 dpdk/examples/fips_validation/Makefile create mode 100644 dpdk/examples/fips_validation/fips_validation.c create mode 100644 dpdk/examples/fips_validation/fips_validation.h create mode 100644 dpdk/examples/fips_validation/fips_validation_aes.c create mode 100644 dpdk/examples/fips_validation/fips_validation_ccm.c create mode 100644 dpdk/examples/fips_validation/fips_validation_cmac.c create mode 100644 dpdk/examples/fips_validation/fips_validation_gcm.c create mode 100644 dpdk/examples/fips_validation/fips_validation_hmac.c create mode 100644 dpdk/examples/fips_validation/fips_validation_tdes.c create mode 100644 dpdk/examples/fips_validation/main.c create mode 100644 dpdk/examples/fips_validation/meson.build create mode 100644 dpdk/examples/flow_classify/meson.build create mode 100644 dpdk/examples/flow_filtering/meson.build create mode 100644 dpdk/examples/helloworld/meson.build create mode 100644 dpdk/examples/ip_fragmentation/meson.build create mode 100644 dpdk/examples/ip_pipeline/action.c create mode 100644 dpdk/examples/ip_pipeline/action.h delete mode 100644 dpdk/examples/ip_pipeline/app.h create mode 100644 dpdk/examples/ip_pipeline/cli.c create mode 100644 dpdk/examples/ip_pipeline/cli.h create mode 100644 dpdk/examples/ip_pipeline/common.h delete mode 100644 dpdk/examples/ip_pipeline/config/action.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/action.sh delete mode 100644 dpdk/examples/ip_pipeline/config/action.txt delete mode 100755 dpdk/examples/ip_pipeline/config/diagram-generator.py delete mode 100644 dpdk/examples/ip_pipeline/config/edge_router_downstream.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/edge_router_downstream.sh delete mode 100644 dpdk/examples/ip_pipeline/config/edge_router_upstream.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/edge_router_upstream.sh delete mode 100644 dpdk/examples/ip_pipeline/config/firewall.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/firewall.sh delete mode 100644 dpdk/examples/ip_pipeline/config/firewall.txt delete mode 100644 dpdk/examples/ip_pipeline/config/flow.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/flow.sh delete mode 100644 dpdk/examples/ip_pipeline/config/flow.txt delete mode 100644 dpdk/examples/ip_pipeline/config/ip_pipeline.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/ip_pipeline.sh delete mode 100644 dpdk/examples/ip_pipeline/config/kni.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/l2fwd.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/l3fwd.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/l3fwd.sh delete mode 100644 dpdk/examples/ip_pipeline/config/l3fwd_arp.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/l3fwd_arp.sh delete mode 100644 dpdk/examples/ip_pipeline/config/network_layers.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/network_layers.sh delete mode 100755 dpdk/examples/ip_pipeline/config/pipeline-to-core-mapping.py delete mode 100644 dpdk/examples/ip_pipeline/config/tap.cfg delete mode 100644 dpdk/examples/ip_pipeline/config/tm_profile.cfg delete mode 100644 dpdk/examples/ip_pipeline/config_check.c delete mode 100644 dpdk/examples/ip_pipeline/config_parse.c delete mode 100644 dpdk/examples/ip_pipeline/config_parse_tm.c create mode 100644 dpdk/examples/ip_pipeline/conn.c create mode 100644 dpdk/examples/ip_pipeline/conn.h delete mode 100644 dpdk/examples/ip_pipeline/cpu_core_map.c delete mode 100644 dpdk/examples/ip_pipeline/cpu_core_map.h create mode 100644 dpdk/examples/ip_pipeline/cryptodev.c create mode 100644 dpdk/examples/ip_pipeline/cryptodev.h create mode 100644 dpdk/examples/ip_pipeline/examples/firewall.cli create mode 100644 dpdk/examples/ip_pipeline/examples/flow.cli create mode 100644 dpdk/examples/ip_pipeline/examples/flow_crypto.cli create mode 100644 dpdk/examples/ip_pipeline/examples/kni.cli create mode 100644 dpdk/examples/ip_pipeline/examples/l2fwd.cli create mode 100644 dpdk/examples/ip_pipeline/examples/route.cli create mode 100644 dpdk/examples/ip_pipeline/examples/route_ecmp.cli create mode 100644 dpdk/examples/ip_pipeline/examples/rss.cli create mode 100644 dpdk/examples/ip_pipeline/examples/tap.cli delete mode 100644 dpdk/examples/ip_pipeline/init.c create mode 100644 dpdk/examples/ip_pipeline/kni.c create mode 100644 dpdk/examples/ip_pipeline/kni.h create mode 100644 dpdk/examples/ip_pipeline/link.c create mode 100644 dpdk/examples/ip_pipeline/link.h create mode 100644 dpdk/examples/ip_pipeline/mempool.c create mode 100644 dpdk/examples/ip_pipeline/mempool.h create mode 100644 dpdk/examples/ip_pipeline/meson.build create mode 100644 dpdk/examples/ip_pipeline/pipeline.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/hash_func.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/hash_func_arm64.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_actions_common.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_common_be.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_common_be.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_common_fe.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_common_fe.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_firewall.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_firewall.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_firewall_be.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_firewall_be.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_flow_actions.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_flow_actions.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_flow_actions_be.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_flow_actions_be.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification_be.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_flow_classification_be.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_master.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_master.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_master_be.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_master_be.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough_be.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_passthrough_be.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_routing.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_routing.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_routing_be.c delete mode 100644 dpdk/examples/ip_pipeline/pipeline/pipeline_routing_be.h delete mode 100644 dpdk/examples/ip_pipeline/pipeline_be.h create mode 100644 dpdk/examples/ip_pipeline/swq.c create mode 100644 dpdk/examples/ip_pipeline/swq.h create mode 100644 dpdk/examples/ip_pipeline/tap.c create mode 100644 dpdk/examples/ip_pipeline/tap.h delete mode 100644 dpdk/examples/ip_pipeline/thread_fe.c delete mode 100644 dpdk/examples/ip_pipeline/thread_fe.h create mode 100644 dpdk/examples/ip_pipeline/tmgr.c create mode 100644 dpdk/examples/ip_pipeline/tmgr.h create mode 100644 dpdk/examples/ip_reassembly/meson.build create mode 100644 dpdk/examples/ipsec-secgw/meson.build create mode 100644 dpdk/examples/ipv4_multicast/meson.build create mode 100644 dpdk/examples/kni/meson.build create mode 100644 dpdk/examples/l2fwd-cat/meson.build create mode 100644 dpdk/examples/l2fwd-crypto/meson.build create mode 100644 dpdk/examples/l2fwd-jobstats/meson.build create mode 100644 dpdk/examples/l2fwd-keepalive/meson.build create mode 100644 dpdk/examples/l2fwd/meson.build create mode 100644 dpdk/examples/l3fwd-acl/meson.build create mode 100644 dpdk/examples/l3fwd-power/main.h create mode 100644 dpdk/examples/l3fwd-power/meson.build create mode 100644 dpdk/examples/l3fwd-power/perf_core.c create mode 100644 dpdk/examples/l3fwd-power/perf_core.h create mode 100644 dpdk/examples/l3fwd-vf/meson.build create mode 100644 dpdk/examples/l3fwd/meson.build create mode 100644 dpdk/examples/link_status_interrupt/meson.build create mode 100644 dpdk/examples/load_balancer/meson.build create mode 100644 dpdk/examples/meson.build create mode 100644 dpdk/examples/multi_process/hotplug_mp/Makefile create mode 100644 dpdk/examples/multi_process/hotplug_mp/commands.c create mode 100644 dpdk/examples/multi_process/hotplug_mp/commands.h create mode 100644 dpdk/examples/multi_process/hotplug_mp/main.c delete mode 100644 dpdk/examples/multi_process/l2fwd_fork/Makefile delete mode 100644 dpdk/examples/multi_process/l2fwd_fork/flib.c delete mode 100644 dpdk/examples/multi_process/l2fwd_fork/flib.h delete mode 100644 dpdk/examples/multi_process/l2fwd_fork/main.c create mode 100644 dpdk/examples/multi_process/meson.build create mode 100644 dpdk/examples/netmap_compat/meson.build create mode 100644 dpdk/examples/packet_ordering/meson.build create mode 100644 dpdk/examples/performance-thread/meson.build create mode 100644 dpdk/examples/ptpclient/meson.build create mode 100644 dpdk/examples/qos_meter/meson.build create mode 100644 dpdk/examples/qos_sched/meson.build create mode 100644 dpdk/examples/quota_watermark/meson.build create mode 100644 dpdk/examples/rxtx_callbacks/meson.build create mode 100644 dpdk/examples/server_node_efd/meson.build create mode 100644 dpdk/examples/service_cores/meson.build create mode 100644 dpdk/examples/skeleton/meson.build create mode 100644 dpdk/examples/tep_termination/meson.build create mode 100644 dpdk/examples/timer/meson.build create mode 100644 dpdk/examples/vdpa/Makefile create mode 100644 dpdk/examples/vdpa/main.c create mode 100644 dpdk/examples/vdpa/meson.build create mode 100644 dpdk/examples/vhost/meson.build create mode 100644 dpdk/examples/vhost_crypto/Makefile create mode 100644 dpdk/examples/vhost_crypto/main.c create mode 100644 dpdk/examples/vhost_crypto/meson.build create mode 100644 dpdk/examples/vhost_scsi/meson.build create mode 100644 dpdk/examples/vm_power_manager/guest_cli/meson.build create mode 100644 dpdk/examples/vm_power_manager/guest_cli/parse.c create mode 100644 dpdk/examples/vm_power_manager/guest_cli/parse.h create mode 100644 dpdk/examples/vm_power_manager/meson.build create mode 100644 dpdk/examples/vm_power_manager/oob_monitor.h create mode 100644 dpdk/examples/vm_power_manager/oob_monitor_nop.c create mode 100644 dpdk/examples/vm_power_manager/oob_monitor_x86.c create mode 100644 dpdk/examples/vm_power_manager/parse.c create mode 100644 dpdk/examples/vm_power_manager/parse.h create mode 100644 dpdk/examples/vmdq/meson.build create mode 100644 dpdk/examples/vmdq_dcb/meson.build create mode 100644 dpdk/kernel/Makefile create mode 100644 dpdk/kernel/freebsd/BSDmakefile.meson create mode 100644 dpdk/kernel/freebsd/Makefile create mode 100644 dpdk/kernel/freebsd/contigmem/BSDmakefile create mode 100644 dpdk/kernel/freebsd/contigmem/Makefile create mode 100644 dpdk/kernel/freebsd/contigmem/contigmem.c create mode 100644 dpdk/kernel/freebsd/contigmem/meson.build create mode 100644 dpdk/kernel/freebsd/meson.build create mode 100644 dpdk/kernel/freebsd/nic_uio/BSDmakefile create mode 100644 dpdk/kernel/freebsd/nic_uio/Makefile create mode 100644 dpdk/kernel/freebsd/nic_uio/meson.build create mode 100644 dpdk/kernel/freebsd/nic_uio/nic_uio.c create mode 100644 dpdk/kernel/linux/Makefile create mode 100644 dpdk/kernel/linux/igb_uio/Kbuild create mode 100644 dpdk/kernel/linux/igb_uio/Makefile create mode 100644 dpdk/kernel/linux/igb_uio/compat.h create mode 100644 dpdk/kernel/linux/igb_uio/igb_uio.c create mode 100644 dpdk/kernel/linux/igb_uio/meson.build create mode 100644 dpdk/kernel/linux/kni/Kbuild create mode 100644 dpdk/kernel/linux/kni/Makefile create mode 100644 dpdk/kernel/linux/kni/compat.h create mode 100644 dpdk/kernel/linux/kni/ethtool/README create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_82575.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_82575.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_api.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_api.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_defines.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_hw.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_i210.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_i210.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_mac.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_mac.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_manage.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_manage.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_mbx.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_mbx.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_nvm.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_nvm.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_osdep.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_phy.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_phy.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/e1000_regs.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/igb.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/igb_ethtool.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/igb_main.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/igb_param.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/igb_regtest.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/igb_vmdq.c create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/igb_vmdq.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/kcompat.h create mode 100644 dpdk/kernel/linux/kni/ethtool/igb/meson.build create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_82598.c create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_82598.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_82599.c create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_82599.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_api.c create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_api.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_common.c create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_common.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_dcb.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_ethtool.c create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_fcoe.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_main.c create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_mbx.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_osdep.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_phy.c create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_phy.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_type.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_x540.c create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/ixgbe_x540.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/kcompat.c create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/kcompat.h create mode 100644 dpdk/kernel/linux/kni/ethtool/ixgbe/meson.build create mode 100644 dpdk/kernel/linux/kni/ethtool/meson.build create mode 100644 dpdk/kernel/linux/kni/kni_dev.h create mode 100644 dpdk/kernel/linux/kni/kni_ethtool.c create mode 100644 dpdk/kernel/linux/kni/kni_fifo.h create mode 100644 dpdk/kernel/linux/kni/kni_misc.c create mode 100644 dpdk/kernel/linux/kni/kni_net.c create mode 100644 dpdk/kernel/linux/kni/meson.build create mode 100644 dpdk/kernel/linux/meson.build create mode 100644 dpdk/kernel/meson.build create mode 100644 dpdk/lib/librte_acl/meson.build create mode 100644 dpdk/lib/librte_bbdev/Makefile create mode 100644 dpdk/lib/librte_bbdev/meson.build create mode 100644 dpdk/lib/librte_bbdev/rte_bbdev.c create mode 100644 dpdk/lib/librte_bbdev/rte_bbdev.h create mode 100644 dpdk/lib/librte_bbdev/rte_bbdev_op.h create mode 100644 dpdk/lib/librte_bbdev/rte_bbdev_pmd.h create mode 100644 dpdk/lib/librte_bbdev/rte_bbdev_version.map create mode 100644 dpdk/lib/librte_bitratestats/meson.build create mode 100644 dpdk/lib/librte_bpf/Makefile create mode 100644 dpdk/lib/librte_bpf/bpf.c create mode 100644 dpdk/lib/librte_bpf/bpf_def.h create mode 100644 dpdk/lib/librte_bpf/bpf_exec.c create mode 100644 dpdk/lib/librte_bpf/bpf_impl.h create mode 100644 dpdk/lib/librte_bpf/bpf_jit_x86.c create mode 100644 dpdk/lib/librte_bpf/bpf_load.c create mode 100644 dpdk/lib/librte_bpf/bpf_load_elf.c create mode 100644 dpdk/lib/librte_bpf/bpf_pkt.c create mode 100644 dpdk/lib/librte_bpf/bpf_validate.c create mode 100644 dpdk/lib/librte_bpf/meson.build create mode 100644 dpdk/lib/librte_bpf/rte_bpf.h create mode 100644 dpdk/lib/librte_bpf/rte_bpf_ethdev.h create mode 100644 dpdk/lib/librte_bpf/rte_bpf_version.map create mode 100644 dpdk/lib/librte_cfgfile/meson.build create mode 100644 dpdk/lib/librte_cmdline/meson.build create mode 100644 dpdk/lib/librte_compat/meson.build create mode 100644 dpdk/lib/librte_compressdev/Makefile create mode 100644 dpdk/lib/librte_compressdev/meson.build create mode 100644 dpdk/lib/librte_compressdev/rte_comp.c create mode 100644 dpdk/lib/librte_compressdev/rte_comp.h create mode 100644 dpdk/lib/librte_compressdev/rte_compressdev.c create mode 100644 dpdk/lib/librte_compressdev/rte_compressdev.h create mode 100644 dpdk/lib/librte_compressdev/rte_compressdev_internal.h create mode 100644 dpdk/lib/librte_compressdev/rte_compressdev_pmd.c create mode 100644 dpdk/lib/librte_compressdev/rte_compressdev_pmd.h create mode 100644 dpdk/lib/librte_compressdev/rte_compressdev_version.map create mode 100644 dpdk/lib/librte_cryptodev/meson.build create mode 100644 dpdk/lib/librte_cryptodev/rte_crypto_asym.h create mode 100644 dpdk/lib/librte_distributor/meson.build create mode 100644 dpdk/lib/librte_eal/bsdapp/BSDmakefile.meson delete mode 100644 dpdk/lib/librte_eal/bsdapp/contigmem/BSDmakefile delete mode 100644 dpdk/lib/librte_eal/bsdapp/contigmem/Makefile delete mode 100644 dpdk/lib/librte_eal/bsdapp/contigmem/contigmem.c create mode 100644 dpdk/lib/librte_eal/bsdapp/eal/eal_alarm_private.h create mode 100644 dpdk/lib/librte_eal/bsdapp/eal/eal_cpuflags.c create mode 100644 dpdk/lib/librte_eal/bsdapp/eal/eal_dev.c create mode 100644 dpdk/lib/librte_eal/bsdapp/eal/eal_memalloc.c create mode 100644 dpdk/lib/librte_eal/bsdapp/eal/meson.build delete mode 100644 dpdk/lib/librte_eal/bsdapp/nic_uio/BSDmakefile delete mode 100644 dpdk/lib/librte_eal/bsdapp/nic_uio/Makefile delete mode 100644 dpdk/lib/librte_eal/bsdapp/nic_uio/nic_uio.c create mode 100644 dpdk/lib/librte_eal/common/arch/arm/meson.build create mode 100644 dpdk/lib/librte_eal/common/arch/arm/rte_hypervisor.c create mode 100644 dpdk/lib/librte_eal/common/arch/ppc_64/meson.build create mode 100644 dpdk/lib/librte_eal/common/arch/ppc_64/rte_hypervisor.c create mode 100644 dpdk/lib/librte_eal/common/arch/x86/meson.build create mode 100644 dpdk/lib/librte_eal/common/arch/x86/rte_cpuid.h create mode 100644 dpdk/lib/librte_eal/common/arch/x86/rte_hypervisor.c delete mode 100644 dpdk/lib/librte_eal/common/arch/x86/rte_memcpy.c create mode 100644 dpdk/lib/librte_eal/common/eal_common_class.c create mode 100644 dpdk/lib/librte_eal/common/eal_common_fbarray.c create mode 100644 dpdk/lib/librte_eal/common/eal_common_hypervisor.c create mode 100644 dpdk/lib/librte_eal/common/eal_common_memalloc.c create mode 100644 dpdk/lib/librte_eal/common/eal_common_uuid.c create mode 100644 dpdk/lib/librte_eal/common/eal_memalloc.h create mode 100644 dpdk/lib/librte_eal/common/hotplug_mp.c create mode 100644 dpdk/lib/librte_eal/common/hotplug_mp.h create mode 100644 dpdk/lib/librte_eal/common/include/arch/arm/meson.build create mode 100644 dpdk/lib/librte_eal/common/include/arch/ppc_64/meson.build create mode 100644 dpdk/lib/librte_eal/common/include/arch/x86/meson.build create mode 100644 dpdk/lib/librte_eal/common/include/rte_class.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_fbarray.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_hypervisor.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_option.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_reciprocal.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_test.h create mode 100644 dpdk/lib/librte_eal/common/include/rte_uuid.h create mode 100644 dpdk/lib/librte_eal/common/malloc_mp.c create mode 100644 dpdk/lib/librte_eal/common/malloc_mp.h create mode 100644 dpdk/lib/librte_eal/common/meson.build create mode 100644 dpdk/lib/librte_eal/common/rte_option.c create mode 100644 dpdk/lib/librte_eal/common/rte_reciprocal.c create mode 100644 dpdk/lib/librte_eal/linuxapp/eal/eal_cpuflags.c create mode 100644 dpdk/lib/librte_eal/linuxapp/eal/eal_dev.c create mode 100644 dpdk/lib/librte_eal/linuxapp/eal/eal_memalloc.c create mode 100644 dpdk/lib/librte_eal/linuxapp/eal/meson.build delete mode 100644 dpdk/lib/librte_eal/linuxapp/igb_uio/Makefile delete mode 100644 dpdk/lib/librte_eal/linuxapp/igb_uio/compat.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/igb_uio/igb_uio.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/Makefile delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/compat.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/README delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_82575.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_api.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_api.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_defines.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_hw.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_i210.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_i210.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mac.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mac.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_manage.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_manage.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mbx.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_mbx.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_nvm.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_nvm.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_osdep.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_phy.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_phy.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/e1000_regs.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_ethtool.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_main.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_param.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_regtest.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_vmdq.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/igb_vmdq.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/igb/kcompat.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82598.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82598.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82599.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_82599.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_api.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_common.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_dcb.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_ethtool.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_fcoe.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_main.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_mbx.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_osdep.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_phy.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_phy.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_type.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_x540.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/ixgbe_x540.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/ethtool/ixgbe/kcompat.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/kni_dev.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/kni_ethtool.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/kni_fifo.h delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/kni_misc.c delete mode 100644 dpdk/lib/librte_eal/linuxapp/kni/kni_net.c create mode 100644 dpdk/lib/librte_eal/meson.build create mode 100644 dpdk/lib/librte_efd/meson.build create mode 100644 dpdk/lib/librte_ethdev/Makefile create mode 100644 dpdk/lib/librte_ethdev/ethdev_private.c create mode 100644 dpdk/lib/librte_ethdev/ethdev_private.h create mode 100644 dpdk/lib/librte_ethdev/ethdev_profile.c create mode 100644 dpdk/lib/librte_ethdev/ethdev_profile.h create mode 100644 dpdk/lib/librte_ethdev/meson.build create mode 100644 dpdk/lib/librte_ethdev/rte_class_eth.c create mode 100644 dpdk/lib/librte_ethdev/rte_dev_info.h create mode 100644 dpdk/lib/librte_ethdev/rte_eth_ctrl.h create mode 100644 dpdk/lib/librte_ethdev/rte_ethdev.c create mode 100644 dpdk/lib/librte_ethdev/rte_ethdev.h create mode 100644 dpdk/lib/librte_ethdev/rte_ethdev_core.h create mode 100644 dpdk/lib/librte_ethdev/rte_ethdev_driver.h create mode 100644 dpdk/lib/librte_ethdev/rte_ethdev_pci.h create mode 100644 dpdk/lib/librte_ethdev/rte_ethdev_vdev.h create mode 100644 dpdk/lib/librte_ethdev/rte_ethdev_version.map create mode 100644 dpdk/lib/librte_ethdev/rte_flow.c create mode 100644 dpdk/lib/librte_ethdev/rte_flow.h create mode 100644 dpdk/lib/librte_ethdev/rte_flow_driver.h create mode 100644 dpdk/lib/librte_ethdev/rte_mtr.c create mode 100644 dpdk/lib/librte_ethdev/rte_mtr.h create mode 100644 dpdk/lib/librte_ethdev/rte_mtr_driver.h create mode 100644 dpdk/lib/librte_ethdev/rte_tm.c create mode 100644 dpdk/lib/librte_ethdev/rte_tm.h create mode 100644 dpdk/lib/librte_ethdev/rte_tm_driver.h delete mode 100644 dpdk/lib/librte_ether/Makefile delete mode 100644 dpdk/lib/librte_ether/ethdev_profile.c delete mode 100644 dpdk/lib/librte_ether/ethdev_profile.h delete mode 100644 dpdk/lib/librte_ether/rte_dev_info.h delete mode 100644 dpdk/lib/librte_ether/rte_eth_ctrl.h delete mode 100644 dpdk/lib/librte_ether/rte_ethdev.c delete mode 100644 dpdk/lib/librte_ether/rte_ethdev.h delete mode 100644 dpdk/lib/librte_ether/rte_ethdev_pci.h delete mode 100644 dpdk/lib/librte_ether/rte_ethdev_vdev.h delete mode 100644 dpdk/lib/librte_ether/rte_ethdev_version.map delete mode 100644 dpdk/lib/librte_ether/rte_flow.c delete mode 100644 dpdk/lib/librte_ether/rte_flow.h delete mode 100644 dpdk/lib/librte_ether/rte_flow_driver.h delete mode 100644 dpdk/lib/librte_ether/rte_mtr.c delete mode 100644 dpdk/lib/librte_ether/rte_mtr.h delete mode 100644 dpdk/lib/librte_ether/rte_mtr_driver.h delete mode 100644 dpdk/lib/librte_ether/rte_tm.c delete mode 100644 dpdk/lib/librte_ether/rte_tm.h delete mode 100644 dpdk/lib/librte_ether/rte_tm_driver.h create mode 100644 dpdk/lib/librte_eventdev/meson.build create mode 100644 dpdk/lib/librte_eventdev/rte_event_crypto_adapter.c create mode 100644 dpdk/lib/librte_eventdev/rte_event_crypto_adapter.h create mode 100644 dpdk/lib/librte_eventdev/rte_event_eth_tx_adapter.c create mode 100644 dpdk/lib/librte_eventdev/rte_event_eth_tx_adapter.h create mode 100644 dpdk/lib/librte_eventdev/rte_event_timer_adapter.c create mode 100644 dpdk/lib/librte_eventdev/rte_event_timer_adapter.h create mode 100644 dpdk/lib/librte_eventdev/rte_event_timer_adapter_pmd.h create mode 100644 dpdk/lib/librte_flow_classify/meson.build create mode 100644 dpdk/lib/librte_gro/gro_vxlan_tcp4.c create mode 100644 dpdk/lib/librte_gro/gro_vxlan_tcp4.h create mode 100644 dpdk/lib/librte_gro/meson.build create mode 100644 dpdk/lib/librte_gso/gso_udp4.c create mode 100644 dpdk/lib/librte_gso/gso_udp4.h create mode 100644 dpdk/lib/librte_gso/meson.build create mode 100644 dpdk/lib/librte_hash/meson.build delete mode 100644 dpdk/lib/librte_hash/rte_cuckoo_hash_x86.h create mode 100644 dpdk/lib/librte_ip_frag/meson.build create mode 100644 dpdk/lib/librte_jobstats/meson.build create mode 100644 dpdk/lib/librte_kni/meson.build create mode 100644 dpdk/lib/librte_kvargs/meson.build create mode 100644 dpdk/lib/librte_latencystats/meson.build create mode 100644 dpdk/lib/librte_lpm/meson.build create mode 100644 dpdk/lib/librte_mbuf/meson.build create mode 100644 dpdk/lib/librte_mbuf/rte_mbuf_pool_ops.c create mode 100644 dpdk/lib/librte_mbuf/rte_mbuf_pool_ops.h create mode 100644 dpdk/lib/librte_member/meson.build create mode 100644 dpdk/lib/librte_mempool/meson.build create mode 100644 dpdk/lib/librte_mempool/rte_mempool_ops_default.c create mode 100644 dpdk/lib/librte_meter/meson.build create mode 100644 dpdk/lib/librte_metrics/meson.build create mode 100644 dpdk/lib/librte_net/meson.build create mode 100644 dpdk/lib/librte_net/rte_arp.c create mode 100644 dpdk/lib/librte_net/rte_mpls.h create mode 100644 dpdk/lib/librte_pci/meson.build create mode 100644 dpdk/lib/librte_pdump/meson.build create mode 100644 dpdk/lib/librte_pipeline/meson.build create mode 100644 dpdk/lib/librte_pipeline/rte_port_in_action.c create mode 100644 dpdk/lib/librte_pipeline/rte_port_in_action.h create mode 100644 dpdk/lib/librte_pipeline/rte_table_action.c create mode 100644 dpdk/lib/librte_pipeline/rte_table_action.h create mode 100644 dpdk/lib/librte_port/meson.build create mode 100644 dpdk/lib/librte_port/rte_port_sym_crypto.c create mode 100644 dpdk/lib/librte_port/rte_port_sym_crypto.h create mode 100644 dpdk/lib/librte_power/meson.build create mode 100644 dpdk/lib/librte_power/power_acpi_cpufreq.c create mode 100644 dpdk/lib/librte_power/power_acpi_cpufreq.h create mode 100644 dpdk/lib/librte_power/power_common.h create mode 100644 dpdk/lib/librte_power/power_kvm_vm.c create mode 100644 dpdk/lib/librte_power/power_kvm_vm.h delete mode 100644 dpdk/lib/librte_power/rte_power_acpi_cpufreq.c delete mode 100644 dpdk/lib/librte_power/rte_power_acpi_cpufreq.h delete mode 100644 dpdk/lib/librte_power/rte_power_common.h create mode 100644 dpdk/lib/librte_power/rte_power_empty_poll.c create mode 100644 dpdk/lib/librte_power/rte_power_empty_poll.h delete mode 100644 dpdk/lib/librte_power/rte_power_kvm_vm.c delete mode 100644 dpdk/lib/librte_power/rte_power_kvm_vm.h create mode 100644 dpdk/lib/librte_rawdev/Makefile create mode 100644 dpdk/lib/librte_rawdev/meson.build create mode 100644 dpdk/lib/librte_rawdev/rte_rawdev.c create mode 100644 dpdk/lib/librte_rawdev/rte_rawdev.h create mode 100644 dpdk/lib/librte_rawdev/rte_rawdev_pmd.h create mode 100644 dpdk/lib/librte_rawdev/rte_rawdev_version.map create mode 100644 dpdk/lib/librte_reorder/meson.build create mode 100644 dpdk/lib/librte_ring/meson.build create mode 100644 dpdk/lib/librte_ring/rte_ring_c11_mem.h create mode 100644 dpdk/lib/librte_ring/rte_ring_generic.h create mode 100644 dpdk/lib/librte_sched/meson.build delete mode 100644 dpdk/lib/librte_sched/rte_reciprocal.c delete mode 100644 dpdk/lib/librte_sched/rte_reciprocal.h create mode 100644 dpdk/lib/librte_security/meson.build create mode 100644 dpdk/lib/librte_table/meson.build create mode 100644 dpdk/lib/librte_table/rte_table_hash_cuckoo.h create mode 100644 dpdk/lib/librte_table/rte_table_hash_func.h create mode 100644 dpdk/lib/librte_table/rte_table_hash_func_arm64.h create mode 100644 dpdk/lib/librte_telemetry/Makefile create mode 100644 dpdk/lib/librte_telemetry/meson.build create mode 100644 dpdk/lib/librte_telemetry/rte_telemetry.c create mode 100644 dpdk/lib/librte_telemetry/rte_telemetry.h create mode 100644 dpdk/lib/librte_telemetry/rte_telemetry_internal.h create mode 100644 dpdk/lib/librte_telemetry/rte_telemetry_parser.c create mode 100644 dpdk/lib/librte_telemetry/rte_telemetry_parser.h create mode 100644 dpdk/lib/librte_telemetry/rte_telemetry_parser_test.c create mode 100644 dpdk/lib/librte_telemetry/rte_telemetry_parser_test.h create mode 100644 dpdk/lib/librte_telemetry/rte_telemetry_socket_tests.h create mode 100644 dpdk/lib/librte_telemetry/rte_telemetry_version.map create mode 100644 dpdk/lib/librte_timer/meson.build create mode 100644 dpdk/lib/librte_vhost/meson.build create mode 100644 dpdk/lib/librte_vhost/rte_vdpa.h create mode 100644 dpdk/lib/librte_vhost/rte_vhost_crypto.h create mode 100644 dpdk/lib/librte_vhost/vdpa.c create mode 100644 dpdk/lib/librte_vhost/vhost_crypto.c create mode 100644 dpdk/lib/librte_vhost/virtio_crypto.h create mode 100644 dpdk/lib/meson.build create mode 100644 dpdk/license/README create mode 100644 dpdk/license/bsd-3-clause.txt create mode 100644 dpdk/license/exceptions.txt create mode 100644 dpdk/license/gpl-2.0.txt create mode 100644 dpdk/license/lgpl-2.1.txt create mode 100644 dpdk/meson.build create mode 100644 dpdk/meson_options.txt create mode 100644 dpdk/test/bpf/dummy.c create mode 100644 dpdk/test/bpf/mbuf.h create mode 100644 dpdk/test/bpf/t1.c create mode 100644 dpdk/test/bpf/t2.c create mode 100644 dpdk/test/bpf/t3.c create mode 100644 dpdk/test/meson.build create mode 100644 dpdk/test/test/meson.build create mode 100644 dpdk/test/test/test_barrier.c create mode 100644 dpdk/test/test/test_bpf.c create mode 100644 dpdk/test/test/test_compressdev.c create mode 100644 dpdk/test/test/test_compressdev_test_buffer.h create mode 100644 dpdk/test/test/test_cryptodev_asym.c create mode 100644 dpdk/test/test/test_cryptodev_asym_util.h create mode 100644 dpdk/test/test/test_cryptodev_dh_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_dsa_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_mod_test_vectors.h create mode 100644 dpdk/test/test/test_cryptodev_rsa_test_vectors.h delete mode 100644 dpdk/test/test/test_devargs.c create mode 100644 dpdk/test/test/test_event_crypto_adapter.c create mode 100644 dpdk/test/test/test_event_eth_tx_adapter.c create mode 100644 dpdk/test/test/test_event_timer_adapter.c delete mode 100644 dpdk/test/test/test_eventdev_octeontx.c delete mode 100644 dpdk/test/test/test_eventdev_sw.c create mode 100644 dpdk/test/test/test_external_mem.c create mode 100644 dpdk/test/test/test_fbarray.c create mode 100644 dpdk/test/test/test_hash_readwrite.c create mode 100644 dpdk/test/test/test_hash_readwrite_lf.c delete mode 100644 dpdk/test/test/test_hash_scaling.c create mode 100644 dpdk/test/test/test_metrics.c create mode 100644 dpdk/test/test/test_rawdev.c create mode 100644 dpdk/test/test/test_reciprocal_division.c create mode 100644 dpdk/test/test/test_reciprocal_division_perf.c create mode 100644 dpdk/usertools/dpdk-telemetry-client.py create mode 100644 dpdk/usertools/meson.build diff --git a/dpdk/.gitignore b/dpdk/.gitignore index 6df5ba06..9105e26c 100644 --- a/dpdk/.gitignore +++ b/dpdk/.gitignore @@ -3,6 +3,7 @@ doc/guides/cryptodevs/overview_feature_table.txt doc/guides/cryptodevs/overview_cipher_table.txt doc/guides/cryptodevs/overview_auth_table.txt doc/guides/cryptodevs/overview_aead_table.txt +doc/guides/compressdevs/overview_feature_table.txt cscope.out.po cscope.out.in cscope.out diff --git a/dpdk/GNUmakefile b/dpdk/GNUmakefile index 45b7fbbe..ae80720e 100644 --- a/dpdk/GNUmakefile +++ b/dpdk/GNUmakefile @@ -1,33 +1,5 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2015 Intel Corporation # # Head Makefile for compiling rte SDK @@ -40,7 +12,7 @@ export RTE_SDK # directory list # -ROOTDIRS-y := buildtools lib drivers app +ROOTDIRS-y := buildtools lib kernel drivers app ROOTDIRS- := test include $(RTE_SDK)/mk/rte.sdkroot.mk diff --git a/dpdk/LICENSE.GPL b/dpdk/LICENSE.GPL deleted file mode 100644 index d511905c..00000000 --- a/dpdk/LICENSE.GPL +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/dpdk/LICENSE.LGPL b/dpdk/LICENSE.LGPL deleted file mode 100644 index 4362b491..00000000 --- a/dpdk/LICENSE.LGPL +++ /dev/null @@ -1,502 +0,0 @@ - GNU LESSER GENERAL PUBLIC LICENSE - Version 2.1, February 1999 - - Copyright (C) 1991, 1999 Free Software Foundation, Inc. - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - -[This is the first released version of the Lesser GPL. It also counts - as the successor of the GNU Library Public License, version 2, hence - the version number 2.1.] - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -Licenses are intended to guarantee your freedom to share and change -free software--to make sure the software is free for all its users. - - This license, the Lesser General Public License, applies to some -specially designated software packages--typically libraries--of the -Free Software Foundation and other authors who decide to use it. You -can use it too, but we suggest you first think carefully about whether -this license or the ordinary General Public License is the better -strategy to use in any particular case, based on the explanations below. - - When we speak of free software, we are referring to freedom of use, -not price. Our General Public Licenses are designed to make sure that -you have the freedom to distribute copies of free software (and charge -for this service if you wish); that you receive source code or can get -it if you want it; that you can change the software and use pieces of -it in new free programs; and that you are informed that you can do -these things. - - To protect your rights, we need to make restrictions that forbid -distributors to deny you these rights or to ask you to surrender these -rights. These restrictions translate to certain responsibilities for -you if you distribute copies of the library or if you modify it. - - For example, if you distribute copies of the library, whether gratis -or for a fee, you must give the recipients all the rights that we gave -you. You must make sure that they, too, receive or can get the source -code. If you link other code with the library, you must provide -complete object files to the recipients, so that they can relink them -with the library after making changes to the library and recompiling -it. And you must show them these terms so they know their rights. - - We protect your rights with a two-step method: (1) we copyright the -library, and (2) we offer you this license, which gives you legal -permission to copy, distribute and/or modify the library. - - To protect each distributor, we want to make it very clear that -there is no warranty for the free library. Also, if the library is -modified by someone else and passed on, the recipients should know -that what they have is not the original version, so that the original -author's reputation will not be affected by problems that might be -introduced by others. - - Finally, software patents pose a constant threat to the existence of -any free program. We wish to make sure that a company cannot -effectively restrict the users of a free program by obtaining a -restrictive license from a patent holder. Therefore, we insist that -any patent license obtained for a version of the library must be -consistent with the full freedom of use specified in this license. - - Most GNU software, including some libraries, is covered by the -ordinary GNU General Public License. This license, the GNU Lesser -General Public License, applies to certain designated libraries, and -is quite different from the ordinary General Public License. We use -this license for certain libraries in order to permit linking those -libraries into non-free programs. - - When a program is linked with a library, whether statically or using -a shared library, the combination of the two is legally speaking a -combined work, a derivative of the original library. The ordinary -General Public License therefore permits such linking only if the -entire combination fits its criteria of freedom. The Lesser General -Public License permits more lax criteria for linking other code with -the library. - - We call this license the "Lesser" General Public License because it -does Less to protect the user's freedom than the ordinary General -Public License. It also provides other free software developers Less -of an advantage over competing non-free programs. These disadvantages -are the reason we use the ordinary General Public License for many -libraries. However, the Lesser license provides advantages in certain -special circumstances. - - For example, on rare occasions, there may be a special need to -encourage the widest possible use of a certain library, so that it becomes -a de-facto standard. To achieve this, non-free programs must be -allowed to use the library. A more frequent case is that a free -library does the same job as widely used non-free libraries. In this -case, there is little to gain by limiting the free library to free -software only, so we use the Lesser General Public License. - - In other cases, permission to use a particular library in non-free -programs enables a greater number of people to use a large body of -free software. For example, permission to use the GNU C Library in -non-free programs enables many more people to use the whole GNU -operating system, as well as its variant, the GNU/Linux operating -system. - - Although the Lesser General Public License is Less protective of the -users' freedom, it does ensure that the user of a program that is -linked with the Library has the freedom and the wherewithal to run -that program using a modified version of the Library. - - The precise terms and conditions for copying, distribution and -modification follow. Pay close attention to the difference between a -"work based on the library" and a "work that uses the library". The -former contains code derived from the library, whereas the latter must -be combined with the library in order to run. - - GNU LESSER GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License Agreement applies to any software library or other -program which contains a notice placed by the copyright holder or -other authorized party saying it may be distributed under the terms of -this Lesser General Public License (also called "this License"). -Each licensee is addressed as "you". - - A "library" means a collection of software functions and/or data -prepared so as to be conveniently linked with application programs -(which use some of those functions and data) to form executables. - - The "Library", below, refers to any such software library or work -which has been distributed under these terms. A "work based on the -Library" means either the Library or any derivative work under -copyright law: that is to say, a work containing the Library or a -portion of it, either verbatim or with modifications and/or translated -straightforwardly into another language. (Hereinafter, translation is -included without limitation in the term "modification".) - - "Source code" for a work means the preferred form of the work for -making modifications to it. For a library, complete source code means -all the source code for all modules it contains, plus any associated -interface definition files, plus the scripts used to control compilation -and installation of the library. - - Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running a program using the Library is not restricted, and output from -such a program is covered only if its contents constitute a work based -on the Library (independent of the use of the Library in a tool for -writing it). Whether that is true depends on what the Library does -and what the program that uses the Library does. - - 1. You may copy and distribute verbatim copies of the Library's -complete source code as you receive it, in any medium, provided that -you conspicuously and appropriately publish on each copy an -appropriate copyright notice and disclaimer of warranty; keep intact -all the notices that refer to this License and to the absence of any -warranty; and distribute a copy of this License along with the -Library. - - You may charge a fee for the physical act of transferring a copy, -and you may at your option offer warranty protection in exchange for a -fee. - - 2. You may modify your copy or copies of the Library or any portion -of it, thus forming a work based on the Library, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices - stating that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no - charge to all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a - table of data to be supplied by an application program that uses - the facility, other than as an argument passed when the facility - is invoked, then you must make a good faith effort to ensure that, - in the event an application does not supply such function or - table, the facility still operates, and performs whatever part of - its purpose remains meaningful. - - (For example, a function in a library to compute square roots has - a purpose that is entirely well-defined independent of the - application. Therefore, Subsection 2d requires that any - application-supplied function or table used by this function must - be optional: if the application does not supply it, the square - root function must still compute square roots.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Library, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Library, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote -it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Library. - -In addition, mere aggregation of another work not based on the Library -with the Library (or with a work based on the Library) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may opt to apply the terms of the ordinary GNU General Public -License instead of this License to a given copy of the Library. To do -this, you must alter all the notices that refer to this License, so -that they refer to the ordinary GNU General Public License, version 2, -instead of to this License. (If a newer version than version 2 of the -ordinary GNU General Public License has appeared, then you can specify -that version instead if you wish.) Do not make any other change in -these notices. - - Once this change is made in a given copy, it is irreversible for -that copy, so the ordinary GNU General Public License applies to all -subsequent copies and derivative works made from that copy. - - This option is useful when you wish to copy part of the code of -the Library into a program that is not a library. - - 4. You may copy and distribute the Library (or a portion or -derivative of it, under Section 2) in object code or executable form -under the terms of Sections 1 and 2 above provided that you accompany -it with the complete corresponding machine-readable source code, which -must be distributed under the terms of Sections 1 and 2 above on a -medium customarily used for software interchange. - - If distribution of object code is made by offering access to copy -from a designated place, then offering equivalent access to copy the -source code from the same place satisfies the requirement to -distribute the source code, even though third parties are not -compelled to copy the source along with the object code. - - 5. A program that contains no derivative of any portion of the -Library, but is designed to work with the Library by being compiled or -linked with it, is called a "work that uses the Library". Such a -work, in isolation, is not a derivative work of the Library, and -therefore falls outside the scope of this License. - - However, linking a "work that uses the Library" with the Library -creates an executable that is a derivative of the Library (because it -contains portions of the Library), rather than a "work that uses the -library". The executable is therefore covered by this License. -Section 6 states terms for distribution of such executables. - - When a "work that uses the Library" uses material from a header file -that is part of the Library, the object code for the work may be a -derivative work of the Library even though the source code is not. -Whether this is true is especially significant if the work can be -linked without the Library, or if the work is itself a library. The -threshold for this to be true is not precisely defined by law. - - If such an object file uses only numerical parameters, data -structure layouts and accessors, and small macros and small inline -functions (ten lines or less in length), then the use of the object -file is unrestricted, regardless of whether it is legally a derivative -work. (Executables containing this object code plus portions of the -Library will still fall under Section 6.) - - Otherwise, if the work is a derivative of the Library, you may -distribute the object code for the work under the terms of Section 6. -Any executables containing that work also fall under Section 6, -whether or not they are linked directly with the Library itself. - - 6. As an exception to the Sections above, you may also combine or -link a "work that uses the Library" with the Library to produce a -work containing portions of the Library, and distribute that work -under terms of your choice, provided that the terms permit -modification of the work for the customer's own use and reverse -engineering for debugging such modifications. - - You must give prominent notice with each copy of the work that the -Library is used in it and that the Library and its use are covered by -this License. You must supply a copy of this License. If the work -during execution displays copyright notices, you must include the -copyright notice for the Library among them, as well as a reference -directing the user to the copy of this License. Also, you must do one -of these things: - - a) Accompany the work with the complete corresponding - machine-readable source code for the Library including whatever - changes were used in the work (which must be distributed under - Sections 1 and 2 above); and, if the work is an executable linked - with the Library, with the complete machine-readable "work that - uses the Library", as object code and/or source code, so that the - user can modify the Library and then relink to produce a modified - executable containing the modified Library. (It is understood - that the user who changes the contents of definitions files in the - Library will not necessarily be able to recompile the application - to use the modified definitions.) - - b) Use a suitable shared library mechanism for linking with the - Library. A suitable mechanism is one that (1) uses at run time a - copy of the library already present on the user's computer system, - rather than copying library functions into the executable, and (2) - will operate properly with a modified version of the library, if - the user installs one, as long as the modified version is - interface-compatible with the version that the work was made with. - - c) Accompany the work with a written offer, valid for at - least three years, to give the same user the materials - specified in Subsection 6a, above, for a charge no more - than the cost of performing this distribution. - - d) If distribution of the work is made by offering access to copy - from a designated place, offer equivalent access to copy the above - specified materials from the same place. - - e) Verify that the user has already received a copy of these - materials or that you have already sent this user a copy. - - For an executable, the required form of the "work that uses the -Library" must include any data and utility programs needed for -reproducing the executable from it. However, as a special exception, -the materials to be distributed need not include anything that is -normally distributed (in either source or binary form) with the major -components (compiler, kernel, and so on) of the operating system on -which the executable runs, unless that component itself accompanies -the executable. - - It may happen that this requirement contradicts the license -restrictions of other proprietary libraries that do not normally -accompany the operating system. Such a contradiction means you cannot -use both them and the Library together in an executable that you -distribute. - - 7. You may place library facilities that are a work based on the -Library side-by-side in a single library together with other library -facilities not covered by this License, and distribute such a combined -library, provided that the separate distribution of the work based on -the Library and of the other library facilities is otherwise -permitted, and provided that you do these two things: - - a) Accompany the combined library with a copy of the same work - based on the Library, uncombined with any other library - facilities. This must be distributed under the terms of the - Sections above. - - b) Give prominent notice with the combined library of the fact - that part of it is a work based on the Library, and explaining - where to find the accompanying uncombined form of the same work. - - 8. You may not copy, modify, sublicense, link with, or distribute -the Library except as expressly provided under this License. Any -attempt otherwise to copy, modify, sublicense, link with, or -distribute the Library is void, and will automatically terminate your -rights under this License. However, parties who have received copies, -or rights, from you under this License will not have their licenses -terminated so long as such parties remain in full compliance. - - 9. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Library or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Library (or any work based on the -Library), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Library or works based on it. - - 10. Each time you redistribute the Library (or any work based on the -Library), the recipient automatically receives a license from the -original licensor to copy, distribute, link with or modify the Library -subject to these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties with -this License. - - 11. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Library at all. For example, if a patent -license would not permit royalty-free redistribution of the Library by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply, -and the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 12. If the distribution and/or use of the Library is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Library under this License may add -an explicit geographical distribution limitation excluding those countries, -so that distribution is permitted only in or among countries not thus -excluded. In such case, this License incorporates the limitation as if -written in the body of this License. - - 13. The Free Software Foundation may publish revised and/or new -versions of the Lesser General Public License from time to time. -Such new versions will be similar in spirit to the present version, -but may differ in detail to address new problems or concerns. - -Each version is given a distinguishing version number. If the Library -specifies a version number of this License which applies to it and -"any later version", you have the option of following the terms and -conditions either of that version or of any later version published by -the Free Software Foundation. If the Library does not specify a -license version number, you may choose any version ever published by -the Free Software Foundation. - - 14. If you wish to incorporate parts of the Library into other free -programs whose distribution conditions are incompatible with these, -write to the author to ask for permission. For software which is -copyrighted by the Free Software Foundation, write to the Free -Software Foundation; we sometimes make exceptions for this. Our -decision will be guided by the two goals of preserving the free status -of all derivatives of our free software and of promoting the sharing -and reuse of software generally. - - NO WARRANTY - - 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO -WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. -EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR -OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY -KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE -LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME -THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN -WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY -AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU -FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR -CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE -LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING -RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A -FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF -SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH -DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Libraries - - If you develop a new library, and you want it to be of the greatest -possible use to the public, we recommend making it free software that -everyone can redistribute and change. You can do so by permitting -redistribution under these terms (or, alternatively, under the terms of the -ordinary General Public License). - - To apply these terms, attach the following notices to the library. It is -safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - -Also add information on how to contact you by electronic and paper mail. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the - library `Frob' (a library for tweaking knobs) written by James Random Hacker. - - , 1 April 1990 - Ty Coon, President of Vice - -That's all there is to it! diff --git a/dpdk/MAINTAINERS b/dpdk/MAINTAINERS index 449409a4..71ba3120 100644 --- a/dpdk/MAINTAINERS +++ b/dpdk/MAINTAINERS @@ -27,8 +27,44 @@ M: Thomas Monjalon M: Ferruh Yigit T: git://dpdk.org/dpdk +Next-net Tree +M: Ferruh Yigit +T: git://dpdk.org/next/dpdk-next-net + +Next-net-intel Tree +M: Qi Zhang +M: Beilei Xing +T: git://dpdk.org/next/dpdk-next-net-intel + +Next-net-mlx Tree +M: Shahaf Shuler +T: git://dpdk.org/next/dpdk-next-net-mlx + +Next-virtio Tree +M: Maxime Coquelin +M: Tiwei Bie +T: git://dpdk.org/next/dpdk-next-virtio + +Next-crypto Tree +M: Pablo de Lara +M: Akhil Goyal +T: git://dpdk.org/next/dpdk-next-crypto + +Next-eventdev Tree +M: Jerin Jacob +T: git://dpdk.org/next/dpdk-next-eventdev + +Next-qos Tree +M: Cristian Dumitrescu +T: git://dpdk.org/next/dpdk-next-qos + +Next-pipeline Tree +M: Cristian Dumitrescu +T: git://dpdk.org/next/dpdk-next-pipeline + Stable Branches M: Yuanhan Liu +M: Luca Boccassi T: git://dpdk.org/dpdk-stable Security Issues @@ -36,6 +72,7 @@ M: maintainers@dpdk.org Documentation (with overlaps) M: John McNamara +M: Marko Kovacevic F: README F: doc/ @@ -44,13 +81,17 @@ M: Thomas Monjalon F: MAINTAINERS F: devtools/check-dup-includes.sh F: devtools/check-maintainers.sh +F: devtools/check-forbidden-tokens.awk F: devtools/check-git-log.sh F: devtools/check-includes.sh +F: devtools/check-symbol-maps.sh F: devtools/checkpatches.sh F: devtools/get-maintainer.sh F: devtools/git-log-fixes.sh F: devtools/load-devel-config F: devtools/test-build.sh +F: devtools/test-meson-builds.sh +F: license/ Build System @@ -70,13 +111,25 @@ F: doc/guides/prog_guide/build_app.rst F: doc/guides/prog_guide/dev_kit_* F: doc/guides/prog_guide/ext_app_lib_make_help.rst +Meson build +M: Bruce Richardson +F: meson.build +F: lib/librte_eal/bsdapp/BSDmakefile.meson +F: meson_options.txt +F: config/rte_config.h +F: buildtools/gen-pmdinfo-cfile.sh +F: buildtools/symlink-drivers-solibs.sh + ABI versioning M: Neil Horman F: lib/librte_compat/ F: doc/guides/rel_notes/deprecation.rst F: devtools/validate-abi.sh +F: devtools/check-symbol-change.sh +F: buildtools/check-experimental-syms.sh Driver information +M: Neil Horman F: buildtools/pmdinfogen/ F: usertools/dpdk-pmdinfo.py F: doc/guides/tools/pmdinfo.rst @@ -93,12 +146,12 @@ F: lib/librte_eal/rte_eal_version.map F: doc/guides/prog_guide/env_abstraction_layer.rst F: test/test/test_alarm.c F: test/test/test_atomic.c +F: test/test/test_barrier.c F: test/test/test_byteorder.c F: test/test/test_common.c F: test/test/test_cpuflags.c F: test/test/test_cycles.c F: test/test/test_debug.c -F: test/test/test_devargs.c F: test/test/test_eal* F: test/test/test_errno.c F: test/test/test_interrupts.c @@ -106,6 +159,7 @@ F: test/test/test_logs.c F: test/test/test_memcpy* F: test/test/test_per_lcore.c F: test/test/test_prefetch.c +F: test/test/test_reciprocal_division* F: test/test/test_rwlock.c F: test/test/test_spinlock.c F: test/test/test_string_fns.c @@ -113,14 +167,20 @@ F: test/test/test_tailq.c F: test/test/test_version.c Memory Allocation -M: Sergio Gonzalez Monroy +M: Anatoly Burakov +F: lib/librte_eal/common/include/rte_fbarray.h F: lib/librte_eal/common/include/rte_mem* F: lib/librte_eal/common/include/rte_malloc.h F: lib/librte_eal/common/*malloc* +F: lib/librte_eal/common/eal_common_fbarray.c F: lib/librte_eal/common/eal_common_mem* F: lib/librte_eal/common/eal_hugepages.h +F: lib/librte_eal/linuxapp/eal/eal_mem* +F: lib/librte_eal/bsdapp/eal/eal_mem* F: doc/guides/prog_guide/env_abstraction_layer.rst +F: test/test/test_external_mem.c F: test/test/test_func_reentrancy.c +F: test/test/test_fbarray.c F: test/test/test_malloc.c F: test/test/test_memory.c F: test/test/test_memzone.c @@ -133,14 +193,14 @@ F: examples/l2fwd-keepalive/ F: doc/guides/sample_app_ug/keep_alive.rst Secondary process -M: Sergio Gonzalez Monroy +M: Anatoly Burakov K: RTE_PROC_ F: doc/guides/prog_guide/multi_proc_support.rst F: test/test/test_mp_secondary.c F: examples/multi_process/ F: doc/guides/sample_app_ug/multi_process.rst -Service Cores - EXPERIMENTAL +Service Cores M: Harry van Haaren F: lib/librte_eal/common/include/rte_service.h F: lib/librte_eal/common/include/rte_service_component.h @@ -173,7 +233,7 @@ F: drivers/net/ixgbe/ixgbe_rxtx_vec_neon.c F: drivers/net/i40e/i40e_rxtx_vec_neon.c F: drivers/net/virtio/virtio_rxtx_simple_neon.c -IBM POWER +IBM POWER (alpha) M: Chao Zhu F: lib/librte_eal/common/arch/ppc_64/ F: lib/librte_eal/common/include/arch/ppc_64/ @@ -193,7 +253,7 @@ F: doc/guides/linux_gsg/ Linux UIO M: Ferruh Yigit -F: lib/librte_eal/linuxapp/igb_uio/ +F: kernel/linux/igb_uio/ F: drivers/bus/pci/linux/*uio* Linux VFIO @@ -203,20 +263,17 @@ F: drivers/bus/pci/linux/*vfio* FreeBSD EAL (with overlaps) M: Bruce Richardson -M: Sergio Gonzalez Monroy F: lib/librte_eal/bsdapp/Makefile F: lib/librte_eal/bsdapp/eal/ F: doc/guides/freebsd_gsg/ FreeBSD contigmem M: Bruce Richardson -M: Sergio Gonzalez Monroy -F: lib/librte_eal/bsdapp/contigmem/ +F: kernel/freebsd/contigmem/ FreeBSD UIO M: Bruce Richardson -M: Sergio Gonzalez Monroy -F: lib/librte_eal/bsdapp/nic_uio/ +F: kernel/freebsd/nic_uio/ Core Libraries @@ -224,6 +281,7 @@ Core Libraries Memory pool M: Olivier Matz +M: Andrew Rybchenko F: lib/librte_mempool/ F: drivers/mempool/Makefile F: drivers/mempool/ring/ @@ -247,37 +305,66 @@ F: test/test/test_mbuf.c Ethernet API M: Thomas Monjalon +M: Ferruh Yigit +M: Andrew Rybchenko T: git://dpdk.org/next/dpdk-next-net -F: lib/librte_ether/ +F: lib/librte_ethdev/ F: devtools/test-null.sh Flow API M: Adrien Mazarguil T: git://dpdk.org/next/dpdk-next-net -F: lib/librte_ether/rte_flow* +F: app/test-pmd/cmdline_flow.c +F: doc/guides/prog_guide/rte_flow.rst +F: lib/librte_ethdev/rte_flow* Traffic Management API - EXPERIMENTAL M: Cristian Dumitrescu T: git://dpdk.org/next/dpdk-next-tm -F: lib/librte_ether/rte_tm* +F: lib/librte_ethdev/rte_tm* Traffic Metering and Policing API - EXPERIMENTAL M: Cristian Dumitrescu -F: lib/librte_ether/rte_mtr* +F: lib/librte_ethdev/rte_mtr* + +Baseband API - EXPERIMENTAL +M: Amr Mokhtar +T: git://dpdk.org/next/dpdk-next-crypto +F: lib/librte_bbdev/ +F: doc/guides/prog_guide/bbdev.rst +F: drivers/baseband/ +F: doc/guides/bbdevs/ +F: app/test-bbdev/ +F: doc/guides/tools/testbbdev.rst +F: examples/bbdev_app/ +F: doc/guides/sample_app_ug/bbdev_app.rst Crypto API +M: Pablo de Lara M: Declan Doherty T: git://dpdk.org/next/dpdk-next-crypto F: lib/librte_cryptodev/ F: test/test/test_cryptodev* F: examples/l2fwd-crypto/ -Security API - EXPERIMENTAL +Security API M: Akhil Goyal M: Declan Doherty +T: git://dpdk.org/next/dpdk-next-crypto F: lib/librte_security/ F: doc/guides/prog_guide/rte_security.rst +Compression API - EXPERIMENTAL +M: Fiona Trahe +M: Pablo de Lara +M: Ashish Gupta +T: git://dpdk.org/next/dpdk-next-crypto +F: lib/librte_compressdev/ +F: drivers/compress/ +F: test/test/test_compressdev* +F: doc/guides/prog_guide/compressdev.rst +F: doc/guides/compressdevs/features/default.ini + Eventdev API M: Jerin Jacob T: git://dpdk.org/next/dpdk-next-eventdev @@ -292,13 +379,56 @@ F: lib/librte_eventdev/*eth_rx_adapter* F: test/test/test_event_eth_rx_adapter.c F: doc/guides/prog_guide/event_ethernet_rx_adapter.rst +Eventdev Ethdev Tx Adapter API - EXPERIMENTAL +M: Nikhil Rao +T: git://dpdk.org/next/dpdk-next-eventdev +F: lib/librte_eventdev/*eth_tx_adapter* +F: test/test/test_event_eth_tx_adapter.c +F: doc/guides/prog_guide/event_ethernet_tx_adapter.rst + +Eventdev Timer Adapter API - EXPERIMENTAL +M: Erik Gabriel Carrillo +T: git://dpdk.org/next/dpdk-next-eventdev +F: lib/librte_eventdev/*timer_adapter* +F: test/test/test_event_timer_adapter.c +F: doc/guides/prog_guide/event_timer_adapter.rst + +Eventdev Crypto Adapter API - EXPERIMENTAL +M: Abhinandan Gujjar +T: git://dpdk.org/next/dpdk-next-eventdev +F: lib/librte_eventdev/*crypto_adapter* +F: test/test/test_event_crypto_adapter.c +F: doc/guides/prog_guide/event_crypto_adapter.rst + +Raw device API - EXPERIMENTAL +M: Shreyansh Jain +M: Hemant Agrawal +F: lib/librte_rawdev/ +F: drivers/raw/skeleton_rawdev/ +F: test/test/test_rawdev.c +F: doc/guides/prog_guide/rawdev.rst + + +Memory Pool Drivers +------------------- + +Bucket memory pool +M: Artem V. Andreev +M: Andrew Rybchenko +F: drivers/mempool/bucket/ + Bus Drivers ----------- +Intel FPGA bus +M: Rosen Xu +F: drivers/bus/ifpga/ + NXP buses M: Hemant Agrawal M: Shreyansh Jain +F: drivers/common/dpaax/ F: drivers/bus/dpaa/ F: drivers/bus/fslmc/ @@ -306,9 +436,12 @@ PCI bus driver F: drivers/bus/pci/ VDEV bus driver -M: Jianfeng Tan F: drivers/bus/vdev/ +VMBUS bus driver +M: Stephen Hemminger +F: drivers/bus/vmbus/ + Networking Drivers ------------------ @@ -318,6 +451,7 @@ F: doc/guides/nics/features/default.ini Link bonding M: Declan Doherty +M: Chas Williams F: drivers/net/bonding/ F: doc/guides/prog_guide/link_bonding_poll_mode_drv_lib.rst F: test/test/test_link_bonding* @@ -326,7 +460,7 @@ F: doc/guides/nics/features/bonding.ini Linux KNI M: Ferruh Yigit -F: lib/librte_eal/linuxapp/kni/ +F: kernel/linux/kni/ F: lib/librte_kni/ F: doc/guides/prog_guide/kernel_nic_interface.rst F: test/test/test_kni.c @@ -347,6 +481,19 @@ F: drivers/net/ena/ F: doc/guides/nics/ena.rst F: doc/guides/nics/features/ena.ini +AMD axgbe +M: Ravi Kumar +F: drivers/net/axgbe/ +F: doc/guides/nics/axgbe.rst +F: doc/guides/nics/features/axgbe.ini + +Aquantia atlantic +M: Igor Russkikh +M: Pavel Belous +F: drivers/net/atlantic/ +F: doc/guides/nics/atlantic.rst +F: doc/guides/nics/features/atlantic.ini + Atomic Rules ARK M: Shepard Siegel M: Ed Czeck @@ -356,8 +503,8 @@ F: doc/guides/nics/ark.rst F: doc/guides/nics/features/ark.ini Broadcom bnxt -M: Stephen Hurd M: Ajit Khaparde +M: Somnath Kotur F: drivers/net/bnxt/ F: doc/guides/nics/bnxt.rst F: doc/guides/nics/features/bnxt.ini @@ -376,9 +523,9 @@ F: drivers/net/liquidio/ F: doc/guides/nics/liquidio.rst F: doc/guides/nics/features/liquidio.ini -Cavium OcteonTX -M: Santosh Shukla +Cavium OCTEON TX M: Jerin Jacob +F: drivers/common/octeontx/ F: drivers/mempool/octeontx/ F: drivers/net/octeontx/ F: doc/guides/nics/octeontx.rst @@ -392,13 +539,14 @@ F: doc/guides/nics/features/cxgbe.ini Cisco enic M: John Daley -M: Nelson Escobar +M: Hyong Youb Kim F: drivers/net/enic/ F: doc/guides/nics/enic.rst F: doc/guides/nics/features/enic.ini Intel e1000 M: Wenzhuo Lu +T: git://dpdk.org/next/dpdk-next-net-intel F: drivers/net/e1000/ F: doc/guides/nics/e1000em.rst F: doc/guides/nics/intel_vf.rst @@ -408,14 +556,16 @@ F: doc/guides/nics/features/igb*.ini Intel ixgbe M: Wenzhuo Lu M: Konstantin Ananyev +T: git://dpdk.org/next/dpdk-next-net-intel F: drivers/net/ixgbe/ F: doc/guides/nics/ixgbe.rst F: doc/guides/nics/intel_vf.rst F: doc/guides/nics/features/ixgbe*.ini Intel i40e -M: Jingjing Wu M: Beilei Xing +M: Qi Zhang +T: git://dpdk.org/next/dpdk-next-net-intel F: drivers/net/i40e/ F: doc/guides/nics/i40e.rst F: doc/guides/nics/intel_vf.rst @@ -424,12 +574,46 @@ F: doc/guides/nics/features/i40e*.ini Intel fm10k M: Qi Zhang M: Xiao Wang +T: git://dpdk.org/next/dpdk-next-net-intel F: drivers/net/fm10k/ +F: doc/guides/nics/fm10k.rst F: doc/guides/nics/features/fm10k*.ini +Intel avf +M: Jingjing Wu +M: Wenzhuo Lu +T: git://dpdk.org/next/dpdk-next-net-intel +F: drivers/net/avf/ +F: doc/guides/nics/features/avf*.ini + +Intel ifc +M: Xiao Wang +T: git://dpdk.org/next/dpdk-next-net-intel +F: drivers/net/ifc/ +F: doc/guides/nics/ifc.rst +F: doc/guides/nics/features/ifc*.ini + +Marvell mvpp2 +M: Tomasz Duszynski +M: Dmitri Epshtein +M: Natalie Samsonov +F: drivers/common/mvep/ +F: drivers/net/mvpp2/ +F: doc/guides/nics/mvpp2.rst +F: doc/guides/nics/features/mvpp2.ini + +Marvell mvneta +M: Zyta Szpak +M: Dmitri Epshtein +M: Natalie Samsonov +F: drivers/net/mvneta/ +F: doc/guides/nics/mvneta.rst +F: doc/guides/nics/features/mvneta.ini + Mellanox mlx4 M: Matan Azrad M: Shahaf Shuler +T: git://dpdk.org/next/dpdk-next-net-mlx F: drivers/net/mlx4/ F: doc/guides/nics/mlx4.rst F: doc/guides/nics/features/mlx4.ini @@ -437,21 +621,27 @@ F: doc/guides/nics/features/mlx4.ini Mellanox mlx5 M: Shahaf Shuler M: Yongseok Koh +T: git://dpdk.org/next/dpdk-next-net-mlx F: drivers/net/mlx5/ F: doc/guides/nics/mlx5.rst F: doc/guides/nics/features/mlx5.ini -Marvell mrvl -M: Jacek Siuda -M: Tomasz Duszynski -M: Dmitri Epshtein -M: Natalie Samsonov -F: drivers/net/mrvl/ -F: doc/guides/nics/mrvl.rst -F: doc/guides/nics/features/mrvl.ini +Microsoft vdev_netvsc - EXPERIMENTAL +M: Matan Azrad +F: drivers/net/vdev_netvsc/ +F: doc/guides/nics/vdev_netvsc.rst +F: doc/guides/nics/features/vdev_netvsc.ini + +Microsoft Hyper-V netvsc - EXPERIMENTAL +M: Stephen Hemminger +M: K. Y. Srinivasan +M: Haiyang Zhang +F: drivers/net/netvsc/ +F: doc/guides/nics/netvsc.rst +F: doc/guides/nics/features/netvsc.ini Netcope szedata2 -M: Matej Vido +M: Jan Remes F: drivers/net/szedata2/ F: doc/guides/nics/szedata2.rst F: doc/guides/nics/features/szedata2.ini @@ -478,6 +668,13 @@ F: drivers/net/dpaa2/ F: doc/guides/nics/dpaa2.rst F: doc/guides/nics/features/dpaa2.ini +NXP enetc +M: Gagandeep Singh +M: Pankaj Chauhan +F: drivers/net/enetc/ +F: doc/guides/nics/enetc.rst +F: doc/guides/nics/features/enetc.ini + QLogic bnx2x M: Harish Patil M: Rasesh Mody @@ -500,14 +697,15 @@ F: doc/guides/nics/sfc_efx.rst F: doc/guides/nics/features/sfc_efx.ini VMware vmxnet3 -M: Shrikrishna Khare +M: Yong Wang F: drivers/net/vmxnet3/ F: doc/guides/nics/vmxnet3.rst F: doc/guides/nics/features/vmxnet3.ini Vhost-user -M: Yuanhan Liu M: Maxime Coquelin +M: Tiwei Bie +M: Zhihong Wang T: git://dpdk.org/next/dpdk-next-virtio F: lib/librte_vhost/ F: doc/guides/prog_guide/vhost_lib.rst @@ -515,18 +713,23 @@ F: examples/vhost/ F: doc/guides/sample_app_ug/vhost.rst F: examples/vhost_scsi/ F: doc/guides/sample_app_ug/vhost_scsi.rst +F: examples/vhost_crypto/ +F: examples/vdpa/ +F: doc/guides/sample_app_ug/vdpa.rst Vhost PMD -M: Tetsuya Mukawa -M: Yuanhan Liu M: Maxime Coquelin +M: Tiwei Bie +M: Zhihong Wang T: git://dpdk.org/next/dpdk-next-virtio F: drivers/net/vhost/ +F: doc/guides/nics/vhost.rst F: doc/guides/nics/features/vhost.ini Virtio PMD -M: Yuanhan Liu M: Maxime Coquelin +M: Tiwei Bie +M: Zhihong Wang T: git://dpdk.org/next/dpdk-next-virtio F: drivers/net/virtio/ F: doc/guides/nics/virtio.rst @@ -546,7 +749,7 @@ F: doc/guides/nics/pcap_ring.rst F: doc/guides/nics/features/pcap.ini Tap PMD -M: Pascal Mazon +M: Keith Wiles F: drivers/net/tap/ F: doc/guides/nics/tap.rst F: doc/guides/nics/features/tap.ini @@ -574,11 +777,14 @@ Fail-safe PMD M: Gaetan Rivet F: drivers/net/failsafe/ F: doc/guides/nics/fail_safe.rst +F: doc/guides/nics/features/failsafe.ini Softnic PMD M: Jasvinder Singh M: Cristian Dumitrescu F: drivers/net/softnic/ +F: doc/guides/nics/features/softnic.ini +F: doc/guides/nics/softnic.rst Crypto Drivers @@ -587,12 +793,25 @@ M: Pablo de Lara T: git://dpdk.org/next/dpdk-next-crypto F: doc/guides/cryptodevs/features/default.ini +AMD CCP Crypto +M: Ravi Kumar +F: drivers/crypto/ccp/ +F: doc/guides/cryptodevs/ccp.rst +F: doc/guides/cryptodevs/features/ccp.ini + ARMv8 Crypto M: Jerin Jacob F: drivers/crypto/armv8/ F: doc/guides/cryptodevs/armv8.rst F: doc/guides/cryptodevs/features/armv8.ini +Cavium OCTEON TX crypto +M: Anoob Joseph +F: drivers/common/cpt/ +F: drivers/crypto/octeontx/ +F: doc/guides/cryptodevs/octeontx.rst +F: doc/guides/cryptodevs/features/octeontx.ini + Crypto Scheduler M: Fan Zhang F: drivers/crypto/scheduler/ @@ -615,6 +834,7 @@ M: John Griffin M: Fiona Trahe M: Deepak Kumar Jain F: drivers/crypto/qat/ +F: drivers/common/qat/ F: doc/guides/cryptodevs/qat.rst F: doc/guides/cryptodevs/features/qat.ini @@ -625,13 +845,12 @@ F: doc/guides/cryptodevs/kasumi.rst F: doc/guides/cryptodevs/features/kasumi.ini Marvell Mrvl -M: Jacek Siuda M: Tomasz Duszynski M: Dmitri Epshtein M: Natalie Samsonov -F: drivers/crypto/mrvl/ -F: doc/guides/cryptodevs/mrvl.rst -F: doc/guides/cryptodevs/features/mrvl.ini +F: drivers/crypto/mvsam/ +F: doc/guides/cryptodevs/mvsam.rst +F: doc/guides/cryptodevs/features/mvsam.ini Null Crypto M: Declan Doherty @@ -639,6 +858,13 @@ F: drivers/crypto/null/ F: doc/guides/cryptodevs/null.rst F: doc/guides/cryptodevs/features/null.ini +NXP CAAM JR +M: Gagandeep Singh +M: Hemant Agrawal +F: drivers/crypto/caam_jr/ +F: doc/guides/cryptodevs/caam_jr.rst +F: doc/guides/cryptodevs/features/caam_jr.ini + NXP DPAA_SEC M: Akhil Goyal M: Hemant Agrawal @@ -665,6 +891,12 @@ F: drivers/crypto/snow3g/ F: doc/guides/cryptodevs/snow3g.rst F: doc/guides/cryptodevs/features/snow3g.ini +Virtio +M: Jay Zhou +F: drivers/crypto/virtio/ +F: doc/guides/cryptodevs/virtio.rst +F: doc/guides/cryptodevs/features/virtio.ini + ZUC M: Pablo de Lara F: drivers/crypto/zuc/ @@ -672,18 +904,55 @@ F: doc/guides/cryptodevs/zuc.rst F: doc/guides/cryptodevs/features/zuc.ini +Compression Drivers +------------------- +M: Pablo de Lara +T: git://dpdk.org/next/dpdk-next-crypto + +Cavium OCTEON TX zipvf +M: Ashish Gupta +F: drivers/compress/octeontx/ +F: doc/guides/compressdevs/octeontx.rst +F: doc/guides/compressdevs/features/octeontx.ini + +Intel QuickAssist +M: Fiona Trahe +F: drivers/compress/qat/ +F: drivers/common/qat/ + +ISA-L +M: Lee Daly +F: drivers/compress/isal/ +F: doc/guides/compressdevs/isal.rst +F: doc/guides/compressdevs/features/isal.ini + +ZLIB +M: Sunila Sahu +F: drivers/compress/zlib/ +F: doc/guides/compressdevs/zlib.rst +F: doc/guides/compressdevs/features/zlib.ini + + Eventdev Drivers ---------------- M: Jerin Jacob T: git://dpdk.org/next/dpdk-next-eventdev -Cavium OCTEONTX ssovf +Cavium OCTEON TX ssovf M: Jerin Jacob -M: Santosh Shukla F: drivers/event/octeontx/ -F: test/test/test_eventdev_octeontx.c F: doc/guides/eventdevs/octeontx.rst +Cavium OCTEON TX timvf +M: Pavan Nikhilesh +F: drivers/event/octeontx/timvf_* + +NXP DPAA eventdev +M: Hemant Agrawal +M: Sunil Kumar Kori +F: drivers/event/dpaa/ +F: doc/guides/eventdevs/dpaa.rst + NXP DPAA2 eventdev M: Hemant Agrawal M: Nipun Gupta @@ -693,10 +962,40 @@ F: doc/guides/eventdevs/dpaa2.rst Software Eventdev PMD M: Harry van Haaren F: drivers/event/sw/ -F: test/test/test_eventdev_sw.c F: doc/guides/eventdevs/sw.rst -F: examples/eventdev_pipeline_sw_pmd/ -F: doc/guides/sample_app_ug/eventdev_pipeline_sw_pmd.rst +F: examples/eventdev_pipeline/ +F: doc/guides/sample_app_ug/eventdev_pipeline.rst + +Distributed Software Eventdev PMD +M: Mattias Rƶnnblom +F: drivers/event/dsw/ +F: doc/guides/eventdevs/dsw.rst + +Software OPDL Eventdev PMD +M: Liang Ma +M: Peter Mccarthy +F: drivers/event/opdl/ +F: doc/guides/eventdevs/opdl.rst + + +Rawdev Drivers +-------------- + +Intel FPGA +M: Rosen Xu +M: Tianfei zhang +F: drivers/raw/ifpga_rawdev/ +F: doc/guides/rawdevs/ifpga_rawdev.rst + +NXP DPAA2 QDMA +M: Nipun Gupta +F: drivers/raw/dpaa2_qdma/ +F: doc/guides/rawdevs/dpaa2_qdma.rst + +DPAA2 CMDIF +M: Nipun Gupta +F: drivers/raw/dpaa2_cmdif/ +F: doc/guides/rawdevs/dpaa2_cmdif.rst Packet processing @@ -728,7 +1027,6 @@ F: doc/guides/prog_guide/generic_receive_offload_lib.rst Generic Segmentation Offload M: Jiayu Hu -M: Mark Kavanagh F: lib/librte_gso/ F: doc/guides/prog_guide/generic_segmentation_offload_lib.rst @@ -773,6 +1071,7 @@ F: doc/guides/prog_guide/pdump_lib.rst F: app/pdump/ F: doc/guides/tools/pdump.rst + Packet Framework ---------------- M: Cristian Dumitrescu @@ -839,6 +1138,7 @@ F: test/test/test_meter.c F: examples/qos_meter/ F: doc/guides/sample_app_ug/qos_metering.rst + Other libraries --------------- @@ -892,6 +1192,7 @@ F: doc/guides/sample_app_ug/l2_forward_job_stats.rst Metrics M: Remy Horton F: lib/librte_metrics/ +F: test/test/test_metrics.c Bit-rate statistics M: Remy Horton @@ -901,6 +1202,19 @@ Latency statistics M: Reshma Pattan F: lib/librte_latencystats/ +Telemetry - EXPERIMENTAL +M: Kevin Laatz +F: lib/librte_telemetry/ +F: usertools/dpdk-telemetry-client.py +F: doc/guides/howto/telemetry.rst + +BPF - EXPERIMENTAL +M: Konstantin Ananyev +F: lib/librte_bpf/ +F: test/bpf/ +F: test/test/test_bpf.c +F: doc/guides/prog_guide/bpf_lib.rst + Test Applications ----------------- @@ -922,7 +1236,9 @@ F: test/test/virtual_pmd.c F: test/test/virtual_pmd.h Driver testing tool +M: Wenzhuo Lu M: Jingjing Wu +M: Bernard Iremonger F: app/test-pmd/ F: doc/guides/testpmd_app_ug/ @@ -941,7 +1257,7 @@ F: test/test/test_event_ring.c Procinfo tool M: Maryam Tahhan M: Reshma Pattan -F: app/proc_info/ +F: app/proc-info/ F: doc/guides/tools/proc_info.rst @@ -955,6 +1271,10 @@ F: doc/guides/sample_app_ug/ethtool.rst F: examples/exception_path/ F: doc/guides/sample_app_ug/exception_path.rst +M: Marko Kovacevic +F: examples/fips_validation/ +F: doc/guides/sample_app_ug/fips_validation.rst + M: Ori Kam F: examples/flow_filtering/ F: doc/guides/sample_app_ug/flow_filtering.rst @@ -964,8 +1284,8 @@ M: Pablo de Lara F: examples/helloworld/ F: doc/guides/sample_app_ug/hello_world.rst -M: Sergio Gonzalez Monroy M: Radu Nicolau +M: Akhil Goyal F: examples/ipsec-secgw/ F: doc/guides/sample_app_ug/ipsec_secgw.rst @@ -1021,7 +1341,7 @@ M: John McNamara F: examples/skeleton/ F: doc/guides/sample_app_ug/skeleton.rst -M: Jijiang Liu +M: Xiaoyun Li F: examples/tep_termination/ F: examples/vmdq/ diff --git a/dpdk/Makefile b/dpdk/Makefile index f4b807eb..6cd70a09 100644 --- a/dpdk/Makefile +++ b/dpdk/Makefile @@ -1,33 +1,5 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation .error Error please compile using GNU Make (gmake) diff --git a/dpdk/README b/dpdk/README index 29ba0e0e..55df1586 100644 --- a/dpdk/README +++ b/dpdk/README @@ -1,8 +1,8 @@ DPDK is a set of libraries and drivers for fast packet processing. It supports many processor architectures and both FreeBSD and Linux. -The DPDK uses the Open Source BSD license for the core libraries and -drivers. The kernel components are GPLv2 licensed. +The DPDK uses the Open Source BSD-3-Clause license for the core libraries +and drivers. The kernel components are GPL-2.0 licensed. Please check the doc directory for release notes, API documentation, and sample application information. diff --git a/dpdk/app/Makefile b/dpdk/app/Makefile index 6398c184..069fa984 100644 --- a/dpdk/app/Makefile +++ b/dpdk/app/Makefile @@ -1,40 +1,16 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2014 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation include $(RTE_SDK)/mk/rte.vars.mk DIRS-$(CONFIG_RTE_TEST_PMD) += test-pmd -DIRS-$(CONFIG_RTE_PROC_INFO) += proc_info +DIRS-$(CONFIG_RTE_PROC_INFO) += proc-info DIRS-$(CONFIG_RTE_LIBRTE_PDUMP) += pdump +ifeq ($(CONFIG_RTE_LIBRTE_BBDEV),y) +DIRS-$(CONFIG_RTE_TEST_BBDEV) += test-bbdev +endif + ifeq ($(CONFIG_RTE_LIBRTE_CRYPTODEV),y) DIRS-$(CONFIG_RTE_APP_CRYPTO_PERF) += test-crypto-perf endif diff --git a/dpdk/app/meson.build b/dpdk/app/meson.build new file mode 100644 index 00000000..a9a026bb --- /dev/null +++ b/dpdk/app/meson.build @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +apps = ['pdump', + 'proc-info', + 'test-bbdev', + 'test-crypto-perf', + 'test-eventdev', + 'test-pmd'] + +# for BSD only +lib_execinfo = cc.find_library('execinfo', required: false) + +default_cflags = machine_args + +# specify -D_GNU_SOURCE unconditionally +default_cflags += '-D_GNU_SOURCE' + +foreach app:apps + build = true + name = app + allow_experimental_apis = false + sources = [] + includes = [] + cflags = default_cflags + objs = [] # other object files to link against, used e.g. for + # instruction-set optimized versions of code + + # use "deps" for internal DPDK dependencies, and "ext_deps" for + # external package/library requirements + ext_deps = [] + deps = dpdk_app_link_libraries + + subdir(name) + + if build + dep_objs = [] + foreach d:deps + dep_objs += get_variable(get_option('default_library') + + '_rte_' + d) + endforeach + dep_objs += lib_execinfo + + link_libs = [] + if get_option('default_library') == 'static' + link_libs = dpdk_static_libraries + dpdk_drivers + endif + + if allow_experimental_apis + cflags += '-DALLOW_EXPERIMENTAL_API' + endif + + executable('dpdk-' + name, + sources, + c_args: cflags, + link_whole: link_libs, + dependencies: dep_objs, + install_rpath: join_paths(get_option('prefix'), + driver_install_path), + install: true) + endif +endforeach diff --git a/dpdk/app/pdump/Makefile b/dpdk/app/pdump/Makefile index 38ac3e9a..bd3c2081 100644 --- a/dpdk/app/pdump/Makefile +++ b/dpdk/app/pdump/Makefile @@ -1,33 +1,5 @@ -# BSD LICENSE -# -# Copyright(c) 2016 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2016 Intel Corporation include $(RTE_SDK)/mk/rte.vars.mk @@ -35,6 +7,7 @@ ifeq ($(CONFIG_RTE_LIBRTE_PDUMP),y) APP = dpdk-pdump +CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += $(WERROR_FLAGS) # all source are stored in SRCS-y diff --git a/dpdk/app/pdump/main.c b/dpdk/app/pdump/main.c index 66272f59..9e86bf62 100644 --- a/dpdk/app/pdump/main.c +++ b/dpdk/app/pdump/main.c @@ -1,34 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright(c) 2016 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016 Intel Corporation */ #include @@ -53,6 +24,7 @@ #include #include #include +#include #include #define CMD_LINE_OPT_PDUMP "pdump" @@ -65,11 +37,10 @@ #define PDUMP_RING_SIZE_ARG "ring-size" #define PDUMP_MSIZE_ARG "mbuf-size" #define PDUMP_NUM_MBUFS_ARG "total-num-mbufs" -#define CMD_LINE_OPT_SER_SOCK_PATH "server-socket-path" -#define CMD_LINE_OPT_CLI_SOCK_PATH "client-socket-path" -#define VDEV_PCAP "net_pcap_%s_%d,tx_pcap=%s" -#define VDEV_IFACE "net_pcap_%s_%d,tx_iface=%s" +#define VDEV_NAME_FMT "net_pcap_%s_%d" +#define VDEV_PCAP_ARGS_FMT "tx_pcap=%s" +#define VDEV_IFACE_ARGS_FMT "tx_iface=%s" #define TX_STREAM_SIZE 64 #define MP_NAME "pdump_pool_%d" @@ -110,7 +81,7 @@ enum pdump_by { DEVICE_ID = 2 }; -const char *valid_pdump_arguments[] = { +static const char * const valid_pdump_arguments[] = { PDUMP_PORT_ARG, PDUMP_PCI_ARG, PDUMP_QUEUE_ARG, @@ -148,8 +119,8 @@ struct pdump_tuples { /* params for packet dumping */ enum pdump_by dump_by_type; - int rx_vdev_id; - int tx_vdev_id; + uint16_t rx_vdev_id; + uint16_t tx_vdev_id; enum pcap_stream rx_vdev_stream_type; enum pcap_stream tx_vdev_stream_type; bool single_pdump_dev; @@ -165,11 +136,9 @@ struct parse_val { uint64_t val; }; -int num_tuples; +static int num_tuples; static struct rte_eth_conf port_conf_default; -volatile uint8_t quit_signal; -static char server_socket_path[PATH_MAX]; -static char client_socket_path[PATH_MAX]; +static volatile uint8_t quit_signal; /**< display usage */ static void @@ -182,11 +151,7 @@ pdump_usage(const char *prgname) " tx-dev=," "[ring-size=default:16384]," "[mbuf-size=default:2176]," - "[total-num-mbufs=default:65535]'\n" - "[--server-socket-path=" - "default:/var/run/.dpdk/ (or) ~/.dpdk/]\n" - "[--client-socket-path=" - "default:/var/run/.dpdk/ (or) ~/.dpdk/]\n", + "[total-num-mbufs=default:65535]'\n", prgname); } @@ -301,7 +266,7 @@ parse_pdump(const char *optarg) &parse_uint_value, &v); if (ret < 0) goto free_kvlist; - pt->port = (uint8_t) v.val; + pt->port = (uint16_t) v.val; pt->dump_by_type = PORT_ID; } else if (cnt2 == 1) { ret = rte_kvargs_process(kvlist, PDUMP_PCI_ARG, @@ -411,8 +376,6 @@ launch_args_parse(int argc, char **argv, char *prgname) int option_index; static struct option long_option[] = { {"pdump", 1, 0, 0}, - {"server-socket-path", 1, 0, 0}, - {"client-socket-path", 1, 0, 0}, {NULL, 0, 0, 0} }; @@ -433,23 +396,6 @@ launch_args_parse(int argc, char **argv, char *prgname) return -1; } } - - if (!strncmp(long_option[option_index].name, - CMD_LINE_OPT_SER_SOCK_PATH, - sizeof(CMD_LINE_OPT_SER_SOCK_PATH))) { - snprintf(server_socket_path, - sizeof(server_socket_path), "%s", - optarg); - } - - if (!strncmp(long_option[option_index].name, - CMD_LINE_OPT_CLI_SOCK_PATH, - sizeof(CMD_LINE_OPT_CLI_SOCK_PATH))) { - snprintf(client_socket_path, - sizeof(client_socket_path), "%s", - optarg); - } - break; default: pdump_usage(prgname); @@ -489,7 +435,7 @@ disable_pdump(struct pdump_tuples *pt) } static inline void -pdump_rxtx(struct rte_ring *ring, uint8_t vdev_id, struct pdump_stats *stats) +pdump_rxtx(struct rte_ring *ring, uint16_t vdev_id, struct pdump_stats *stats) { /* write input packets of port to vdev for pdump */ struct rte_mbuf *rxtx_bufs[BURST_SIZE]; @@ -516,7 +462,7 @@ pdump_rxtx(struct rte_ring *ring, uint8_t vdev_id, struct pdump_stats *stats) } static void -free_ring_data(struct rte_ring *ring, uint8_t vdev_id, +free_ring_data(struct rte_ring *ring, uint16_t vdev_id, struct pdump_stats *stats) { while (rte_ring_count(ring)) @@ -583,11 +529,10 @@ configure_vdev(uint16_t port_id) { struct ether_addr addr; const uint16_t rxRings = 0, txRings = 1; - const uint8_t nb_ports = rte_eth_dev_count(); int ret; uint16_t q; - if (port_id > nb_ports) + if (!rte_eth_dev_is_valid_port(port_id)) return -1; ret = rte_eth_dev_configure(port_id, rxRings, txRings, @@ -626,6 +571,7 @@ create_mp_ring_vdev(void) uint16_t portid; struct pdump_tuples *pt = NULL; struct rte_mempool *mbuf_pool = NULL; + char vdev_name[SIZE]; char vdev_args[SIZE]; char ring_name[SIZE]; char mempool_name[SIZE]; @@ -675,17 +621,28 @@ create_mp_ring_vdev(void) } /* create vdevs */ + snprintf(vdev_name, sizeof(vdev_name), + VDEV_NAME_FMT, RX_STR, i); (pt->rx_vdev_stream_type == IFACE) ? - snprintf(vdev_args, SIZE, VDEV_IFACE, RX_STR, i, - pt->rx_dev) : - snprintf(vdev_args, SIZE, VDEV_PCAP, RX_STR, i, - pt->rx_dev); - if (rte_eth_dev_attach(vdev_args, &portid) < 0) { + snprintf(vdev_args, sizeof(vdev_args), + VDEV_IFACE_ARGS_FMT, pt->rx_dev) : + snprintf(vdev_args, sizeof(vdev_args), + VDEV_PCAP_ARGS_FMT, pt->rx_dev); + if (rte_eal_hotplug_add("vdev", vdev_name, + vdev_args) < 0) { cleanup_rings(); rte_exit(EXIT_FAILURE, "vdev creation failed:%s:%d\n", __func__, __LINE__); } + if (rte_eth_dev_get_port_by_name(vdev_name, + &portid) != 0) { + rte_eal_hotplug_remove("vdev", vdev_name); + cleanup_rings(); + rte_exit(EXIT_FAILURE, + "cannot find added vdev %s:%s:%d\n", + vdev_name, __func__, __LINE__); + } pt->rx_vdev_id = portid; /* configure vdev */ @@ -694,18 +651,29 @@ create_mp_ring_vdev(void) if (pt->single_pdump_dev) pt->tx_vdev_id = portid; else { - (pt->tx_vdev_stream_type == IFACE) ? - snprintf(vdev_args, SIZE, VDEV_IFACE, TX_STR, i, - pt->tx_dev) : - snprintf(vdev_args, SIZE, VDEV_PCAP, TX_STR, i, - pt->tx_dev); - if (rte_eth_dev_attach(vdev_args, - &portid) < 0) { + snprintf(vdev_name, sizeof(vdev_name), + VDEV_NAME_FMT, TX_STR, i); + (pt->rx_vdev_stream_type == IFACE) ? + snprintf(vdev_args, sizeof(vdev_args), + VDEV_IFACE_ARGS_FMT, pt->tx_dev) : + snprintf(vdev_args, sizeof(vdev_args), + VDEV_PCAP_ARGS_FMT, pt->tx_dev); + if (rte_eal_hotplug_add("vdev", vdev_name, + vdev_args) < 0) { cleanup_rings(); rte_exit(EXIT_FAILURE, "vdev creation failed:" "%s:%d\n", __func__, __LINE__); } + if (rte_eth_dev_get_port_by_name(vdev_name, + &portid) != 0) { + rte_eal_hotplug_remove("vdev", + vdev_name); + cleanup_rings(); + rte_exit(EXIT_FAILURE, + "cannot find added vdev %s:%s:%d\n", + vdev_name, __func__, __LINE__); + } pt->tx_vdev_id = portid; /* configure vdev */ @@ -723,17 +691,28 @@ create_mp_ring_vdev(void) rte_strerror(rte_errno)); } + snprintf(vdev_name, sizeof(vdev_name), + VDEV_NAME_FMT, RX_STR, i); (pt->rx_vdev_stream_type == IFACE) ? - snprintf(vdev_args, SIZE, VDEV_IFACE, RX_STR, i, - pt->rx_dev) : - snprintf(vdev_args, SIZE, VDEV_PCAP, RX_STR, i, - pt->rx_dev); - if (rte_eth_dev_attach(vdev_args, &portid) < 0) { + snprintf(vdev_args, sizeof(vdev_args), + VDEV_IFACE_ARGS_FMT, pt->rx_dev) : + snprintf(vdev_args, sizeof(vdev_args), + VDEV_PCAP_ARGS_FMT, pt->rx_dev); + if (rte_eal_hotplug_add("vdev", vdev_name, + vdev_args) < 0) { cleanup_rings(); rte_exit(EXIT_FAILURE, "vdev creation failed:%s:%d\n", __func__, __LINE__); } + if (rte_eth_dev_get_port_by_name(vdev_name, + &portid) != 0) { + rte_eal_hotplug_remove("vdev", vdev_name); + cleanup_rings(); + rte_exit(EXIT_FAILURE, + "cannot find added vdev %s:%s:%d\n", + vdev_name, __func__, __LINE__); + } pt->rx_vdev_id = portid; /* configure vdev */ configure_vdev(pt->rx_vdev_id); @@ -749,16 +728,27 @@ create_mp_ring_vdev(void) rte_strerror(rte_errno)); } + snprintf(vdev_name, sizeof(vdev_name), + VDEV_NAME_FMT, TX_STR, i); (pt->tx_vdev_stream_type == IFACE) ? - snprintf(vdev_args, SIZE, VDEV_IFACE, TX_STR, i, - pt->tx_dev) : - snprintf(vdev_args, SIZE, VDEV_PCAP, TX_STR, i, - pt->tx_dev); - if (rte_eth_dev_attach(vdev_args, &portid) < 0) { + snprintf(vdev_args, sizeof(vdev_args), + VDEV_IFACE_ARGS_FMT, pt->tx_dev) : + snprintf(vdev_args, sizeof(vdev_args), + VDEV_PCAP_ARGS_FMT, pt->tx_dev); + if (rte_eal_hotplug_add("vdev", vdev_name, + vdev_args) < 0) { cleanup_rings(); rte_exit(EXIT_FAILURE, "vdev creation failed\n"); } + if (rte_eth_dev_get_port_by_name(vdev_name, + &portid) != 0) { + rte_eal_hotplug_remove("vdev", vdev_name); + cleanup_rings(); + rte_exit(EXIT_FAILURE, + "cannot find added vdev %s:%s:%d\n", + vdev_name, __func__, __LINE__); + } pt->tx_vdev_id = portid; /* configure vdev */ @@ -774,22 +764,6 @@ enable_pdump(void) struct pdump_tuples *pt; int ret = 0, ret1 = 0; - if (server_socket_path[0] != 0) - ret = rte_pdump_set_socket_dir(server_socket_path, - RTE_PDUMP_SOCKET_SERVER); - if (ret == 0 && client_socket_path[0] != 0) { - ret = rte_pdump_set_socket_dir(client_socket_path, - RTE_PDUMP_SOCKET_CLIENT); - } - if (ret < 0) { - cleanup_pdump_resources(); - rte_exit(EXIT_FAILURE, - "failed to set socket paths of server:%s, " - "client:%s\n", - server_socket_path, - client_socket_path); - } - for (i = 0; i < num_tuples; i++) { pt = &pdump_t[i]; if (pt->dir == RTE_PDUMP_FLAG_RXTX) { @@ -892,6 +866,9 @@ main(int argc, char **argv) if (diag < 0) rte_panic("Cannot init EAL\n"); + if (rte_eth_dev_count_avail() == 0) + rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n"); + argc -= diag; argv += (diag - 3); @@ -911,5 +888,9 @@ main(int argc, char **argv) /* dump debug stats */ print_pdump_stats(); + ret = rte_eal_cleanup(); + if (ret) + printf("Error from rte_eal_cleanup(), %d\n", ret); + return 0; } diff --git a/dpdk/app/pdump/meson.build b/dpdk/app/pdump/meson.build new file mode 100644 index 00000000..116c27f0 --- /dev/null +++ b/dpdk/app/pdump/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Intel Corporation + +sources = files('main.c') +allow_experimental_apis = true +deps += ['ethdev', 'kvargs', 'pdump'] diff --git a/dpdk/app/proc-info/Makefile b/dpdk/app/proc-info/Makefile new file mode 100644 index 00000000..9e87f524 --- /dev/null +++ b/dpdk/app/proc-info/Makefile @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2015 Intel Corporation + +include $(RTE_SDK)/mk/rte.vars.mk + +APP = dpdk-procinfo + +CFLAGS += -DALLOW_EXPERIMENTAL_API +CFLAGS += $(WERROR_FLAGS) + +# all source are stored in SRCS-y + +SRCS-y := main.c + +include $(RTE_SDK)/mk/rte.app.mk diff --git a/dpdk/app/proc-info/main.c b/dpdk/app/proc-info/main.c new file mode 100644 index 00000000..c20effa4 --- /dev/null +++ b/dpdk/app/proc-info/main.c @@ -0,0 +1,672 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2017 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Maximum long option length for option parsing. */ +#define MAX_LONG_OPT_SZ 64 +#define RTE_LOGTYPE_APP RTE_LOGTYPE_USER1 + +#define MAX_STRING_LEN 256 + +/**< mask of enabled ports */ +static uint32_t enabled_port_mask; +/**< Enable stats. */ +static uint32_t enable_stats; +/**< Enable xstats. */ +static uint32_t enable_xstats; +/**< Enable collectd format*/ +static uint32_t enable_collectd_format; +/**< FD to send collectd format messages to STDOUT*/ +static int stdout_fd; +/**< Host id process is running on */ +static char host_id[MAX_LONG_OPT_SZ]; +/**< Enable metrics. */ +static uint32_t enable_metrics; +/**< Enable stats reset. */ +static uint32_t reset_stats; +/**< Enable xstats reset. */ +static uint32_t reset_xstats; +/**< Enable memory info. */ +static uint32_t mem_info; +/**< Enable displaying xstat name. */ +static uint32_t enable_xstats_name; +static char *xstats_name; + +/**< Enable xstats by ids. */ +#define MAX_NB_XSTATS_IDS 1024 +static uint32_t nb_xstats_ids; +static uint64_t xstats_ids[MAX_NB_XSTATS_IDS]; + +/**< display usage */ +static void +proc_info_usage(const char *prgname) +{ + printf("%s [EAL options] -- -p PORTMASK\n" + " -m to display DPDK memory zones, segments and TAILQ information\n" + " -p PORTMASK: hexadecimal bitmask of ports to retrieve stats for\n" + " --stats: to display port statistics, enabled by default\n" + " --xstats: to display extended port statistics, disabled by " + "default\n" + " --metrics: to display derived metrics of the ports, disabled by " + "default\n" + " --xstats-name NAME: to display single xstat id by NAME\n" + " --xstats-ids IDLIST: to display xstat values by id. " + "The argument is comma-separated list of xstat ids to print out.\n" + " --stats-reset: to reset port statistics\n" + " --xstats-reset: to reset port extended statistics\n" + " --collectd-format: to print statistics to STDOUT in expected by collectd format\n" + " --host-id STRING: host id used to identify the system process is running on\n", + prgname); +} + +/* + * Parse the portmask provided at run time. + */ +static int +parse_portmask(const char *portmask) +{ + char *end = NULL; + unsigned long pm; + + errno = 0; + + /* parse hexadecimal string */ + pm = strtoul(portmask, &end, 16); + if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0') || + (errno != 0)) { + printf("%s ERROR parsing the port mask\n", __func__); + return -1; + } + + if (pm == 0) + return -1; + + return pm; + +} + +/* + * Parse ids value list into array + */ +static int +parse_xstats_ids(char *list, uint64_t *ids, int limit) { + int length; + char *token; + char *ctx = NULL; + char *endptr; + + length = 0; + token = strtok_r(list, ",", &ctx); + while (token != NULL) { + ids[length] = strtoull(token, &endptr, 10); + if (*endptr != '\0') + return -EINVAL; + + length++; + if (length >= limit) + return -E2BIG; + + token = strtok_r(NULL, ",", &ctx); + } + + return length; +} + +static int +proc_info_preparse_args(int argc, char **argv) +{ + char *prgname = argv[0]; + int i; + + for (i = 0; i < argc; i++) { + /* Print stats or xstats to STDOUT in collectd format */ + if (!strncmp(argv[i], "--collectd-format", MAX_LONG_OPT_SZ)) { + enable_collectd_format = 1; + stdout_fd = dup(STDOUT_FILENO); + close(STDOUT_FILENO); + } + if (!strncmp(argv[i], "--host-id", MAX_LONG_OPT_SZ)) { + if ((i + 1) == argc) { + printf("Invalid host id or not specified\n"); + proc_info_usage(prgname); + return -1; + } + snprintf(host_id, sizeof(host_id), "%s", argv[i+1]); + } + } + + if (!strlen(host_id)) { + int err = gethostname(host_id, MAX_LONG_OPT_SZ-1); + + if (err) + strcpy(host_id, "unknown"); + } + + return 0; +} + +/* Parse the argument given in the command line of the application */ +static int +proc_info_parse_args(int argc, char **argv) +{ + int opt; + int option_index; + char *prgname = argv[0]; + static struct option long_option[] = { + {"stats", 0, NULL, 0}, + {"stats-reset", 0, NULL, 0}, + {"xstats", 0, NULL, 0}, + {"metrics", 0, NULL, 0}, + {"xstats-reset", 0, NULL, 0}, + {"xstats-name", required_argument, NULL, 1}, + {"collectd-format", 0, NULL, 0}, + {"xstats-ids", 1, NULL, 1}, + {"host-id", 0, NULL, 0}, + {NULL, 0, 0, 0} + }; + + if (argc == 1) + proc_info_usage(prgname); + + /* Parse command line */ + while ((opt = getopt_long(argc, argv, "p:m", + long_option, &option_index)) != EOF) { + switch (opt) { + /* portmask */ + case 'p': + enabled_port_mask = parse_portmask(optarg); + if (enabled_port_mask == 0) { + printf("invalid portmask\n"); + proc_info_usage(prgname); + return -1; + } + break; + case 'm': + mem_info = 1; + break; + case 0: + /* Print stats */ + if (!strncmp(long_option[option_index].name, "stats", + MAX_LONG_OPT_SZ)) + enable_stats = 1; + /* Print xstats */ + else if (!strncmp(long_option[option_index].name, "xstats", + MAX_LONG_OPT_SZ)) + enable_xstats = 1; + else if (!strncmp(long_option[option_index].name, + "metrics", + MAX_LONG_OPT_SZ)) + enable_metrics = 1; + /* Reset stats */ + if (!strncmp(long_option[option_index].name, "stats-reset", + MAX_LONG_OPT_SZ)) + reset_stats = 1; + /* Reset xstats */ + else if (!strncmp(long_option[option_index].name, "xstats-reset", + MAX_LONG_OPT_SZ)) + reset_xstats = 1; + break; + case 1: + /* Print xstat single value given by name*/ + if (!strncmp(long_option[option_index].name, + "xstats-name", MAX_LONG_OPT_SZ)) { + enable_xstats_name = 1; + xstats_name = optarg; + printf("name:%s:%s\n", + long_option[option_index].name, + optarg); + } else if (!strncmp(long_option[option_index].name, + "xstats-ids", + MAX_LONG_OPT_SZ)) { + nb_xstats_ids = parse_xstats_ids(optarg, + xstats_ids, MAX_NB_XSTATS_IDS); + + if (nb_xstats_ids <= 0) { + printf("xstats-id list parse error.\n"); + return -1; + } + + } + break; + default: + proc_info_usage(prgname); + return -1; + } + } + return 0; +} + +static void +meminfo_display(void) +{ + printf("----------- MEMORY_SEGMENTS -----------\n"); + rte_dump_physmem_layout(stdout); + printf("--------- END_MEMORY_SEGMENTS ---------\n"); + + printf("------------ MEMORY_ZONES -------------\n"); + rte_memzone_dump(stdout); + printf("---------- END_MEMORY_ZONES -----------\n"); + + printf("------------- TAIL_QUEUES -------------\n"); + rte_dump_tailq(stdout); + printf("---------- END_TAIL_QUEUES ------------\n"); +} + +static void +nic_stats_display(uint16_t port_id) +{ + struct rte_eth_stats stats; + uint8_t i; + + static const char *nic_stats_border = "########################"; + + rte_eth_stats_get(port_id, &stats); + printf("\n %s NIC statistics for port %-2d %s\n", + nic_stats_border, port_id, nic_stats_border); + + printf(" RX-packets: %-10"PRIu64" RX-errors: %-10"PRIu64 + " RX-bytes: %-10"PRIu64"\n", stats.ipackets, stats.ierrors, + stats.ibytes); + printf(" RX-nombuf: %-10"PRIu64"\n", stats.rx_nombuf); + printf(" TX-packets: %-10"PRIu64" TX-errors: %-10"PRIu64 + " TX-bytes: %-10"PRIu64"\n", stats.opackets, stats.oerrors, + stats.obytes); + + printf("\n"); + for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { + printf(" Stats reg %2d RX-packets: %-10"PRIu64 + " RX-errors: %-10"PRIu64 + " RX-bytes: %-10"PRIu64"\n", + i, stats.q_ipackets[i], stats.q_errors[i], stats.q_ibytes[i]); + } + + printf("\n"); + for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { + printf(" Stats reg %2d TX-packets: %-10"PRIu64 + " TX-bytes: %-10"PRIu64"\n", + i, stats.q_opackets[i], stats.q_obytes[i]); + } + + printf(" %s############################%s\n", + nic_stats_border, nic_stats_border); +} + +static void +nic_stats_clear(uint16_t port_id) +{ + printf("\n Clearing NIC stats for port %d\n", port_id); + rte_eth_stats_reset(port_id); + printf("\n NIC statistics for port %d cleared\n", port_id); +} + +static void collectd_resolve_cnt_type(char *cnt_type, size_t cnt_type_len, + const char *cnt_name) { + char *type_end = strrchr(cnt_name, '_'); + + if ((type_end != NULL) && + (strncmp(cnt_name, "rx_", strlen("rx_")) == 0)) { + if (strncmp(type_end, "_errors", strlen("_errors")) == 0) + strncpy(cnt_type, "if_rx_errors", cnt_type_len); + else if (strncmp(type_end, "_dropped", strlen("_dropped")) == 0) + strncpy(cnt_type, "if_rx_dropped", cnt_type_len); + else if (strncmp(type_end, "_bytes", strlen("_bytes")) == 0) + strncpy(cnt_type, "if_rx_octets", cnt_type_len); + else if (strncmp(type_end, "_packets", strlen("_packets")) == 0) + strncpy(cnt_type, "if_rx_packets", cnt_type_len); + else if (strncmp(type_end, "_placement", + strlen("_placement")) == 0) + strncpy(cnt_type, "if_rx_errors", cnt_type_len); + else if (strncmp(type_end, "_buff", strlen("_buff")) == 0) + strncpy(cnt_type, "if_rx_errors", cnt_type_len); + else + /* Does not fit obvious type: use a more generic one */ + strncpy(cnt_type, "derive", cnt_type_len); + } else if ((type_end != NULL) && + (strncmp(cnt_name, "tx_", strlen("tx_"))) == 0) { + if (strncmp(type_end, "_errors", strlen("_errors")) == 0) + strncpy(cnt_type, "if_tx_errors", cnt_type_len); + else if (strncmp(type_end, "_dropped", strlen("_dropped")) == 0) + strncpy(cnt_type, "if_tx_dropped", cnt_type_len); + else if (strncmp(type_end, "_bytes", strlen("_bytes")) == 0) + strncpy(cnt_type, "if_tx_octets", cnt_type_len); + else if (strncmp(type_end, "_packets", strlen("_packets")) == 0) + strncpy(cnt_type, "if_tx_packets", cnt_type_len); + else + /* Does not fit obvious type: use a more generic one */ + strncpy(cnt_type, "derive", cnt_type_len); + } else if ((type_end != NULL) && + (strncmp(cnt_name, "flow_", strlen("flow_"))) == 0) { + if (strncmp(type_end, "_filters", strlen("_filters")) == 0) + strncpy(cnt_type, "operations", cnt_type_len); + else if (strncmp(type_end, "_errors", strlen("_errors")) == 0) + strncpy(cnt_type, "errors", cnt_type_len); + else if (strncmp(type_end, "_filters", strlen("_filters")) == 0) + strncpy(cnt_type, "filter_result", cnt_type_len); + } else if ((type_end != NULL) && + (strncmp(cnt_name, "mac_", strlen("mac_"))) == 0) { + if (strncmp(type_end, "_errors", strlen("_errors")) == 0) + strncpy(cnt_type, "errors", cnt_type_len); + } else { + /* Does not fit obvious type, or strrchr error: */ + /* use a more generic type */ + strncpy(cnt_type, "derive", cnt_type_len); + } +} + +static void +nic_xstats_by_name_display(uint16_t port_id, char *name) +{ + uint64_t id; + + printf("###### NIC statistics for port %-2d, statistic name '%s':\n", + port_id, name); + + if (rte_eth_xstats_get_id_by_name(port_id, name, &id) == 0) + printf("%s: %"PRIu64"\n", name, id); + else + printf("Statistic not found...\n"); + +} + +static void +nic_xstats_by_ids_display(uint16_t port_id, uint64_t *ids, int len) +{ + struct rte_eth_xstat_name *xstats_names; + uint64_t *values; + int ret, i; + static const char *nic_stats_border = "########################"; + + values = malloc(sizeof(*values) * len); + if (values == NULL) { + printf("Cannot allocate memory for xstats\n"); + return; + } + + xstats_names = malloc(sizeof(struct rte_eth_xstat_name) * len); + if (xstats_names == NULL) { + printf("Cannot allocate memory for xstat names\n"); + free(values); + return; + } + + if (len != rte_eth_xstats_get_names_by_id( + port_id, xstats_names, len, ids)) { + printf("Cannot get xstat names\n"); + goto err; + } + + printf("###### NIC extended statistics for port %-2d #########\n", + port_id); + printf("%s############################\n", nic_stats_border); + ret = rte_eth_xstats_get_by_id(port_id, ids, values, len); + if (ret < 0 || ret > len) { + printf("Cannot get xstats\n"); + goto err; + } + + for (i = 0; i < len; i++) + printf("%s: %"PRIu64"\n", + xstats_names[i].name, + values[i]); + + printf("%s############################\n", nic_stats_border); +err: + free(values); + free(xstats_names); +} + +static void +nic_xstats_display(uint16_t port_id) +{ + struct rte_eth_xstat_name *xstats_names; + uint64_t *values; + int len, ret, i; + static const char *nic_stats_border = "########################"; + + len = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL); + if (len < 0) { + printf("Cannot get xstats count\n"); + return; + } + values = malloc(sizeof(*values) * len); + if (values == NULL) { + printf("Cannot allocate memory for xstats\n"); + return; + } + + xstats_names = malloc(sizeof(struct rte_eth_xstat_name) * len); + if (xstats_names == NULL) { + printf("Cannot allocate memory for xstat names\n"); + free(values); + return; + } + if (len != rte_eth_xstats_get_names_by_id( + port_id, xstats_names, len, NULL)) { + printf("Cannot get xstat names\n"); + goto err; + } + + printf("###### NIC extended statistics for port %-2d #########\n", + port_id); + printf("%s############################\n", + nic_stats_border); + ret = rte_eth_xstats_get_by_id(port_id, NULL, values, len); + if (ret < 0 || ret > len) { + printf("Cannot get xstats\n"); + goto err; + } + + for (i = 0; i < len; i++) { + if (enable_collectd_format) { + char counter_type[MAX_STRING_LEN]; + char buf[MAX_STRING_LEN]; + size_t n; + + collectd_resolve_cnt_type(counter_type, + sizeof(counter_type), + xstats_names[i].name); + n = snprintf(buf, MAX_STRING_LEN, + "PUTVAL %s/dpdkstat-port.%u/%s-%s N:%" + PRIu64"\n", host_id, port_id, counter_type, + xstats_names[i].name, values[i]); + if (n > sizeof(buf) - 1) + n = sizeof(buf) - 1; + ret = write(stdout_fd, buf, n); + if (ret < 0) + goto err; + } else { + printf("%s: %"PRIu64"\n", xstats_names[i].name, + values[i]); + } + } + + printf("%s############################\n", + nic_stats_border); +err: + free(values); + free(xstats_names); +} + +static void +nic_xstats_clear(uint16_t port_id) +{ + printf("\n Clearing NIC xstats for port %d\n", port_id); + rte_eth_xstats_reset(port_id); + printf("\n NIC extended statistics for port %d cleared\n", port_id); +} + +static void +metrics_display(int port_id) +{ + struct rte_metric_value *metrics; + struct rte_metric_name *names; + int len, ret; + static const char *nic_stats_border = "########################"; + + len = rte_metrics_get_names(NULL, 0); + if (len < 0) { + printf("Cannot get metrics count\n"); + return; + } + if (len == 0) { + printf("No metrics to display (none have been registered)\n"); + return; + } + + metrics = rte_malloc("proc_info_metrics", + sizeof(struct rte_metric_value) * len, 0); + if (metrics == NULL) { + printf("Cannot allocate memory for metrics\n"); + return; + } + + names = rte_malloc(NULL, sizeof(struct rte_metric_name) * len, 0); + if (names == NULL) { + printf("Cannot allocate memory for metrcis names\n"); + rte_free(metrics); + return; + } + + if (len != rte_metrics_get_names(names, len)) { + printf("Cannot get metrics names\n"); + rte_free(metrics); + rte_free(names); + return; + } + + if (port_id == RTE_METRICS_GLOBAL) + printf("###### Non port specific metrics #########\n"); + else + printf("###### metrics for port %-2d #########\n", port_id); + printf("%s############################\n", nic_stats_border); + ret = rte_metrics_get_values(port_id, metrics, len); + if (ret < 0 || ret > len) { + printf("Cannot get metrics values\n"); + rte_free(metrics); + rte_free(names); + return; + } + + int i; + for (i = 0; i < len; i++) + printf("%s: %"PRIu64"\n", names[i].name, metrics[i].value); + + printf("%s############################\n", nic_stats_border); + rte_free(metrics); + rte_free(names); +} + +int +main(int argc, char **argv) +{ + int ret; + int i; + char c_flag[] = "-c1"; + char n_flag[] = "-n4"; + char mp_flag[] = "--proc-type=secondary"; + char *argp[argc + 3]; + uint16_t nb_ports; + + /* preparse app arguments */ + ret = proc_info_preparse_args(argc, argv); + if (ret < 0) { + printf("Failed to parse arguments\n"); + return -1; + } + + argp[0] = argv[0]; + argp[1] = c_flag; + argp[2] = n_flag; + argp[3] = mp_flag; + + for (i = 1; i < argc; i++) + argp[i + 3] = argv[i]; + + argc += 3; + + ret = rte_eal_init(argc, argp); + if (ret < 0) + rte_panic("Cannot init EAL\n"); + + argc -= ret; + argv += (ret - 3); + + if (!rte_eal_primary_proc_alive(NULL)) + rte_exit(EXIT_FAILURE, "No primary DPDK process is running.\n"); + + /* parse app arguments */ + ret = proc_info_parse_args(argc, argv); + if (ret < 0) + rte_exit(EXIT_FAILURE, "Invalid argument\n"); + + if (mem_info) { + meminfo_display(); + return 0; + } + + nb_ports = rte_eth_dev_count_avail(); + if (nb_ports == 0) + rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n"); + + /* If no port mask was specified*/ + if (enabled_port_mask == 0) + enabled_port_mask = 0xffff; + + RTE_ETH_FOREACH_DEV(i) { + if (enabled_port_mask & (1 << i)) { + if (enable_stats) + nic_stats_display(i); + else if (enable_xstats) + nic_xstats_display(i); + else if (reset_stats) + nic_stats_clear(i); + else if (reset_xstats) + nic_xstats_clear(i); + else if (enable_xstats_name) + nic_xstats_by_name_display(i, xstats_name); + else if (nb_xstats_ids > 0) + nic_xstats_by_ids_display(i, xstats_ids, + nb_xstats_ids); + else if (enable_metrics) + metrics_display(i); + } + } + + /* print port independent stats */ + if (enable_metrics) + metrics_display(RTE_METRICS_GLOBAL); + + ret = rte_eal_cleanup(); + if (ret) + printf("Error from rte_eal_cleanup(), %d\n", ret); + + return 0; +} diff --git a/dpdk/app/proc-info/meson.build b/dpdk/app/proc-info/meson.build new file mode 100644 index 00000000..a52b2ee4 --- /dev/null +++ b/dpdk/app/proc-info/meson.build @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Intel Corporation + +sources = files('main.c') +allow_experimental_apis = true +deps += ['ethdev', 'metrics'] diff --git a/dpdk/app/proc_info/Makefile b/dpdk/app/proc_info/Makefile deleted file mode 100644 index 9e90438e..00000000 --- a/dpdk/app/proc_info/Makefile +++ /dev/null @@ -1,42 +0,0 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -include $(RTE_SDK)/mk/rte.vars.mk - -APP = dpdk-procinfo - -CFLAGS += $(WERROR_FLAGS) - -# all source are stored in SRCS-y - -SRCS-y := main.c - -include $(RTE_SDK)/mk/rte.app.mk diff --git a/dpdk/app/proc_info/main.c b/dpdk/app/proc_info/main.c deleted file mode 100644 index 875d91ea..00000000 --- a/dpdk/app/proc_info/main.c +++ /dev/null @@ -1,693 +0,0 @@ -/* - * BSD LICENSE - * - * Copyright(c) 2010-2017 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* Maximum long option length for option parsing. */ -#define MAX_LONG_OPT_SZ 64 -#define RTE_LOGTYPE_APP RTE_LOGTYPE_USER1 - -#define MAX_STRING_LEN 256 - -/**< mask of enabled ports */ -static uint32_t enabled_port_mask; -/**< Enable stats. */ -static uint32_t enable_stats; -/**< Enable xstats. */ -static uint32_t enable_xstats; -/**< Enable collectd format*/ -static uint32_t enable_collectd_format; -/**< FD to send collectd format messages to STDOUT*/ -static int stdout_fd; -/**< Host id process is running on */ -static char host_id[MAX_LONG_OPT_SZ]; -/**< Enable metrics. */ -static uint32_t enable_metrics; -/**< Enable stats reset. */ -static uint32_t reset_stats; -/**< Enable xstats reset. */ -static uint32_t reset_xstats; -/**< Enable memory info. */ -static uint32_t mem_info; -/**< Enable displaying xstat name. */ -static uint32_t enable_xstats_name; -static char *xstats_name; - -/**< Enable xstats by ids. */ -#define MAX_NB_XSTATS_IDS 1024 -static uint32_t nb_xstats_ids; -static uint64_t xstats_ids[MAX_NB_XSTATS_IDS]; - -/**< display usage */ -static void -proc_info_usage(const char *prgname) -{ - printf("%s [EAL options] -- -p PORTMASK\n" - " -m to display DPDK memory zones, segments and TAILQ information\n" - " -p PORTMASK: hexadecimal bitmask of ports to retrieve stats for\n" - " --stats: to display port statistics, enabled by default\n" - " --xstats: to display extended port statistics, disabled by " - "default\n" - " --metrics: to display derived metrics of the ports, disabled by " - "default\n" - " --xstats-name NAME: to display single xstat id by NAME\n" - " --xstats-ids IDLIST: to display xstat values by id. " - "The argument is comma-separated list of xstat ids to print out.\n" - " --stats-reset: to reset port statistics\n" - " --xstats-reset: to reset port extended statistics\n" - " --collectd-format: to print statistics to STDOUT in expected by collectd format\n" - " --host-id STRING: host id used to identify the system process is running on\n", - prgname); -} - -/* - * Parse the portmask provided at run time. - */ -static int -parse_portmask(const char *portmask) -{ - char *end = NULL; - unsigned long pm; - - errno = 0; - - /* parse hexadecimal string */ - pm = strtoul(portmask, &end, 16); - if ((portmask[0] == '\0') || (end == NULL) || (*end != '\0') || - (errno != 0)) { - printf("%s ERROR parsing the port mask\n", __func__); - return -1; - } - - if (pm == 0) - return -1; - - return pm; - -} - -/* - * Parse ids value list into array - */ -static int -parse_xstats_ids(char *list, uint64_t *ids, int limit) { - int length; - char *token; - char *ctx = NULL; - char *endptr; - - length = 0; - token = strtok_r(list, ",", &ctx); - while (token != NULL) { - ids[length] = strtoull(token, &endptr, 10); - if (*endptr != '\0') - return -EINVAL; - - length++; - if (length >= limit) - return -E2BIG; - - token = strtok_r(NULL, ",", &ctx); - } - - return length; -} - -static int -proc_info_preparse_args(int argc, char **argv) -{ - char *prgname = argv[0]; - int i; - - for (i = 0; i < argc; i++) { - /* Print stats or xstats to STDOUT in collectd format */ - if (!strncmp(argv[i], "--collectd-format", MAX_LONG_OPT_SZ)) { - enable_collectd_format = 1; - stdout_fd = dup(STDOUT_FILENO); - close(STDOUT_FILENO); - } - if (!strncmp(argv[i], "--host-id", MAX_LONG_OPT_SZ)) { - if ((i + 1) == argc) { - printf("Invalid host id or not specified\n"); - proc_info_usage(prgname); - return -1; - } - snprintf(host_id, sizeof(host_id), "%s", argv[i+1]); - } - } - - if (!strlen(host_id)) { - int err = gethostname(host_id, MAX_LONG_OPT_SZ-1); - - if (err) - strcpy(host_id, "unknown"); - } - - return 0; -} - -/* Parse the argument given in the command line of the application */ -static int -proc_info_parse_args(int argc, char **argv) -{ - int opt; - int option_index; - char *prgname = argv[0]; - static struct option long_option[] = { - {"stats", 0, NULL, 0}, - {"stats-reset", 0, NULL, 0}, - {"xstats", 0, NULL, 0}, - {"metrics", 0, NULL, 0}, - {"xstats-reset", 0, NULL, 0}, - {"xstats-name", required_argument, NULL, 1}, - {"collectd-format", 0, NULL, 0}, - {"xstats-ids", 1, NULL, 1}, - {"host-id", 0, NULL, 0}, - {NULL, 0, 0, 0} - }; - - if (argc == 1) - proc_info_usage(prgname); - - /* Parse command line */ - while ((opt = getopt_long(argc, argv, "p:m", - long_option, &option_index)) != EOF) { - switch (opt) { - /* portmask */ - case 'p': - enabled_port_mask = parse_portmask(optarg); - if (enabled_port_mask == 0) { - printf("invalid portmask\n"); - proc_info_usage(prgname); - return -1; - } - break; - case 'm': - mem_info = 1; - break; - case 0: - /* Print stats */ - if (!strncmp(long_option[option_index].name, "stats", - MAX_LONG_OPT_SZ)) - enable_stats = 1; - /* Print xstats */ - else if (!strncmp(long_option[option_index].name, "xstats", - MAX_LONG_OPT_SZ)) - enable_xstats = 1; - else if (!strncmp(long_option[option_index].name, - "metrics", - MAX_LONG_OPT_SZ)) - enable_metrics = 1; - /* Reset stats */ - if (!strncmp(long_option[option_index].name, "stats-reset", - MAX_LONG_OPT_SZ)) - reset_stats = 1; - /* Reset xstats */ - else if (!strncmp(long_option[option_index].name, "xstats-reset", - MAX_LONG_OPT_SZ)) - reset_xstats = 1; - break; - case 1: - /* Print xstat single value given by name*/ - if (!strncmp(long_option[option_index].name, - "xstats-name", MAX_LONG_OPT_SZ)) { - enable_xstats_name = 1; - xstats_name = optarg; - printf("name:%s:%s\n", - long_option[option_index].name, - optarg); - } else if (!strncmp(long_option[option_index].name, - "xstats-ids", - MAX_LONG_OPT_SZ)) { - nb_xstats_ids = parse_xstats_ids(optarg, - xstats_ids, MAX_NB_XSTATS_IDS); - - if (nb_xstats_ids <= 0) { - printf("xstats-id list parse error.\n"); - return -1; - } - - } - break; - default: - proc_info_usage(prgname); - return -1; - } - } - return 0; -} - -static void -meminfo_display(void) -{ - printf("----------- MEMORY_SEGMENTS -----------\n"); - rte_dump_physmem_layout(stdout); - printf("--------- END_MEMORY_SEGMENTS ---------\n"); - - printf("------------ MEMORY_ZONES -------------\n"); - rte_memzone_dump(stdout); - printf("---------- END_MEMORY_ZONES -----------\n"); - - printf("------------- TAIL_QUEUES -------------\n"); - rte_dump_tailq(stdout); - printf("---------- END_TAIL_QUEUES ------------\n"); -} - -static void -nic_stats_display(uint16_t port_id) -{ - struct rte_eth_stats stats; - uint8_t i; - - static const char *nic_stats_border = "########################"; - - rte_eth_stats_get(port_id, &stats); - printf("\n %s NIC statistics for port %-2d %s\n", - nic_stats_border, port_id, nic_stats_border); - - printf(" RX-packets: %-10"PRIu64" RX-errors: %-10"PRIu64 - " RX-bytes: %-10"PRIu64"\n", stats.ipackets, stats.ierrors, - stats.ibytes); - printf(" RX-nombuf: %-10"PRIu64"\n", stats.rx_nombuf); - printf(" TX-packets: %-10"PRIu64" TX-errors: %-10"PRIu64 - " TX-bytes: %-10"PRIu64"\n", stats.opackets, stats.oerrors, - stats.obytes); - - printf("\n"); - for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { - printf(" Stats reg %2d RX-packets: %-10"PRIu64 - " RX-errors: %-10"PRIu64 - " RX-bytes: %-10"PRIu64"\n", - i, stats.q_ipackets[i], stats.q_errors[i], stats.q_ibytes[i]); - } - - printf("\n"); - for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) { - printf(" Stats reg %2d TX-packets: %-10"PRIu64 - " TX-bytes: %-10"PRIu64"\n", - i, stats.q_opackets[i], stats.q_obytes[i]); - } - - printf(" %s############################%s\n", - nic_stats_border, nic_stats_border); -} - -static void -nic_stats_clear(uint16_t port_id) -{ - printf("\n Clearing NIC stats for port %d\n", port_id); - rte_eth_stats_reset(port_id); - printf("\n NIC statistics for port %d cleared\n", port_id); -} - -static void collectd_resolve_cnt_type(char *cnt_type, size_t cnt_type_len, - const char *cnt_name) { - char *type_end = strrchr(cnt_name, '_'); - - if ((type_end != NULL) && - (strncmp(cnt_name, "rx_", strlen("rx_")) == 0)) { - if (strncmp(type_end, "_errors", strlen("_errors")) == 0) - strncpy(cnt_type, "if_rx_errors", cnt_type_len); - else if (strncmp(type_end, "_dropped", strlen("_dropped")) == 0) - strncpy(cnt_type, "if_rx_dropped", cnt_type_len); - else if (strncmp(type_end, "_bytes", strlen("_bytes")) == 0) - strncpy(cnt_type, "if_rx_octets", cnt_type_len); - else if (strncmp(type_end, "_packets", strlen("_packets")) == 0) - strncpy(cnt_type, "if_rx_packets", cnt_type_len); - else if (strncmp(type_end, "_placement", - strlen("_placement")) == 0) - strncpy(cnt_type, "if_rx_errors", cnt_type_len); - else if (strncmp(type_end, "_buff", strlen("_buff")) == 0) - strncpy(cnt_type, "if_rx_errors", cnt_type_len); - else - /* Does not fit obvious type: use a more generic one */ - strncpy(cnt_type, "derive", cnt_type_len); - } else if ((type_end != NULL) && - (strncmp(cnt_name, "tx_", strlen("tx_"))) == 0) { - if (strncmp(type_end, "_errors", strlen("_errors")) == 0) - strncpy(cnt_type, "if_tx_errors", cnt_type_len); - else if (strncmp(type_end, "_dropped", strlen("_dropped")) == 0) - strncpy(cnt_type, "if_tx_dropped", cnt_type_len); - else if (strncmp(type_end, "_bytes", strlen("_bytes")) == 0) - strncpy(cnt_type, "if_tx_octets", cnt_type_len); - else if (strncmp(type_end, "_packets", strlen("_packets")) == 0) - strncpy(cnt_type, "if_tx_packets", cnt_type_len); - else - /* Does not fit obvious type: use a more generic one */ - strncpy(cnt_type, "derive", cnt_type_len); - } else if ((type_end != NULL) && - (strncmp(cnt_name, "flow_", strlen("flow_"))) == 0) { - if (strncmp(type_end, "_filters", strlen("_filters")) == 0) - strncpy(cnt_type, "operations", cnt_type_len); - else if (strncmp(type_end, "_errors", strlen("_errors")) == 0) - strncpy(cnt_type, "errors", cnt_type_len); - else if (strncmp(type_end, "_filters", strlen("_filters")) == 0) - strncpy(cnt_type, "filter_result", cnt_type_len); - } else if ((type_end != NULL) && - (strncmp(cnt_name, "mac_", strlen("mac_"))) == 0) { - if (strncmp(type_end, "_errors", strlen("_errors")) == 0) - strncpy(cnt_type, "errors", cnt_type_len); - } else { - /* Does not fit obvious type, or strrchr error: */ - /* use a more generic type */ - strncpy(cnt_type, "derive", cnt_type_len); - } -} - -static void -nic_xstats_by_name_display(uint16_t port_id, char *name) -{ - uint64_t id; - - printf("###### NIC statistics for port %-2d, statistic name '%s':\n", - port_id, name); - - if (rte_eth_xstats_get_id_by_name(port_id, name, &id) == 0) - printf("%s: %"PRIu64"\n", name, id); - else - printf("Statistic not found...\n"); - -} - -static void -nic_xstats_by_ids_display(uint16_t port_id, uint64_t *ids, int len) -{ - struct rte_eth_xstat_name *xstats_names; - uint64_t *values; - int ret, i; - static const char *nic_stats_border = "########################"; - - values = malloc(sizeof(*values) * len); - if (values == NULL) { - printf("Cannot allocate memory for xstats\n"); - return; - } - - xstats_names = malloc(sizeof(struct rte_eth_xstat_name) * len); - if (xstats_names == NULL) { - printf("Cannot allocate memory for xstat names\n"); - free(values); - return; - } - - if (len != rte_eth_xstats_get_names_by_id( - port_id, xstats_names, len, ids)) { - printf("Cannot get xstat names\n"); - goto err; - } - - printf("###### NIC extended statistics for port %-2d #########\n", - port_id); - printf("%s############################\n", nic_stats_border); - ret = rte_eth_xstats_get_by_id(port_id, ids, values, len); - if (ret < 0 || ret > len) { - printf("Cannot get xstats\n"); - goto err; - } - - for (i = 0; i < len; i++) - printf("%s: %"PRIu64"\n", - xstats_names[i].name, - values[i]); - - printf("%s############################\n", nic_stats_border); -err: - free(values); - free(xstats_names); -} - -static void -nic_xstats_display(uint16_t port_id) -{ - struct rte_eth_xstat_name *xstats_names; - uint64_t *values; - int len, ret, i; - static const char *nic_stats_border = "########################"; - - len = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL); - if (len < 0) { - printf("Cannot get xstats count\n"); - return; - } - values = malloc(sizeof(*values) * len); - if (values == NULL) { - printf("Cannot allocate memory for xstats\n"); - return; - } - - xstats_names = malloc(sizeof(struct rte_eth_xstat_name) * len); - if (xstats_names == NULL) { - printf("Cannot allocate memory for xstat names\n"); - free(values); - return; - } - if (len != rte_eth_xstats_get_names_by_id( - port_id, xstats_names, len, NULL)) { - printf("Cannot get xstat names\n"); - goto err; - } - - printf("###### NIC extended statistics for port %-2d #########\n", - port_id); - printf("%s############################\n", - nic_stats_border); - ret = rte_eth_xstats_get_by_id(port_id, NULL, values, len); - if (ret < 0 || ret > len) { - printf("Cannot get xstats\n"); - goto err; - } - - for (i = 0; i < len; i++) { - if (enable_collectd_format) { - char counter_type[MAX_STRING_LEN]; - char buf[MAX_STRING_LEN]; - - collectd_resolve_cnt_type(counter_type, - sizeof(counter_type), - xstats_names[i].name); - sprintf(buf, "PUTVAL %s/dpdkstat-port.%u/%s-%s N:%" - PRIu64"\n", host_id, port_id, counter_type, - xstats_names[i].name, values[i]); - ret = write(stdout_fd, buf, strlen(buf)); - if (ret < 0) - goto err; - } else { - printf("%s: %"PRIu64"\n", xstats_names[i].name, - values[i]); - } - } - - printf("%s############################\n", - nic_stats_border); -err: - free(values); - free(xstats_names); -} - -static void -nic_xstats_clear(uint16_t port_id) -{ - printf("\n Clearing NIC xstats for port %d\n", port_id); - rte_eth_xstats_reset(port_id); - printf("\n NIC extended statistics for port %d cleared\n", port_id); -} - -static void -metrics_display(int port_id) -{ - struct rte_metric_value *metrics; - struct rte_metric_name *names; - int len, ret; - static const char *nic_stats_border = "########################"; - - len = rte_metrics_get_names(NULL, 0); - if (len < 0) { - printf("Cannot get metrics count\n"); - return; - } - if (len == 0) { - printf("No metrics to display (none have been registered)\n"); - return; - } - - metrics = rte_malloc("proc_info_metrics", - sizeof(struct rte_metric_value) * len, 0); - if (metrics == NULL) { - printf("Cannot allocate memory for metrics\n"); - return; - } - - names = rte_malloc(NULL, sizeof(struct rte_metric_name) * len, 0); - if (names == NULL) { - printf("Cannot allocate memory for metrcis names\n"); - rte_free(metrics); - return; - } - - if (len != rte_metrics_get_names(names, len)) { - printf("Cannot get metrics names\n"); - rte_free(metrics); - rte_free(names); - return; - } - - if (port_id == RTE_METRICS_GLOBAL) - printf("###### Non port specific metrics #########\n"); - else - printf("###### metrics for port %-2d #########\n", port_id); - printf("%s############################\n", nic_stats_border); - ret = rte_metrics_get_values(port_id, metrics, len); - if (ret < 0 || ret > len) { - printf("Cannot get metrics values\n"); - rte_free(metrics); - rte_free(names); - return; - } - - int i; - for (i = 0; i < len; i++) - printf("%s: %"PRIu64"\n", names[i].name, metrics[i].value); - - printf("%s############################\n", nic_stats_border); - rte_free(metrics); - rte_free(names); -} - -int -main(int argc, char **argv) -{ - int ret; - int i; - char c_flag[] = "-c1"; - char n_flag[] = "-n4"; - char mp_flag[] = "--proc-type=secondary"; - char *argp[argc + 3]; - uint16_t nb_ports; - - /* preparse app arguments */ - ret = proc_info_preparse_args(argc, argv); - if (ret < 0) { - printf("Failed to parse arguments\n"); - return -1; - } - - argp[0] = argv[0]; - argp[1] = c_flag; - argp[2] = n_flag; - argp[3] = mp_flag; - - for (i = 1; i < argc; i++) - argp[i + 3] = argv[i]; - - argc += 3; - - ret = rte_eal_init(argc, argp); - if (ret < 0) - rte_panic("Cannot init EAL\n"); - - argc -= ret; - argv += (ret - 3); - - if (!rte_eal_primary_proc_alive(NULL)) - rte_exit(EXIT_FAILURE, "No primary DPDK process is running.\n"); - - /* parse app arguments */ - ret = proc_info_parse_args(argc, argv); - if (ret < 0) - rte_exit(EXIT_FAILURE, "Invalid argument\n"); - - if (mem_info) { - meminfo_display(); - return 0; - } - - nb_ports = rte_eth_dev_count(); - if (nb_ports == 0) - rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n"); - - /* If no port mask was specified*/ - if (enabled_port_mask == 0) - enabled_port_mask = 0xffff; - - for (i = 0; i < nb_ports; i++) { - if (enabled_port_mask & (1 << i)) { - if (enable_stats) - nic_stats_display(i); - else if (enable_xstats) - nic_xstats_display(i); - else if (reset_stats) - nic_stats_clear(i); - else if (reset_xstats) - nic_xstats_clear(i); - else if (enable_xstats_name) - nic_xstats_by_name_display(i, xstats_name); - else if (nb_xstats_ids > 0) - nic_xstats_by_ids_display(i, xstats_ids, - nb_xstats_ids); - else if (enable_metrics) - metrics_display(i); - } - } - - /* print port independent stats */ - if (enable_metrics) - metrics_display(RTE_METRICS_GLOBAL); - - return 0; -} diff --git a/dpdk/app/test-bbdev/Makefile b/dpdk/app/test-bbdev/Makefile new file mode 100644 index 00000000..6da0c8e0 --- /dev/null +++ b/dpdk/app/test-bbdev/Makefile @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +include $(RTE_SDK)/mk/rte.vars.mk + +# +# library name +# +APP = testbbdev + +CFLAGS += -DALLOW_EXPERIMENTAL_API +CFLAGS += -O3 +CFLAGS += $(WERROR_FLAGS) + +# +# all sources are stored in SRCS-y +# +SRCS-$(CONFIG_RTE_TEST_BBDEV) += main.c +SRCS-$(CONFIG_RTE_TEST_BBDEV) += test_bbdev.c +SRCS-$(CONFIG_RTE_TEST_BBDEV) += test_bbdev_perf.c +SRCS-$(CONFIG_RTE_TEST_BBDEV) += test_bbdev_vector.c + +LDLIBS += -lm + +include $(RTE_SDK)/mk/rte.app.mk diff --git a/dpdk/app/test-bbdev/main.c b/dpdk/app/test-bbdev/main.c new file mode 100644 index 00000000..41b54bb1 --- /dev/null +++ b/dpdk/app/test-bbdev/main.c @@ -0,0 +1,326 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "main.h" + +/* Defines how many testcases can be specified as cmdline args */ +#define MAX_CMDLINE_TESTCASES 8 + +static const char tc_sep = ','; + +static struct test_params { + struct test_command *test_to_run[MAX_CMDLINE_TESTCASES]; + unsigned int num_tests; + unsigned int num_ops; + unsigned int burst_sz; + unsigned int num_lcores; + char test_vector_filename[PATH_MAX]; +} test_params; + +static struct test_commands_list commands_list = + TAILQ_HEAD_INITIALIZER(commands_list); + +void +add_test_command(struct test_command *t) +{ + TAILQ_INSERT_TAIL(&commands_list, t, next); +} + +int +unit_test_suite_runner(struct unit_test_suite *suite) +{ + int test_result = TEST_SUCCESS; + unsigned int total = 0, skipped = 0, succeeded = 0, failed = 0; + uint64_t start, end; + + printf( + "\n + ------------------------------------------------------- +\n"); + printf(" + Starting Test Suite : %s\n", suite->suite_name); + + start = rte_rdtsc_precise(); + + if (suite->setup) { + test_result = suite->setup(); + if (test_result == TEST_FAILED) { + printf(" + Test suite setup %s failed!\n", + suite->suite_name); + printf( + " + ------------------------------------------------------- +\n"); + return 1; + } + if (test_result == TEST_SKIPPED) { + printf(" + Test suite setup %s skipped!\n", + suite->suite_name); + printf( + " + ------------------------------------------------------- +\n"); + return 0; + } + } + + while (suite->unit_test_cases[total].testcase) { + if (suite->unit_test_cases[total].setup) + test_result = suite->unit_test_cases[total].setup(); + + if (test_result == TEST_SUCCESS) + test_result = suite->unit_test_cases[total].testcase(); + + if (suite->unit_test_cases[total].teardown) + suite->unit_test_cases[total].teardown(); + + if (test_result == TEST_SUCCESS) { + succeeded++; + printf(" + TestCase [%2d] : %s passed\n", total, + suite->unit_test_cases[total].name); + } else if (test_result == TEST_SKIPPED) { + skipped++; + printf(" + TestCase [%2d] : %s skipped\n", total, + suite->unit_test_cases[total].name); + } else { + failed++; + printf(" + TestCase [%2d] : %s failed\n", total, + suite->unit_test_cases[total].name); + } + + total++; + } + + /* Run test suite teardown */ + if (suite->teardown) + suite->teardown(); + + end = rte_rdtsc_precise(); + + printf(" + ------------------------------------------------------- +\n"); + printf(" + Test Suite Summary : %s\n", suite->suite_name); + printf(" + Tests Total : %2d\n", total); + printf(" + Tests Skipped : %2d\n", skipped); + printf(" + Tests Passed : %2d\n", succeeded); + printf(" + Tests Failed : %2d\n", failed); + printf(" + Tests Lasted : %lg ms\n", + ((end - start) * 1000) / (double)rte_get_tsc_hz()); + printf(" + ------------------------------------------------------- +\n"); + + return (failed > 0) ? 1 : 0; +} + +const char * +get_vector_filename(void) +{ + return test_params.test_vector_filename; +} + +unsigned int +get_num_ops(void) +{ + return test_params.num_ops; +} + +unsigned int +get_burst_sz(void) +{ + return test_params.burst_sz; +} + +unsigned int +get_num_lcores(void) +{ + return test_params.num_lcores; +} + +static void +print_usage(const char *prog_name) +{ + struct test_command *t; + + printf("Usage: %s [EAL params] [-- [-n/--num-ops NUM_OPS]\n" + "\t[-b/--burst-size BURST_SIZE]\n" + "\t[-v/--test-vector VECTOR_FILE]\n" + "\t[-c/--test-cases TEST_CASE[,TEST_CASE,...]]]\n", + prog_name); + + printf("Available testcases: "); + TAILQ_FOREACH(t, &commands_list, next) + printf("%s ", t->command); + printf("\n"); +} + +static int +parse_args(int argc, char **argv, struct test_params *tp) +{ + int opt, option_index; + unsigned int num_tests = 0; + bool test_cases_present = false; + bool test_vector_present = false; + struct test_command *t; + char *tokens[MAX_CMDLINE_TESTCASES]; + int tc, ret; + + static struct option lgopts[] = { + { "num-ops", 1, 0, 'n' }, + { "burst-size", 1, 0, 'b' }, + { "test-cases", 1, 0, 'c' }, + { "test-vector", 1, 0, 'v' }, + { "lcores", 1, 0, 'l' }, + { "help", 0, 0, 'h' }, + { NULL, 0, 0, 0 } + }; + + while ((opt = getopt_long(argc, argv, "hn:b:c:v:l:", lgopts, + &option_index)) != EOF) + switch (opt) { + case 'n': + TEST_ASSERT(strlen(optarg) > 0, + "Num of operations is not provided"); + tp->num_ops = strtol(optarg, NULL, 10); + break; + case 'b': + TEST_ASSERT(strlen(optarg) > 0, + "Burst size is not provided"); + tp->burst_sz = strtol(optarg, NULL, 10); + TEST_ASSERT(tp->burst_sz <= MAX_BURST, + "Burst size mustn't be greater than %u", + MAX_BURST); + break; + case 'c': + TEST_ASSERT(test_cases_present == false, + "Test cases provided more than once"); + test_cases_present = true; + + ret = rte_strsplit(optarg, strlen(optarg), + tokens, MAX_CMDLINE_TESTCASES, tc_sep); + + TEST_ASSERT(ret <= MAX_CMDLINE_TESTCASES, + "Too many test cases (max=%d)", + MAX_CMDLINE_TESTCASES); + + for (tc = 0; tc < ret; ++tc) { + /* Find matching test case */ + TAILQ_FOREACH(t, &commands_list, next) + if (!strcmp(tokens[tc], t->command)) + tp->test_to_run[num_tests] = t; + + TEST_ASSERT(tp->test_to_run[num_tests] != NULL, + "Unknown test case: %s", + tokens[tc]); + ++num_tests; + } + break; + case 'v': + TEST_ASSERT(test_vector_present == false, + "Test vector provided more than once"); + test_vector_present = true; + + TEST_ASSERT(strlen(optarg) > 0, + "Config file name is null"); + + snprintf(tp->test_vector_filename, + sizeof(tp->test_vector_filename), + "%s", optarg); + break; + case 'l': + TEST_ASSERT(strlen(optarg) > 0, + "Num of lcores is not provided"); + tp->num_lcores = strtol(optarg, NULL, 10); + TEST_ASSERT(tp->num_lcores <= RTE_MAX_LCORE, + "Num of lcores mustn't be greater than %u", + RTE_MAX_LCORE); + break; + case 'h': + print_usage(argv[0]); + return 0; + default: + printf("ERROR: Unknown option: -%c\n", opt); + return -1; + } + + if (tp->num_ops == 0) { + printf( + "WARNING: Num of operations was not provided or was set 0. Set to default (%u)\n", + DEFAULT_OPS); + tp->num_ops = DEFAULT_OPS; + } + if (tp->burst_sz == 0) { + printf( + "WARNING: Burst size was not provided or was set 0. Set to default (%u)\n", + DEFAULT_BURST); + tp->burst_sz = DEFAULT_BURST; + } + if (tp->num_lcores == 0) { + printf( + "WARNING: Num of lcores was not provided or was set 0. Set to value from RTE config (%u)\n", + rte_lcore_count()); + tp->num_lcores = rte_lcore_count(); + } + + TEST_ASSERT(tp->burst_sz <= tp->num_ops, + "Burst size (%u) mustn't be greater than num ops (%u)", + tp->burst_sz, tp->num_ops); + + tp->num_tests = num_tests; + return 0; +} + +static int +run_all_tests(void) +{ + int ret = TEST_SUCCESS; + struct test_command *t; + + TAILQ_FOREACH(t, &commands_list, next) + ret |= t->callback(); + + return ret; +} + +static int +run_parsed_tests(struct test_params *tp) +{ + int ret = TEST_SUCCESS; + unsigned int i; + + for (i = 0; i < tp->num_tests; ++i) + ret |= tp->test_to_run[i]->callback(); + + return ret; +} + +int +main(int argc, char **argv) +{ + int ret; + + /* Init EAL */ + ret = rte_eal_init(argc, argv); + if (ret < 0) + return 1; + argc -= ret; + argv += ret; + + /* Parse application arguments (after the EAL ones) */ + ret = parse_args(argc, argv, &test_params); + if (ret < 0) { + print_usage(argv[0]); + return 1; + } + + rte_log_set_global_level(RTE_LOG_INFO); + + /* If no argument provided - run all tests */ + if (test_params.num_tests == 0) + return run_all_tests(); + else + return run_parsed_tests(&test_params); +} diff --git a/dpdk/app/test-bbdev/main.h b/dpdk/app/test-bbdev/main.h new file mode 100644 index 00000000..20a55efc --- /dev/null +++ b/dpdk/app/test-bbdev/main.h @@ -0,0 +1,120 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#ifndef _MAIN_H_ +#define _MAIN_H_ + +#include +#include + +#include +#include +#include + +#define TEST_SUCCESS 0 +#define TEST_FAILED -1 +#define TEST_SKIPPED 1 + +#define MAX_BURST 512U +#define DEFAULT_BURST 32U +#define DEFAULT_OPS 64U + +#define TEST_ASSERT(cond, msg, ...) do { \ + if (!(cond)) { \ + printf("TestCase %s() line %d failed: " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__); \ + return TEST_FAILED; \ + } \ +} while (0) + +/* Compare two buffers (length in bytes) */ +#define TEST_ASSERT_BUFFERS_ARE_EQUAL(a, b, len, msg, ...) do { \ + if (memcmp((a), (b), len)) { \ + printf("TestCase %s() line %d failed: " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__); \ + rte_memdump(stdout, "Buffer A", (a), len); \ + rte_memdump(stdout, "Buffer B", (b), len); \ + return TEST_FAILED; \ + } \ +} while (0) + +#define TEST_ASSERT_SUCCESS(val, msg, ...) do { \ + typeof(val) _val = (val); \ + if (!(_val == 0)) { \ + printf("TestCase %s() line %d failed (err %d): " \ + msg "\n", __func__, __LINE__, _val, \ + ##__VA_ARGS__); \ + return TEST_FAILED; \ + } \ +} while (0) + +#define TEST_ASSERT_FAIL(val, msg, ...) \ + TEST_ASSERT_SUCCESS(!(val), msg, ##__VA_ARGS__) + +#define TEST_ASSERT_NOT_NULL(val, msg, ...) do { \ + if ((val) == NULL) { \ + printf("TestCase %s() line %d failed (null): " \ + msg "\n", __func__, __LINE__, ##__VA_ARGS__); \ + return TEST_FAILED; \ + } \ +} while (0) + +struct unit_test_case { + int (*setup)(void); + void (*teardown)(void); + int (*testcase)(void); + const char *name; +}; + +#define TEST_CASE(testcase) {NULL, NULL, testcase, #testcase} + +#define TEST_CASE_ST(setup, teardown, testcase) \ + {setup, teardown, testcase, #testcase} + +#define TEST_CASES_END() {NULL, NULL, NULL, NULL} + +struct unit_test_suite { + const char *suite_name; + int (*setup)(void); + void (*teardown)(void); + struct unit_test_case unit_test_cases[]; +}; + +int unit_test_suite_runner(struct unit_test_suite *suite); + +typedef int (test_callback)(void); +TAILQ_HEAD(test_commands_list, test_command); +struct test_command { + TAILQ_ENTRY(test_command) next; + const char *command; + test_callback *callback; +}; + +void add_test_command(struct test_command *t); + +/* Register a test function */ +#define REGISTER_TEST_COMMAND(name, testsuite) \ + static int test_func_##name(void) \ + { \ + return unit_test_suite_runner(&testsuite); \ + } \ + static struct test_command test_struct_##name = { \ + .command = RTE_STR(name), \ + .callback = test_func_##name, \ + }; \ + static void __attribute__((constructor, used)) \ + test_register_##name(void) \ + { \ + add_test_command(&test_struct_##name); \ + } + +const char *get_vector_filename(void); + +unsigned int get_num_ops(void); + +unsigned int get_burst_sz(void); + +unsigned int get_num_lcores(void); + +#endif diff --git a/dpdk/app/test-bbdev/meson.build b/dpdk/app/test-bbdev/meson.build new file mode 100644 index 00000000..eb8cc049 --- /dev/null +++ b/dpdk/app/test-bbdev/meson.build @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Intel Corporation + +sources = files('main.c', + 'test_bbdev.c', + 'test_bbdev_perf.c', + 'test_bbdev_vector.c') +allow_experimental_apis = true +deps += ['bbdev', 'bus_vdev'] diff --git a/dpdk/app/test-bbdev/test-bbdev.py b/dpdk/app/test-bbdev/test-bbdev.py new file mode 100755 index 00000000..acab9eb1 --- /dev/null +++ b/dpdk/app/test-bbdev/test-bbdev.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python + +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +import sys +import os +import argparse +import subprocess +import shlex + +from threading import Timer + +def kill(process): + print "ERROR: Test app timed out" + process.kill() + +if "RTE_SDK" in os.environ: + dpdk_path = os.environ["RTE_SDK"] +else: + dpdk_path = "../.." + +if "RTE_TARGET" in os.environ: + dpdk_target = os.environ["RTE_TARGET"] +else: + dpdk_target = "x86_64-native-linuxapp-gcc" + +parser = argparse.ArgumentParser( + description='BBdev Unit Test Application', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument("-p", "--testapp-path", + help="specifies path to the bbdev test app", + default=dpdk_path + "/" + dpdk_target + "/app/testbbdev") +parser.add_argument("-e", "--eal-params", + help="EAL arguments which are passed to the test app", + default="--vdev=baseband_null0") +parser.add_argument("-t", "--timeout", + type=int, + help="Timeout in seconds", + default=300) +parser.add_argument("-c", "--test-cases", + nargs="+", + help="Defines test cases to run. Run all if not specified") +parser.add_argument("-v", "--test-vector", + nargs="+", + help="Specifies paths to the test vector files.", + default=[dpdk_path + + "/app/test-bbdev/test_vectors/bbdev_null.data"]) +parser.add_argument("-n", "--num-ops", + type=int, + help="Number of operations to process on device.", + default=32) +parser.add_argument("-b", "--burst-size", + nargs="+", + type=int, + help="Operations enqueue/dequeue burst size.", + default=[32]) +parser.add_argument("-l", "--num-lcores", + type=int, + help="Number of lcores to run.", + default=16) + +args = parser.parse_args() + +if not os.path.exists(args.testapp_path): + print "No such file: " + args.testapp_path + sys.exit(1) + +params = [args.testapp_path] +if args.eal_params: + params.extend(shlex.split(args.eal_params)) + +params.extend(["--"]) + +if args.num_ops: + params.extend(["-n", str(args.num_ops)]) + +if args.num_lcores: + params.extend(["-l", str(args.num_lcores)]) + +if args.test_cases: + params.extend(["-c"]) + params.extend([",".join(args.test_cases)]) + +exit_status = 0 +for vector in args.test_vector: + for burst_size in args.burst_size: + call_params = params[:] + call_params.extend(["-v", vector]) + call_params.extend(["-b", str(burst_size)]) + params_string = " ".join(call_params) + + print("Executing: {}".format(params_string)) + app_proc = subprocess.Popen(call_params) + if args.timeout > 0: + timer = Timer(args.timeout, kill, [app_proc]) + timer.start() + + try: + app_proc.communicate() + except: + print("Error: failed to execute: {}".format(params_string)) + finally: + timer.cancel() + + if app_proc.returncode != 0: + exit_status = 1 + print("ERROR TestCase failed. Failed test for vector {}. Return code: {}".format( + vector, app_proc.returncode)) + +sys.exit(exit_status) diff --git a/dpdk/app/test-bbdev/test_bbdev.c b/dpdk/app/test-bbdev/test_bbdev.c new file mode 100644 index 00000000..a914817b --- /dev/null +++ b/dpdk/app/test-bbdev/test_bbdev.c @@ -0,0 +1,1371 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +#include "main.h" + + +#define BBDEV_NAME_NULL ("bbdev_null") + +struct bbdev_testsuite_params { + struct rte_bbdev_queue_conf qconf; +}; + +static struct bbdev_testsuite_params testsuite_params; + +static uint8_t null_dev_id; + +static int +testsuite_setup(void) +{ + uint8_t nb_devs; + int ret; + char buf[RTE_BBDEV_NAME_MAX_LEN]; + + /* Create test device */ + snprintf(buf, sizeof(buf), "%s_unittest", BBDEV_NAME_NULL); + ret = rte_vdev_init(buf, NULL); + TEST_ASSERT(ret == 0, "Failed to create instance of pmd: %s", buf); + + nb_devs = rte_bbdev_count(); + TEST_ASSERT(nb_devs != 0, "No devices found"); + + /* Most recently created device is our device */ + null_dev_id = nb_devs - 1; + + return TEST_SUCCESS; +} + +static void +testsuite_teardown(void) +{ + char buf[RTE_BBDEV_NAME_MAX_LEN]; + + snprintf(buf, sizeof(buf), "%s_unittest", BBDEV_NAME_NULL); + rte_vdev_uninit(buf); +} + +static int +ut_setup(void) +{ + struct bbdev_testsuite_params *ts_params = &testsuite_params; + uint8_t num_queues; + + /* Valid queue configuration */ + ts_params->qconf.priority = 0; + ts_params->qconf.socket = SOCKET_ID_ANY; + ts_params->qconf.deferred_start = 1; + + num_queues = 1; + TEST_ASSERT_SUCCESS(rte_bbdev_setup_queues(null_dev_id, num_queues, + SOCKET_ID_ANY), "Failed to setup queues for bbdev %u", + 0); + + /* Start the device */ + TEST_ASSERT_SUCCESS(rte_bbdev_start(null_dev_id), + "Failed to start bbdev %u", 0); + + return TEST_SUCCESS; +} + +static void +ut_teardown(void) +{ + rte_bbdev_close(null_dev_id); +} + +static int +test_bbdev_configure_invalid_dev_id(void) +{ + uint8_t dev_id; + uint8_t num_queues; + + num_queues = 1; + for (dev_id = 0; dev_id < RTE_BBDEV_MAX_DEVS; dev_id++) { + if (!rte_bbdev_is_valid(dev_id)) { + TEST_ASSERT_FAIL(rte_bbdev_setup_queues(dev_id, + num_queues, SOCKET_ID_ANY), + "Failed test for rte_bbdev_setup_queues: " + "invalid dev_num %u", dev_id); + TEST_ASSERT(rte_bbdev_intr_enable(dev_id) == -ENODEV, + "Failed test for rte_bbdev_intr_enable: " + "invalid dev_num %u", dev_id); + break; + } + } + + return TEST_SUCCESS; +} + +static int +test_bbdev_configure_invalid_num_queues(void) +{ + struct rte_bbdev_info info; + uint8_t dev_id, num_devs; + uint8_t num_queues; + int return_value; + + TEST_ASSERT((num_devs = rte_bbdev_count()) >= 1, + "Need at least %d devices for test", 1); + + /* valid num_queues values */ + num_queues = 8; + + /* valid dev_id values */ + dev_id = null_dev_id; + + /* Stop the device in case it's started so it can be configured */ + rte_bbdev_stop(dev_id); + + TEST_ASSERT_FAIL(rte_bbdev_setup_queues(dev_id, 0, SOCKET_ID_ANY), + "Failed test for rte_bbdev_setup_queues: " + "invalid num_queues %d", 0); + + TEST_ASSERT_SUCCESS(rte_bbdev_setup_queues(dev_id, num_queues, + SOCKET_ID_ANY), + "Failed test for rte_bbdev_setup_queues: " + "invalid dev_num %u", dev_id); + + TEST_ASSERT_FAIL(return_value = rte_bbdev_info_get(dev_id, NULL), + "Failed test for rte_bbdev_info_get: " + "returned value:%i", return_value); + + TEST_ASSERT_SUCCESS(return_value = rte_bbdev_info_get(dev_id, &info), + "Failed test for rte_bbdev_info_get: " + "invalid return value:%i", return_value); + + TEST_ASSERT(info.num_queues == num_queues, + "Failed test for rte_bbdev_info_get: " + "invalid num_queues:%u", info.num_queues); + + num_queues = info.drv.max_num_queues; + TEST_ASSERT_SUCCESS(rte_bbdev_setup_queues(dev_id, num_queues, + SOCKET_ID_ANY), + "Failed test for rte_bbdev_setup_queues: " + "invalid num_queues: %u", num_queues); + + num_queues++; + TEST_ASSERT_FAIL(rte_bbdev_setup_queues(dev_id, num_queues, + SOCKET_ID_ANY), + "Failed test for rte_bbdev_setup_queues: " + "invalid num_queues: %u", num_queues); + + return TEST_SUCCESS; +} + +static int +test_bbdev_configure_stop_device(void) +{ + struct rte_bbdev_info info; + uint8_t dev_id; + int return_value; + + /* valid dev_id values */ + dev_id = null_dev_id; + + /* Stop the device so it can be configured */ + rte_bbdev_stop(dev_id); + + TEST_ASSERT_SUCCESS(return_value = rte_bbdev_info_get(dev_id, &info), + "Failed test for rte_bbdev_info_get: " + "invalid return value from " + "rte_bbdev_info_get function: %i", return_value); + + TEST_ASSERT_SUCCESS(info.started, "Failed test for rte_bbdev_info_get: " + "started value: %u", info.started); + + TEST_ASSERT_SUCCESS(rte_bbdev_setup_queues(dev_id, + info.drv.max_num_queues, SOCKET_ID_ANY), + "Failed test for rte_bbdev_setup_queues: " + "device should be stopped, dev_id: %u", dev_id); + + return_value = rte_bbdev_intr_enable(dev_id); + TEST_ASSERT(return_value != -EBUSY, + "Failed test for rte_bbdev_intr_enable: device should be stopped, dev_id: %u", + dev_id); + + /* Start the device so it cannot be configured */ + TEST_ASSERT_FAIL(rte_bbdev_start(RTE_BBDEV_MAX_DEVS), + "Failed to start bbdev %u", dev_id); + + TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), + "Failed to start bbdev %u", dev_id); + + TEST_ASSERT_SUCCESS(return_value = rte_bbdev_info_get(dev_id, &info), + "Failed test for rte_bbdev_info_get: " + "invalid return value from " + "rte_bbdev_info_get function: %i", return_value); + + TEST_ASSERT_FAIL(info.started, "Failed test for rte_bbdev_info_get: " + "started value: %u", info.started); + + TEST_ASSERT_FAIL(rte_bbdev_setup_queues(dev_id, + info.drv.max_num_queues, SOCKET_ID_ANY), + "Failed test for rte_bbdev_setup_queues: " + "device should be started, dev_id: %u", dev_id); + + return_value = rte_bbdev_intr_enable(dev_id); + TEST_ASSERT(return_value == -EBUSY, + "Failed test for rte_bbdev_intr_enable: device should be started, dev_id: %u", + dev_id); + + /* Stop again the device so it can be once again configured */ + TEST_ASSERT_FAIL(rte_bbdev_stop(RTE_BBDEV_MAX_DEVS), + "Failed to start bbdev %u", dev_id); + + TEST_ASSERT_SUCCESS(rte_bbdev_stop(dev_id), "Failed to stop bbdev %u", + dev_id); + + TEST_ASSERT_SUCCESS(return_value = rte_bbdev_info_get(dev_id, &info), + "Failed test for rte_bbdev_info_get: " + "invalid return value from " + "rte_bbdev_info_get function: %i", return_value); + + TEST_ASSERT_SUCCESS(info.started, "Failed test for rte_bbdev_info_get: " + "started value: %u", info.started); + + TEST_ASSERT_SUCCESS(rte_bbdev_setup_queues(dev_id, + info.drv.max_num_queues, SOCKET_ID_ANY), + "Failed test for rte_bbdev_setup_queues: " + "device should be stopped, dev_id: %u", dev_id); + + return_value = rte_bbdev_intr_enable(dev_id); + TEST_ASSERT(return_value != -EBUSY, + "Failed test for rte_bbdev_intr_enable: device should be stopped, dev_id: %u", + dev_id); + + return TEST_SUCCESS; +} + +static int +test_bbdev_configure_stop_queue(void) +{ + struct bbdev_testsuite_params *ts_params = &testsuite_params; + struct rte_bbdev_info info; + struct rte_bbdev_queue_info qinfo; + uint8_t dev_id; + uint16_t queue_id; + int return_value; + + /* Valid dev_id values */ + dev_id = null_dev_id; + + /* Valid queue_id values */ + queue_id = 0; + + rte_bbdev_stop(dev_id); + TEST_ASSERT_SUCCESS(return_value = rte_bbdev_info_get(dev_id, &info), + "Failed test for rte_bbdev_info_get: " + "invalid return value:%i", return_value); + + /* Valid queue configuration */ + ts_params->qconf.queue_size = info.drv.queue_size_lim; + ts_params->qconf.priority = info.drv.max_ul_queue_priority; + + /* Device - started; queue - started */ + rte_bbdev_start(dev_id); + + TEST_ASSERT_FAIL(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed test for rte_bbdev_queue_configure: " + "queue:%u on device:%u should be stopped", + queue_id, dev_id); + + /* Device - stopped; queue - started */ + rte_bbdev_stop(dev_id); + + TEST_ASSERT_FAIL(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed test for rte_bbdev_queue_configure: " + "queue:%u on device:%u should be stopped", + queue_id, dev_id); + + TEST_ASSERT_FAIL(rte_bbdev_queue_stop(RTE_BBDEV_MAX_DEVS, queue_id), + "Failed test for rte_bbdev_queue_stop " + "invalid dev_id "); + + TEST_ASSERT_FAIL(rte_bbdev_queue_stop(dev_id, RTE_MAX_QUEUES_PER_PORT), + "Failed test for rte_bbdev_queue_stop " + "invalid queue_id "); + + /* Device - stopped; queue - stopped */ + rte_bbdev_queue_stop(dev_id, queue_id); + + TEST_ASSERT_SUCCESS(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed test for rte_bbdev_queue_configure: " + "queue:%u on device:%u should be stopped", queue_id, + dev_id); + + TEST_ASSERT_SUCCESS(return_value = rte_bbdev_queue_info_get(dev_id, + queue_id, &qinfo), + "Failed test for rte_bbdev_info_get: " + "invalid return value from " + "rte_bbdev_queue_info_get function: %i", return_value); + + TEST_ASSERT(qinfo.conf.socket == ts_params->qconf.socket, + "Failed test for rte_bbdev_queue_info_get: " + "invalid queue_size:%u", qinfo.conf.socket); + + TEST_ASSERT(qinfo.conf.queue_size == ts_params->qconf.queue_size, + "Failed test for rte_bbdev_queue_info_get: " + "invalid queue_size:%u", qinfo.conf.queue_size); + + TEST_ASSERT(qinfo.conf.priority == ts_params->qconf.priority, + "Failed test for rte_bbdev_queue_info_get: " + "invalid queue_size:%u", qinfo.conf.priority); + + TEST_ASSERT(qinfo.conf.deferred_start == + ts_params->qconf.deferred_start, + "Failed test for rte_bbdev_queue_info_get: " + "invalid queue_size:%u", qinfo.conf.deferred_start); + + /* Device - started; queue - stopped */ + rte_bbdev_start(dev_id); + rte_bbdev_queue_stop(dev_id, queue_id); + + TEST_ASSERT_FAIL(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed test for rte_bbdev_queue_configure: " + "queue:%u on device:%u should be stopped", queue_id, + dev_id); + + rte_bbdev_stop(dev_id); + + /* After rte_bbdev_start(dev_id): + * - queue should be still stopped if deferred_start == + */ + rte_bbdev_start(dev_id); + + TEST_ASSERT_SUCCESS(return_value = rte_bbdev_queue_info_get(dev_id, + queue_id, &qinfo), + "Failed test for rte_bbdev_info_get: " + "invalid return value from " + "rte_bbdev_queue_info_get function: %i", return_value); + + TEST_ASSERT(qinfo.started == 0, + "Failed test for rte_bbdev_queue_info_get: " + "invalid value for qinfo.started:%u", qinfo.started); + + rte_bbdev_stop(dev_id); + + /* After rte_bbdev_start(dev_id): + * - queue should be started if deferred_start == + */ + ts_params->qconf.deferred_start = 0; + rte_bbdev_queue_configure(dev_id, queue_id, &ts_params->qconf); + rte_bbdev_start(dev_id); + + TEST_ASSERT_SUCCESS(return_value = rte_bbdev_queue_info_get(dev_id, + queue_id, &qinfo), + "Failed test for rte_bbdev_info_get: " + "invalid return value from " + "rte_bbdev_queue_info_get function: %i", return_value); + + TEST_ASSERT(qinfo.started == 1, + "Failed test for rte_bbdev_queue_info_get: " + "invalid value for qinfo.started:%u", qinfo.started); + + return TEST_SUCCESS; +} + +static int +test_bbdev_configure_invalid_queue_configure(void) +{ + struct bbdev_testsuite_params *ts_params = &testsuite_params; + int return_value; + struct rte_bbdev_info info; + uint8_t dev_id; + uint16_t queue_id; + + /* Valid dev_id values */ + dev_id = null_dev_id; + + /* Valid queue_id values */ + queue_id = 0; + + rte_bbdev_stop(dev_id); + + TEST_ASSERT_SUCCESS(return_value = rte_bbdev_info_get(dev_id, &info), + "Failed test for rte_bbdev_info_get: " + "invalid return value:%i", return_value); + + rte_bbdev_queue_stop(dev_id, queue_id); + + ts_params->qconf.queue_size = info.drv.queue_size_lim + 1; + TEST_ASSERT_FAIL(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed test for rte_bbdev_queue_configure: " + "invalid value qconf.queue_size: %u", + ts_params->qconf.queue_size); + + ts_params->qconf.queue_size = info.drv.queue_size_lim; + ts_params->qconf.priority = info.drv.max_ul_queue_priority; + queue_id = info.num_queues; + TEST_ASSERT_FAIL(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed test for rte_bbdev_queue_configure: " + "invalid value queue_id: %u", queue_id); + + queue_id = 0; + TEST_ASSERT_SUCCESS(rte_bbdev_queue_configure(dev_id, queue_id, NULL), + "Failed test for rte_bbdev_queue_configure: " + "NULL qconf structure "); + + ts_params->qconf.socket = RTE_MAX_NUMA_NODES; + TEST_ASSERT_FAIL(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed test for rte_bbdev_queue_configure: " + "invalid socket number "); + + ts_params->qconf.socket = SOCKET_ID_ANY; + TEST_ASSERT_SUCCESS(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed test for rte_bbdev_queue_configure: " + "invalid value qconf.queue_size: %u", + ts_params->qconf.queue_size); + + TEST_ASSERT_FAIL(rte_bbdev_queue_configure(RTE_BBDEV_MAX_DEVS, queue_id, + &ts_params->qconf), + "Failed test for rte_bbdev_queue_configure: " + "invalid dev_id"); + + TEST_ASSERT_SUCCESS(rte_bbdev_queue_configure(dev_id, queue_id, NULL), + "Failed test for rte_bbdev_queue_configure: " + "invalid value qconf.queue_size: %u", + ts_params->qconf.queue_size); + + return TEST_SUCCESS; +} + +static int +test_bbdev_op_pool(void) +{ + struct rte_mempool *mp; + + unsigned int dec_size = sizeof(struct rte_bbdev_dec_op); + unsigned int enc_size = sizeof(struct rte_bbdev_enc_op); + + const char *pool_dec = "Test_DEC"; + const char *pool_enc = "Test_ENC"; + + /* Valid pool configuration */ + uint32_t size = 256; + uint32_t cache_size = 128; + + TEST_ASSERT(rte_bbdev_op_pool_create(NULL, + RTE_BBDEV_OP_TURBO_DEC, size, cache_size, 0) == NULL, + "Failed test for rte_bbdev_op_pool_create: " + "NULL name parameter"); + + TEST_ASSERT((mp = rte_bbdev_op_pool_create(pool_dec, + RTE_BBDEV_OP_TURBO_DEC, size, cache_size, 0)) != NULL, + "Failed test for rte_bbdev_op_pool_create: " + "returned value is empty"); + + TEST_ASSERT(mp->size == size, + "Failed test for rte_bbdev_op_pool_create: " + "invalid size of the mempool, mp->size: %u", mp->size); + + TEST_ASSERT(mp->cache_size == cache_size, + "Failed test for rte_bbdev_op_pool_create: " + "invalid size of the mempool, mp->size: %u", + mp->cache_size); + + TEST_ASSERT_SUCCESS(strcmp(mp->name, pool_dec), + "Failed test for rte_bbdev_op_pool_create: " + "invalid name of mempool, mp->name: %s", mp->name); + + TEST_ASSERT(mp->elt_size == dec_size, + "Failed test for rte_bbdev_op_pool_create: " + "invalid element size for RTE_BBDEV_OP_TURBO_DEC, " + "mp->elt_size: %u", mp->elt_size); + + rte_mempool_free(mp); + + TEST_ASSERT((mp = rte_bbdev_op_pool_create(pool_enc, + RTE_BBDEV_OP_TURBO_ENC, size, cache_size, 0)) != NULL, + "Failed test for rte_bbdev_op_pool_create: " + "returned value is empty"); + + TEST_ASSERT(mp->elt_size == enc_size, + "Failed test for rte_bbdev_op_pool_create: " + "invalid element size for RTE_BBDEV_OP_TURBO_ENC, " + "mp->elt_size: %u", mp->elt_size); + + rte_mempool_free(mp); + + TEST_ASSERT((mp = rte_bbdev_op_pool_create("Test_NONE", + RTE_BBDEV_OP_NONE, size, cache_size, 0)) != NULL, + "Failed test for rte_bbdev_op_pool_create: " + "returned value is empty for RTE_BBDEV_OP_NONE"); + + TEST_ASSERT(mp->elt_size == (enc_size > dec_size ? enc_size : dec_size), + "Failed test for rte_bbdev_op_pool_create: " + "invalid size for RTE_BBDEV_OP_NONE, mp->elt_size: %u", + mp->elt_size); + + rte_mempool_free(mp); + + TEST_ASSERT((mp = rte_bbdev_op_pool_create("Test_INV", + RTE_BBDEV_OP_TYPE_COUNT, size, cache_size, 0)) == NULL, + "Failed test for rte_bbdev_op_pool_create: " + "returned value is not NULL for invalid type"); + + /* Invalid pool configuration */ + size = 128; + cache_size = 256; + + TEST_ASSERT((mp = rte_bbdev_op_pool_create("Test_InvSize", + RTE_BBDEV_OP_NONE, size, cache_size, 0)) == NULL, + "Failed test for rte_bbdev_op_pool_create: " + "returned value should be empty " + "because size of per-lcore local cache " + "is greater than size of the mempool."); + + return TEST_SUCCESS; +} + +/** + * Create pool of OP types RTE_BBDEV_OP_NONE, RTE_BBDEV_OP_TURBO_DEC and + * RTE_BBDEV_OP_TURBO_ENC and check that only ops of that type can be + * allocated + */ +static int +test_bbdev_op_type(void) +{ + struct rte_mempool *mp_dec; + + const unsigned int OPS_COUNT = 32; + struct rte_bbdev_dec_op *dec_ops_arr[OPS_COUNT]; + struct rte_bbdev_enc_op *enc_ops_arr[OPS_COUNT]; + + const char *pool_dec = "Test_op_dec"; + + /* Valid pool configuration */ + uint32_t num_elements = 256; + uint32_t cache_size = 128; + + /* mempool type : RTE_BBDEV_OP_TURBO_DEC */ + mp_dec = rte_bbdev_op_pool_create(pool_dec, + RTE_BBDEV_OP_TURBO_DEC, num_elements, cache_size, 0); + TEST_ASSERT(mp_dec != NULL, "Failed to create %s mempool", pool_dec); + + TEST_ASSERT(rte_bbdev_dec_op_alloc_bulk(mp_dec, dec_ops_arr, 1) == 0, + "Failed test for rte_bbdev_op_alloc_bulk TURBO_DEC: " + "OPs type: RTE_BBDEV_OP_TURBO_DEC"); + + TEST_ASSERT(rte_bbdev_enc_op_alloc_bulk(mp_dec, enc_ops_arr, 1) != 0, + "Failed test for rte_bbdev_op_alloc_bulk TURBO_DEC: " + "OPs type: RTE_BBDEV_OP_TURBO_ENC"); + + rte_mempool_free(mp_dec); + + return TEST_SUCCESS; +} + +static int +test_bbdev_op_pool_size(void) +{ + struct rte_mempool *mp_none; + + const unsigned int OPS_COUNT = 128; + struct rte_bbdev_enc_op *ops_enc_arr[OPS_COUNT]; + struct rte_bbdev_enc_op *ops_ext_arr[OPS_COUNT]; + struct rte_bbdev_enc_op *ops_ext2_arr[OPS_COUNT]; + + const char *pool_none = "Test_pool_size"; + + /* Valid pool configuration */ + uint32_t num_elements = 256; + uint32_t cache_size = 0; + + /* Create mempool type : RTE_BBDEV_OP_TURBO_ENC, size : 256 */ + mp_none = rte_bbdev_op_pool_create(pool_none, RTE_BBDEV_OP_TURBO_ENC, + num_elements, cache_size, 0); + TEST_ASSERT(mp_none != NULL, "Failed to create %s mempool", pool_none); + + /* Add 128 RTE_BBDEV_OP_TURBO_ENC ops */ + rte_bbdev_enc_op_alloc_bulk(mp_none, ops_enc_arr, OPS_COUNT); + + /* Add 128 RTE_BBDEV_OP_TURBO_ENC ops */ + TEST_ASSERT(rte_bbdev_enc_op_alloc_bulk(mp_none, ops_ext_arr, + OPS_COUNT) == 0, + "Failed test for allocating bbdev ops: " + "Mempool size: 256, Free : 128, Attempted to add: 128"); + + /* Try adding 128 more RTE_BBDEV_OP_TURBO_ENC ops, this should fail */ + TEST_ASSERT(rte_bbdev_enc_op_alloc_bulk(mp_none, ops_ext2_arr, + OPS_COUNT) != 0, + "Failed test for allocating bbdev ops: " + "Mempool size: 256, Free : 0, Attempted to add: 128"); + + /* Free-up 128 RTE_BBDEV_OP_TURBO_ENC ops */ + rte_bbdev_enc_op_free_bulk(ops_enc_arr, OPS_COUNT); + + /* Try adding 128 RTE_BBDEV_OP_TURBO_DEC ops, this should succeed */ + /* Cache size > 0 causes reallocation of ops size > 127 fail */ + TEST_ASSERT(rte_bbdev_enc_op_alloc_bulk(mp_none, ops_ext2_arr, + OPS_COUNT) == 0, + "Failed test for allocating ops after mempool freed: " + "Mempool size: 256, Free : 128, Attempted to add: 128"); + + rte_mempool_free(mp_none); + + return TEST_SUCCESS; +} + +static int +test_bbdev_count(void) +{ + uint8_t num_devs, num_valid_devs = 0; + + for (num_devs = 0; num_devs < RTE_BBDEV_MAX_DEVS; num_devs++) { + if (rte_bbdev_is_valid(num_devs)) + num_valid_devs++; + } + + num_devs = rte_bbdev_count(); + TEST_ASSERT(num_valid_devs == num_devs, + "Failed test for rte_bbdev_is_valid: " + "invalid num_devs %u ", num_devs); + + return TEST_SUCCESS; +} + +static int +test_bbdev_stats(void) +{ + uint8_t dev_id = null_dev_id; + uint16_t queue_id = 0; + struct rte_bbdev_dec_op *dec_ops[4096] = { 0 }; + struct rte_bbdev_dec_op *dec_proc_ops[4096] = { 0 }; + struct rte_bbdev_enc_op *enc_ops[4096] = { 0 }; + struct rte_bbdev_enc_op *enc_proc_ops[4096] = { 0 }; + uint16_t num_ops = 236; + struct rte_bbdev_stats stats; + struct bbdev_testsuite_params *ts_params = &testsuite_params; + + TEST_ASSERT_SUCCESS(rte_bbdev_queue_stop(dev_id, queue_id), + "Failed to stop queue %u on device %u ", queue_id, + dev_id); + TEST_ASSERT_SUCCESS(rte_bbdev_stop(dev_id), + "Failed to stop bbdev %u ", dev_id); + + TEST_ASSERT_SUCCESS(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed to configure queue %u on device %u ", + queue_id, dev_id); + + TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), + "Failed to start bbdev %u ", dev_id); + + TEST_ASSERT_SUCCESS(rte_bbdev_queue_start(dev_id, queue_id), + "Failed to start queue %u on device %u ", queue_id, + dev_id); + + TEST_ASSERT_SUCCESS(rte_bbdev_queue_start(dev_id, queue_id), + "Failed to start queue %u on device %u ", queue_id, + dev_id); + + /* Tests after enqueue operation */ + rte_bbdev_enqueue_enc_ops(dev_id, queue_id, enc_ops, num_ops); + rte_bbdev_enqueue_dec_ops(dev_id, queue_id, dec_ops, num_ops); + + TEST_ASSERT_FAIL(rte_bbdev_stats_get(RTE_BBDEV_MAX_DEVS, &stats), + "Failed test for rte_bbdev_stats_get on device %u ", + dev_id); + + TEST_ASSERT_FAIL(rte_bbdev_stats_get(dev_id, NULL), + "Failed test for rte_bbdev_stats_get on device %u ", + dev_id); + + TEST_ASSERT_SUCCESS(rte_bbdev_stats_get(dev_id, &stats), + "Failed test for rte_bbdev_stats_get on device %u ", + dev_id); + + TEST_ASSERT(stats.enqueued_count == 2 * num_ops, + "Failed test for rte_bbdev_enqueue_ops: " + "invalid enqueued_count %" PRIu64 " ", + stats.enqueued_count); + + TEST_ASSERT(stats.dequeued_count == 0, + "Failed test for rte_bbdev_stats_reset: " + "invalid dequeued_count %" PRIu64 " ", + stats.dequeued_count); + + /* Tests after dequeue operation */ + rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_proc_ops, num_ops); + rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_proc_ops, num_ops); + + TEST_ASSERT_SUCCESS(rte_bbdev_stats_get(dev_id, &stats), + "Failed test for rte_bbdev_stats_get on device %u ", + dev_id); + + TEST_ASSERT(stats.dequeued_count == 2 * num_ops, + "Failed test for rte_bbdev_dequeue_ops: " + "invalid enqueued_count %" PRIu64 " ", + stats.dequeued_count); + + TEST_ASSERT(stats.enqueue_err_count == 0, + "Failed test for rte_bbdev_stats_reset: " + "invalid enqueue_err_count %" PRIu64 " ", + stats.enqueue_err_count); + + TEST_ASSERT(stats.dequeue_err_count == 0, + "Failed test for rte_bbdev_stats_reset: " + "invalid dequeue_err_count %" PRIu64 " ", + stats.dequeue_err_count); + + /* Tests after reset operation */ + TEST_ASSERT_FAIL(rte_bbdev_stats_reset(RTE_BBDEV_MAX_DEVS), + "Failed to reset statistic for device %u ", dev_id); + + TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), + "Failed to reset statistic for device %u ", dev_id); + TEST_ASSERT_SUCCESS(rte_bbdev_stats_get(dev_id, &stats), + "Failed test for rte_bbdev_stats_get on device %u ", + dev_id); + + TEST_ASSERT(stats.enqueued_count == 0, + "Failed test for rte_bbdev_stats_reset: " + "invalid enqueued_count %" PRIu64 " ", + stats.enqueued_count); + + TEST_ASSERT(stats.dequeued_count == 0, + "Failed test for rte_bbdev_stats_reset: " + "invalid dequeued_count %" PRIu64 " ", + stats.dequeued_count); + + TEST_ASSERT(stats.enqueue_err_count == 0, + "Failed test for rte_bbdev_stats_reset: " + "invalid enqueue_err_count %" PRIu64 " ", + stats.enqueue_err_count); + + TEST_ASSERT(stats.dequeue_err_count == 0, + "Failed test for rte_bbdev_stats_reset: " + "invalid dequeue_err_count %" PRIu64 " ", + stats.dequeue_err_count); + + return TEST_SUCCESS; +} + +static int +test_bbdev_driver_init(void) +{ + struct rte_bbdev *dev1, *dev2; + const char *name = "dev_name"; + char name_tmp[16]; + int num_devs, num_devs_tmp; + + dev1 = rte_bbdev_allocate(NULL); + TEST_ASSERT(dev1 == NULL, + "Failed initialize bbdev driver with NULL name"); + + dev1 = rte_bbdev_allocate(name); + TEST_ASSERT(dev1 != NULL, "Failed to initialize bbdev driver"); + + dev2 = rte_bbdev_allocate(name); + TEST_ASSERT(dev2 == NULL, + "Failed to initialize bbdev driver: " + "driver with the same name has been initialized before"); + + num_devs = rte_bbdev_count() - 1; + num_devs_tmp = num_devs; + + /* Initialize the maximum amount of devices */ + do { + sprintf(name_tmp, "%s%i", "name_", num_devs); + dev2 = rte_bbdev_allocate(name_tmp); + TEST_ASSERT(dev2 != NULL, + "Failed to initialize bbdev driver"); + ++num_devs; + } while (num_devs < (RTE_BBDEV_MAX_DEVS - 1)); + + sprintf(name_tmp, "%s%i", "name_", num_devs); + dev2 = rte_bbdev_allocate(name_tmp); + TEST_ASSERT(dev2 == NULL, "Failed to initialize bbdev driver number %d " + "more drivers than RTE_BBDEV_MAX_DEVS: %d ", num_devs, + RTE_BBDEV_MAX_DEVS); + + num_devs--; + + while (num_devs >= num_devs_tmp) { + sprintf(name_tmp, "%s%i", "name_", num_devs); + dev2 = rte_bbdev_get_named_dev(name_tmp); + TEST_ASSERT_SUCCESS(rte_bbdev_release(dev2), + "Failed to uninitialize bbdev driver %s ", + name_tmp); + num_devs--; + } + + TEST_ASSERT(dev1->data->dev_id < RTE_BBDEV_MAX_DEVS, + "Failed test rte_bbdev_allocate: " + "invalid dev_id %" PRIu8 ", max number of devices %d ", + dev1->data->dev_id, RTE_BBDEV_MAX_DEVS); + + TEST_ASSERT(dev1->state == RTE_BBDEV_INITIALIZED, + "Failed test rte_bbdev_allocate: " + "invalid state %d (0 - RTE_BBDEV_UNUSED, 1 - RTE_BBDEV_INITIALIZED", + dev1->state); + + TEST_ASSERT_FAIL(rte_bbdev_release(NULL), + "Failed to uninitialize bbdev driver with NULL bbdev"); + + sprintf(name_tmp, "%s", "invalid_name"); + dev2 = rte_bbdev_get_named_dev(name_tmp); + TEST_ASSERT_FAIL(rte_bbdev_release(dev2), + "Failed to uninitialize bbdev driver with invalid name"); + + dev2 = rte_bbdev_get_named_dev(name); + TEST_ASSERT_SUCCESS(rte_bbdev_release(dev2), + "Failed to uninitialize bbdev driver: %s ", name); + + return TEST_SUCCESS; +} + +static void +event_callback(uint16_t dev_id, enum rte_bbdev_event_type type, void *param, + void *ret_param) +{ + RTE_SET_USED(dev_id); + RTE_SET_USED(ret_param); + + if (param == NULL) + return; + + if (type == RTE_BBDEV_EVENT_UNKNOWN || + type == RTE_BBDEV_EVENT_ERROR || + type == RTE_BBDEV_EVENT_MAX) + *(int *)param = type; +} + +static int +test_bbdev_callback(void) +{ + struct rte_bbdev *dev1, *dev2; + const char *name = "dev_name1"; + const char *name2 = "dev_name2"; + int event_status; + uint8_t invalid_dev_id = RTE_BBDEV_MAX_DEVS; + enum rte_bbdev_event_type invalid_event_type = RTE_BBDEV_EVENT_MAX; + uint8_t dev_id; + + dev1 = rte_bbdev_allocate(name); + TEST_ASSERT(dev1 != NULL, "Failed to initialize bbdev driver"); + + /* + * RTE_BBDEV_EVENT_UNKNOWN - unregistered + * RTE_BBDEV_EVENT_ERROR - unregistered + */ + event_status = -1; + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_UNKNOWN, NULL); + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_ERROR, NULL); + TEST_ASSERT(event_status == -1, + "Failed test for rte_bbdev_pmd_callback_process: " + "events were not registered "); + + TEST_ASSERT_FAIL(rte_bbdev_callback_register(dev1->data->dev_id, + RTE_BBDEV_EVENT_MAX, event_callback, NULL), + "Failed to callback register for RTE_BBDEV_EVENT_MAX "); + + TEST_ASSERT_FAIL(rte_bbdev_callback_unregister(dev1->data->dev_id, + RTE_BBDEV_EVENT_MAX, event_callback, NULL), + "Failed to unregister RTE_BBDEV_EVENT_MAX "); + + /* + * RTE_BBDEV_EVENT_UNKNOWN - registered + * RTE_BBDEV_EVENT_ERROR - unregistered + */ + TEST_ASSERT_SUCCESS(rte_bbdev_callback_register(dev1->data->dev_id, + RTE_BBDEV_EVENT_UNKNOWN, event_callback, &event_status), + "Failed to callback rgstr for RTE_BBDEV_EVENT_UNKNOWN"); + + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_UNKNOWN, NULL); + TEST_ASSERT(event_status == (int) RTE_BBDEV_EVENT_UNKNOWN, + "Failed test for rte_bbdev_pmd_callback_process " + "for RTE_BBDEV_EVENT_UNKNOWN "); + + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_ERROR, NULL); + TEST_ASSERT(event_status == (int) RTE_BBDEV_EVENT_UNKNOWN, + "Failed test for rte_bbdev_pmd_callback_process: " + "event RTE_BBDEV_EVENT_ERROR was not registered "); + + /* + * RTE_BBDEV_EVENT_UNKNOWN - registered + * RTE_BBDEV_EVENT_ERROR - registered + */ + TEST_ASSERT_SUCCESS(rte_bbdev_callback_register(dev1->data->dev_id, + RTE_BBDEV_EVENT_ERROR, event_callback, &event_status), + "Failed to callback rgstr for RTE_BBDEV_EVENT_ERROR "); + + TEST_ASSERT_SUCCESS(rte_bbdev_callback_register(dev1->data->dev_id, + RTE_BBDEV_EVENT_ERROR, event_callback, &event_status), + "Failed to callback register for RTE_BBDEV_EVENT_ERROR" + "(re-registration) "); + + event_status = -1; + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_UNKNOWN, NULL); + TEST_ASSERT(event_status == (int) RTE_BBDEV_EVENT_UNKNOWN, + "Failed test for rte_bbdev_pmd_callback_process " + "for RTE_BBDEV_EVENT_UNKNOWN "); + + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_ERROR, NULL); + TEST_ASSERT(event_status == (int) RTE_BBDEV_EVENT_ERROR, + "Failed test for rte_bbdev_pmd_callback_process " + "for RTE_BBDEV_EVENT_ERROR "); + + /* + * RTE_BBDEV_EVENT_UNKNOWN - registered + * RTE_BBDEV_EVENT_ERROR - unregistered + */ + TEST_ASSERT_SUCCESS(rte_bbdev_callback_unregister(dev1->data->dev_id, + RTE_BBDEV_EVENT_ERROR, event_callback, &event_status), + "Failed to unregister RTE_BBDEV_EVENT_ERROR "); + + event_status = -1; + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_UNKNOWN, NULL); + TEST_ASSERT(event_status == (int) RTE_BBDEV_EVENT_UNKNOWN, + "Failed test for rte_bbdev_pmd_callback_process " + "for RTE_BBDEV_EVENT_UNKNOWN "); + + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_ERROR, NULL); + TEST_ASSERT(event_status == (int) RTE_BBDEV_EVENT_UNKNOWN, + "Failed test for rte_bbdev_pmd_callback_process: " + "event RTE_BBDEV_EVENT_ERROR was unregistered "); + + /* rte_bbdev_callback_register with invalid inputs */ + TEST_ASSERT_FAIL(rte_bbdev_callback_register(invalid_dev_id, + RTE_BBDEV_EVENT_ERROR, event_callback, &event_status), + "Failed test for rte_bbdev_callback_register " + "for invalid_dev_id "); + + TEST_ASSERT_FAIL(rte_bbdev_callback_register(dev1->data->dev_id, + invalid_event_type, event_callback, &event_status), + "Failed to callback register for invalid event type "); + + TEST_ASSERT_FAIL(rte_bbdev_callback_register(dev1->data->dev_id, + RTE_BBDEV_EVENT_ERROR, NULL, &event_status), + "Failed to callback register - no callback function "); + + /* The impact of devices on each other */ + dev2 = rte_bbdev_allocate(name2); + TEST_ASSERT(dev2 != NULL, + "Failed to initialize bbdev driver"); + + /* + * dev2: + * RTE_BBDEV_EVENT_UNKNOWN - unregistered + * RTE_BBDEV_EVENT_ERROR - unregistered + */ + event_status = -1; + rte_bbdev_pmd_callback_process(dev2, RTE_BBDEV_EVENT_UNKNOWN, NULL); + rte_bbdev_pmd_callback_process(dev2, RTE_BBDEV_EVENT_ERROR, NULL); + TEST_ASSERT(event_status == -1, + "Failed test for rte_bbdev_pmd_callback_process: " + "events were not registered "); + + /* + * dev1: RTE_BBDEV_EVENT_ERROR - unregistered + * dev2: RTE_BBDEV_EVENT_ERROR - registered + */ + TEST_ASSERT_SUCCESS(rte_bbdev_callback_register(dev2->data->dev_id, + RTE_BBDEV_EVENT_ERROR, event_callback, &event_status), + "Failed to callback rgstr for RTE_BBDEV_EVENT_ERROR"); + + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_ERROR, NULL); + TEST_ASSERT(event_status == -1, + "Failed test for rte_bbdev_pmd_callback_process in dev1 " + "for RTE_BBDEV_EVENT_ERROR "); + + rte_bbdev_pmd_callback_process(dev2, RTE_BBDEV_EVENT_ERROR, NULL); + TEST_ASSERT(event_status == (int) RTE_BBDEV_EVENT_ERROR, + "Failed test for rte_bbdev_pmd_callback_process in dev2 " + "for RTE_BBDEV_EVENT_ERROR "); + + /* + * dev1: RTE_BBDEV_EVENT_UNKNOWN - registered + * dev2: RTE_BBDEV_EVENT_UNKNOWN - unregistered + */ + TEST_ASSERT_SUCCESS(rte_bbdev_callback_register(dev2->data->dev_id, + RTE_BBDEV_EVENT_UNKNOWN, event_callback, &event_status), + "Failed to callback register for RTE_BBDEV_EVENT_UNKNOWN " + "in dev 2 "); + + rte_bbdev_pmd_callback_process(dev2, RTE_BBDEV_EVENT_UNKNOWN, NULL); + TEST_ASSERT(event_status == (int) RTE_BBDEV_EVENT_UNKNOWN, + "Failed test for rte_bbdev_pmd_callback_process in dev2" + " for RTE_BBDEV_EVENT_UNKNOWN "); + + TEST_ASSERT_SUCCESS(rte_bbdev_callback_unregister(dev2->data->dev_id, + RTE_BBDEV_EVENT_UNKNOWN, event_callback, &event_status), + "Failed to unregister RTE_BBDEV_EVENT_UNKNOWN "); + + TEST_ASSERT_SUCCESS(rte_bbdev_callback_unregister(dev2->data->dev_id, + RTE_BBDEV_EVENT_UNKNOWN, event_callback, &event_status), + "Failed to unregister RTE_BBDEV_EVENT_UNKNOWN : " + "unregister function called once again "); + + event_status = -1; + rte_bbdev_pmd_callback_process(dev2, RTE_BBDEV_EVENT_UNKNOWN, NULL); + TEST_ASSERT(event_status == -1, + "Failed test for rte_bbdev_pmd_callback_process in dev2" + " for RTE_BBDEV_EVENT_UNKNOWN "); + + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_UNKNOWN, NULL); + TEST_ASSERT(event_status == (int) RTE_BBDEV_EVENT_UNKNOWN, + "Failed test for rte_bbdev_pmd_callback_process in dev2 " + "for RTE_BBDEV_EVENT_UNKNOWN "); + + /* rte_bbdev_pmd_callback_process with invalid inputs */ + rte_bbdev_pmd_callback_process(NULL, RTE_BBDEV_EVENT_UNKNOWN, NULL); + + event_status = -1; + rte_bbdev_pmd_callback_process(dev1, invalid_event_type, NULL); + TEST_ASSERT(event_status == -1, + "Failed test for rte_bbdev_pmd_callback_process: " + "for invalid event type "); + + /* rte_dev_callback_unregister with invalid inputs */ + TEST_ASSERT_FAIL(rte_bbdev_callback_unregister(invalid_dev_id, + RTE_BBDEV_EVENT_UNKNOWN, event_callback, &event_status), + "Failed test for rte_dev_callback_unregister " + "for invalid_dev_id "); + + TEST_ASSERT_FAIL(rte_bbdev_callback_unregister(dev1->data->dev_id, + invalid_event_type, event_callback, &event_status), + "Failed rte_dev_callback_unregister " + "for invalid event type "); + + TEST_ASSERT_FAIL(rte_bbdev_callback_unregister(dev1->data->dev_id, + invalid_event_type, NULL, &event_status), + "Failed rte_dev_callback_unregister " + "when no callback function "); + + dev_id = dev1->data->dev_id; + + rte_bbdev_release(dev1); + rte_bbdev_release(dev2); + + TEST_ASSERT_FAIL(rte_bbdev_callback_register(dev_id, + RTE_BBDEV_EVENT_ERROR, event_callback, &event_status), + "Failed test for rte_bbdev_callback_register: " + "function called after rte_bbdev_driver_uninit ."); + + TEST_ASSERT_FAIL(rte_bbdev_callback_unregister(dev_id, + RTE_BBDEV_EVENT_ERROR, event_callback, &event_status), + "Failed test for rte_dev_callback_unregister: " + "function called after rte_bbdev_driver_uninit. "); + + event_status = -1; + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_UNKNOWN, NULL); + rte_bbdev_pmd_callback_process(dev1, RTE_BBDEV_EVENT_ERROR, NULL); + rte_bbdev_pmd_callback_process(dev2, RTE_BBDEV_EVENT_UNKNOWN, NULL); + rte_bbdev_pmd_callback_process(dev2, RTE_BBDEV_EVENT_ERROR, NULL); + TEST_ASSERT(event_status == -1, + "Failed test for rte_bbdev_pmd_callback_process: " + "callback function was called after rte_bbdev_driver_uninit"); + + return TEST_SUCCESS; +} + +static int +test_bbdev_invalid_driver(void) +{ + struct rte_bbdev dev1, *dev2; + uint8_t dev_id = null_dev_id; + uint16_t queue_id = 0; + struct rte_bbdev_stats stats; + struct bbdev_testsuite_params *ts_params = &testsuite_params; + struct rte_bbdev_queue_info qinfo; + struct rte_bbdev_ops dev_ops_tmp; + + TEST_ASSERT_SUCCESS(rte_bbdev_stop(dev_id), "Failed to stop bbdev %u ", + dev_id); + + dev1 = rte_bbdev_devices[dev_id]; + dev2 = &rte_bbdev_devices[dev_id]; + + /* Tests for rte_bbdev_setup_queues */ + dev2->dev_ops = NULL; + TEST_ASSERT_FAIL(rte_bbdev_setup_queues(dev_id, 1, SOCKET_ID_ANY), + "Failed test for rte_bbdev_setup_queues: " + "NULL dev_ops structure "); + dev2->dev_ops = dev1.dev_ops; + + dev_ops_tmp = *dev2->dev_ops; + dev_ops_tmp.info_get = NULL; + dev2->dev_ops = &dev_ops_tmp; + TEST_ASSERT_FAIL(rte_bbdev_setup_queues(dev_id, 1, SOCKET_ID_ANY), + "Failed test for rte_bbdev_setup_queues: " + "NULL info_get "); + dev2->dev_ops = dev1.dev_ops; + + dev_ops_tmp = *dev2->dev_ops; + dev_ops_tmp.queue_release = NULL; + dev2->dev_ops = &dev_ops_tmp; + TEST_ASSERT_FAIL(rte_bbdev_setup_queues(dev_id, 1, SOCKET_ID_ANY), + "Failed test for rte_bbdev_setup_queues: " + "NULL queue_release "); + dev2->dev_ops = dev1.dev_ops; + + dev2->data->socket_id = SOCKET_ID_ANY; + TEST_ASSERT_SUCCESS(rte_bbdev_setup_queues(dev_id, 1, + SOCKET_ID_ANY), "Failed to configure bbdev %u", dev_id); + + /* Test for rte_bbdev_queue_configure */ + dev2->dev_ops = NULL; + TEST_ASSERT_FAIL(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed to configure queue %u on device %u " + "with NULL dev_ops structure ", queue_id, dev_id); + dev2->dev_ops = dev1.dev_ops; + + dev_ops_tmp = *dev2->dev_ops; + dev_ops_tmp.queue_setup = NULL; + dev2->dev_ops = &dev_ops_tmp; + TEST_ASSERT_FAIL(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed to configure queue %u on device %u " + "with NULL queue_setup ", queue_id, dev_id); + dev2->dev_ops = dev1.dev_ops; + + dev_ops_tmp = *dev2->dev_ops; + dev_ops_tmp.info_get = NULL; + dev2->dev_ops = &dev_ops_tmp; + TEST_ASSERT_FAIL(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed to configure queue %u on device %u " + "with NULL info_get ", queue_id, dev_id); + dev2->dev_ops = dev1.dev_ops; + + TEST_ASSERT_FAIL(rte_bbdev_queue_configure(RTE_BBDEV_MAX_DEVS, + queue_id, &ts_params->qconf), + "Failed to configure queue %u on device %u ", + queue_id, dev_id); + + TEST_ASSERT_SUCCESS(rte_bbdev_queue_configure(dev_id, queue_id, + &ts_params->qconf), + "Failed to configure queue %u on device %u ", + queue_id, dev_id); + + /* Test for rte_bbdev_queue_info_get */ + dev2->dev_ops = NULL; + TEST_ASSERT_SUCCESS(rte_bbdev_queue_info_get(dev_id, queue_id, &qinfo), + "Failed test for rte_bbdev_info_get: " + "NULL dev_ops structure "); + dev2->dev_ops = dev1.dev_ops; + + TEST_ASSERT_FAIL(rte_bbdev_queue_info_get(RTE_BBDEV_MAX_DEVS, + queue_id, &qinfo), + "Failed test for rte_bbdev_info_get: " + "invalid dev_id "); + + TEST_ASSERT_FAIL(rte_bbdev_queue_info_get(dev_id, + RTE_MAX_QUEUES_PER_PORT, &qinfo), + "Failed test for rte_bbdev_info_get: " + "invalid queue_id "); + + TEST_ASSERT_FAIL(rte_bbdev_queue_info_get(dev_id, queue_id, NULL), + "Failed test for rte_bbdev_info_get: " + "invalid dev_info "); + + /* Test for rte_bbdev_start */ + dev2->dev_ops = NULL; + TEST_ASSERT_FAIL(rte_bbdev_start(dev_id), + "Failed to start bbdev %u " + "with NULL dev_ops structure ", dev_id); + dev2->dev_ops = dev1.dev_ops; + + TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), + "Failed to start bbdev %u ", dev_id); + + /* Test for rte_bbdev_queue_start */ + dev2->dev_ops = NULL; + TEST_ASSERT_FAIL(rte_bbdev_queue_start(dev_id, queue_id), + "Failed to start queue %u on device %u: " + "NULL dev_ops structure", queue_id, dev_id); + dev2->dev_ops = dev1.dev_ops; + + TEST_ASSERT_SUCCESS(rte_bbdev_queue_start(dev_id, queue_id), + "Failed to start queue %u on device %u ", queue_id, + dev_id); + + /* Tests for rte_bbdev_stats_get */ + dev2->dev_ops = NULL; + TEST_ASSERT_FAIL(rte_bbdev_stats_get(dev_id, &stats), + "Failed test for rte_bbdev_stats_get on device %u ", + dev_id); + dev2->dev_ops = dev1.dev_ops; + + dev_ops_tmp = *dev2->dev_ops; + dev_ops_tmp.stats_reset = NULL; + dev2->dev_ops = &dev_ops_tmp; + TEST_ASSERT_SUCCESS(rte_bbdev_stats_get(dev_id, &stats), + "Failed test for rte_bbdev_stats_get: " + "NULL stats_get "); + dev2->dev_ops = dev1.dev_ops; + + TEST_ASSERT_SUCCESS(rte_bbdev_stats_get(dev_id, &stats), + "Failed test for rte_bbdev_stats_get on device %u ", + dev_id); + + /* + * Tests for: + * rte_bbdev_callback_register, + * rte_bbdev_pmd_callback_process, + * rte_dev_callback_unregister + */ + dev2->dev_ops = NULL; + TEST_ASSERT_SUCCESS(rte_bbdev_callback_register(dev_id, + RTE_BBDEV_EVENT_UNKNOWN, event_callback, NULL), + "Failed to callback rgstr for RTE_BBDEV_EVENT_UNKNOWN"); + rte_bbdev_pmd_callback_process(dev2, RTE_BBDEV_EVENT_UNKNOWN, NULL); + + TEST_ASSERT_SUCCESS(rte_bbdev_callback_unregister(dev_id, + RTE_BBDEV_EVENT_UNKNOWN, event_callback, NULL), + "Failed to unregister RTE_BBDEV_EVENT_ERROR "); + dev2->dev_ops = dev1.dev_ops; + + /* Tests for rte_bbdev_stats_reset */ + dev2->dev_ops = NULL; + TEST_ASSERT_FAIL(rte_bbdev_stats_reset(dev_id), + "Failed to reset statistic for device %u ", dev_id); + dev2->dev_ops = dev1.dev_ops; + + dev_ops_tmp = *dev2->dev_ops; + dev_ops_tmp.stats_reset = NULL; + dev2->dev_ops = &dev_ops_tmp; + TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), + "Failed test for rte_bbdev_stats_reset: " + "NULL stats_reset "); + dev2->dev_ops = dev1.dev_ops; + + TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), + "Failed to reset statistic for device %u ", dev_id); + + /* Tests for rte_bbdev_queue_stop */ + dev2->dev_ops = NULL; + TEST_ASSERT_FAIL(rte_bbdev_queue_stop(dev_id, queue_id), + "Failed to stop queue %u on device %u: " + "NULL dev_ops structure", queue_id, dev_id); + dev2->dev_ops = dev1.dev_ops; + + TEST_ASSERT_SUCCESS(rte_bbdev_queue_stop(dev_id, queue_id), + "Failed to stop queue %u on device %u ", queue_id, + dev_id); + + /* Tests for rte_bbdev_stop */ + dev2->dev_ops = NULL; + TEST_ASSERT_FAIL(rte_bbdev_stop(dev_id), + "Failed to stop bbdev %u with NULL dev_ops structure ", + dev_id); + dev2->dev_ops = dev1.dev_ops; + + TEST_ASSERT_SUCCESS(rte_bbdev_stop(dev_id), + "Failed to stop bbdev %u ", dev_id); + + /* Tests for rte_bbdev_close */ + TEST_ASSERT_FAIL(rte_bbdev_close(RTE_BBDEV_MAX_DEVS), + "Failed to close bbdev with invalid dev_id"); + + dev2->dev_ops = NULL; + TEST_ASSERT_FAIL(rte_bbdev_close(dev_id), + "Failed to close bbdev %u with NULL dev_ops structure ", + dev_id); + dev2->dev_ops = dev1.dev_ops; + + TEST_ASSERT_SUCCESS(rte_bbdev_close(dev_id), + "Failed to close bbdev %u ", dev_id); + + return TEST_SUCCESS; +} + +static int +test_bbdev_get_named_dev(void) +{ + struct rte_bbdev *dev, *dev_tmp; + const char *name = "name"; + + dev = rte_bbdev_allocate(name); + TEST_ASSERT(dev != NULL, "Failed to initialize bbdev driver"); + + dev_tmp = rte_bbdev_get_named_dev(NULL); + TEST_ASSERT(dev_tmp == NULL, "Failed test for rte_bbdev_get_named_dev: " + "function called with NULL parameter"); + + dev_tmp = rte_bbdev_get_named_dev(name); + + TEST_ASSERT(dev == dev_tmp, "Failed test for rte_bbdev_get_named_dev: " + "wrong device was returned "); + + TEST_ASSERT_SUCCESS(rte_bbdev_release(dev), + "Failed to uninitialize bbdev driver %s ", name); + + return TEST_SUCCESS; +} + +static struct unit_test_suite bbdev_null_testsuite = { + .suite_name = "BBDEV NULL Unit Test Suite", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + + TEST_CASE(test_bbdev_configure_invalid_dev_id), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_configure_invalid_num_queues), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_configure_stop_device), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_configure_stop_queue), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_configure_invalid_queue_configure), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_op_pool), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_op_type), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_op_pool_size), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_stats), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_driver_init), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_callback), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_invalid_driver), + + TEST_CASE_ST(ut_setup, ut_teardown, + test_bbdev_get_named_dev), + + TEST_CASE(test_bbdev_count), + + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +REGISTER_TEST_COMMAND(unittest, bbdev_null_testsuite); diff --git a/dpdk/app/test-bbdev/test_bbdev_perf.c b/dpdk/app/test-bbdev/test_bbdev_perf.c new file mode 100644 index 00000000..fbe6cc91 --- /dev/null +++ b/dpdk/app/test-bbdev/test_bbdev_perf.c @@ -0,0 +1,2399 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "main.h" +#include "test_bbdev_vector.h" + +#define GET_SOCKET(socket_id) (((socket_id) == SOCKET_ID_ANY) ? 0 : (socket_id)) + +#define MAX_QUEUES RTE_MAX_LCORE + +#define OPS_CACHE_SIZE 256U +#define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */ + +#define SYNC_WAIT 0 +#define SYNC_START 1 + +#define INVALID_QUEUE_ID -1 + +static struct test_bbdev_vector test_vector; + +/* Switch between PMD and Interrupt for throughput TC */ +static bool intr_enabled; + +/* Represents tested active devices */ +static struct active_device { + const char *driver_name; + uint8_t dev_id; + uint16_t supported_ops; + uint16_t queue_ids[MAX_QUEUES]; + uint16_t nb_queues; + struct rte_mempool *ops_mempool; + struct rte_mempool *in_mbuf_pool; + struct rte_mempool *hard_out_mbuf_pool; + struct rte_mempool *soft_out_mbuf_pool; +} active_devs[RTE_BBDEV_MAX_DEVS]; + +static uint8_t nb_active_devs; + +/* Data buffers used by BBDEV ops */ +struct test_buffers { + struct rte_bbdev_op_data *inputs; + struct rte_bbdev_op_data *hard_outputs; + struct rte_bbdev_op_data *soft_outputs; +}; + +/* Operation parameters specific for given test case */ +struct test_op_params { + struct rte_mempool *mp; + struct rte_bbdev_dec_op *ref_dec_op; + struct rte_bbdev_enc_op *ref_enc_op; + uint16_t burst_sz; + uint16_t num_to_process; + uint16_t num_lcores; + int vector_mask; + rte_atomic16_t sync; + struct test_buffers q_bufs[RTE_MAX_NUMA_NODES][MAX_QUEUES]; +}; + +/* Contains per lcore params */ +struct thread_params { + uint8_t dev_id; + uint16_t queue_id; + uint64_t start_time; + double mops; + double mbps; + rte_atomic16_t nb_dequeued; + rte_atomic16_t processing_status; + struct test_op_params *op_params; +}; + +#ifdef RTE_BBDEV_OFFLOAD_COST +/* Stores time statistics */ +struct test_time_stats { + /* Stores software enqueue total working time */ + uint64_t enq_sw_tot_time; + /* Stores minimum value of software enqueue working time */ + uint64_t enq_sw_min_time; + /* Stores maximum value of software enqueue working time */ + uint64_t enq_sw_max_time; + /* Stores turbo enqueue total working time */ + uint64_t enq_tur_tot_time; + /* Stores minimum value of turbo enqueue working time */ + uint64_t enq_tur_min_time; + /* Stores maximum value of turbo enqueue working time */ + uint64_t enq_tur_max_time; + /* Stores dequeue total working time */ + uint64_t deq_tot_time; + /* Stores minimum value of dequeue working time */ + uint64_t deq_min_time; + /* Stores maximum value of dequeue working time */ + uint64_t deq_max_time; +}; +#endif + +typedef int (test_case_function)(struct active_device *ad, + struct test_op_params *op_params); + +static inline void +set_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) +{ + ad->supported_ops |= (1 << op_type); +} + +static inline bool +is_avail_op(struct active_device *ad, enum rte_bbdev_op_type op_type) +{ + return ad->supported_ops & (1 << op_type); +} + +static inline bool +flags_match(uint32_t flags_req, uint32_t flags_present) +{ + return (flags_req & flags_present) == flags_req; +} + +static void +clear_soft_out_cap(uint32_t *op_flags) +{ + *op_flags &= ~RTE_BBDEV_TURBO_SOFT_OUTPUT; + *op_flags &= ~RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; + *op_flags &= ~RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; +} + +static int +check_dev_cap(const struct rte_bbdev_info *dev_info) +{ + unsigned int i; + unsigned int nb_inputs, nb_soft_outputs, nb_hard_outputs; + const struct rte_bbdev_op_cap *op_cap = dev_info->drv.capabilities; + + nb_inputs = test_vector.entries[DATA_INPUT].nb_segments; + nb_soft_outputs = test_vector.entries[DATA_SOFT_OUTPUT].nb_segments; + nb_hard_outputs = test_vector.entries[DATA_HARD_OUTPUT].nb_segments; + + for (i = 0; op_cap->type != RTE_BBDEV_OP_NONE; ++i, ++op_cap) { + if (op_cap->type != test_vector.op_type) + continue; + + if (op_cap->type == RTE_BBDEV_OP_TURBO_DEC) { + const struct rte_bbdev_op_cap_turbo_dec *cap = + &op_cap->cap.turbo_dec; + /* Ignore lack of soft output capability, just skip + * checking if soft output is valid. + */ + if ((test_vector.turbo_dec.op_flags & + RTE_BBDEV_TURBO_SOFT_OUTPUT) && + !(cap->capability_flags & + RTE_BBDEV_TURBO_SOFT_OUTPUT)) { + printf( + "WARNING: Device \"%s\" does not support soft output - soft output flags will be ignored.\n", + dev_info->dev_name); + clear_soft_out_cap( + &test_vector.turbo_dec.op_flags); + } + + if (!flags_match(test_vector.turbo_dec.op_flags, + cap->capability_flags)) + return TEST_FAILED; + if (nb_inputs > cap->num_buffers_src) { + printf("Too many inputs defined: %u, max: %u\n", + nb_inputs, cap->num_buffers_src); + return TEST_FAILED; + } + if (nb_soft_outputs > cap->num_buffers_soft_out && + (test_vector.turbo_dec.op_flags & + RTE_BBDEV_TURBO_SOFT_OUTPUT)) { + printf( + "Too many soft outputs defined: %u, max: %u\n", + nb_soft_outputs, + cap->num_buffers_soft_out); + return TEST_FAILED; + } + if (nb_hard_outputs > cap->num_buffers_hard_out) { + printf( + "Too many hard outputs defined: %u, max: %u\n", + nb_hard_outputs, + cap->num_buffers_hard_out); + return TEST_FAILED; + } + if (intr_enabled && !(cap->capability_flags & + RTE_BBDEV_TURBO_DEC_INTERRUPTS)) { + printf( + "Dequeue interrupts are not supported!\n"); + return TEST_FAILED; + } + + return TEST_SUCCESS; + } else if (op_cap->type == RTE_BBDEV_OP_TURBO_ENC) { + const struct rte_bbdev_op_cap_turbo_enc *cap = + &op_cap->cap.turbo_enc; + + if (!flags_match(test_vector.turbo_enc.op_flags, + cap->capability_flags)) + return TEST_FAILED; + if (nb_inputs > cap->num_buffers_src) { + printf("Too many inputs defined: %u, max: %u\n", + nb_inputs, cap->num_buffers_src); + return TEST_FAILED; + } + if (nb_hard_outputs > cap->num_buffers_dst) { + printf( + "Too many hard outputs defined: %u, max: %u\n", + nb_hard_outputs, cap->num_buffers_src); + return TEST_FAILED; + } + if (intr_enabled && !(cap->capability_flags & + RTE_BBDEV_TURBO_ENC_INTERRUPTS)) { + printf( + "Dequeue interrupts are not supported!\n"); + return TEST_FAILED; + } + + return TEST_SUCCESS; + } + } + + if ((i == 0) && (test_vector.op_type == RTE_BBDEV_OP_NONE)) + return TEST_SUCCESS; /* Special case for NULL device */ + + return TEST_FAILED; +} + +/* calculates optimal mempool size not smaller than the val */ +static unsigned int +optimal_mempool_size(unsigned int val) +{ + return rte_align32pow2(val + 1) - 1; +} + +/* allocates mbuf mempool for inputs and outputs */ +static struct rte_mempool * +create_mbuf_pool(struct op_data_entries *entries, uint8_t dev_id, + int socket_id, unsigned int mbuf_pool_size, + const char *op_type_str) +{ + unsigned int i; + uint32_t max_seg_sz = 0; + char pool_name[RTE_MEMPOOL_NAMESIZE]; + + /* find max input segment size */ + for (i = 0; i < entries->nb_segments; ++i) + if (entries->segments[i].length > max_seg_sz) + max_seg_sz = entries->segments[i].length; + + snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, + dev_id); + return rte_pktmbuf_pool_create(pool_name, mbuf_pool_size, 0, 0, + RTE_MAX(max_seg_sz + RTE_PKTMBUF_HEADROOM, + (unsigned int)RTE_MBUF_DEFAULT_BUF_SIZE), socket_id); +} + +static int +create_mempools(struct active_device *ad, int socket_id, + enum rte_bbdev_op_type org_op_type, uint16_t num_ops) +{ + struct rte_mempool *mp; + unsigned int ops_pool_size, mbuf_pool_size = 0; + char pool_name[RTE_MEMPOOL_NAMESIZE]; + const char *op_type_str; + enum rte_bbdev_op_type op_type = org_op_type; + + struct op_data_entries *in = &test_vector.entries[DATA_INPUT]; + struct op_data_entries *hard_out = + &test_vector.entries[DATA_HARD_OUTPUT]; + struct op_data_entries *soft_out = + &test_vector.entries[DATA_SOFT_OUTPUT]; + + /* allocate ops mempool */ + ops_pool_size = optimal_mempool_size(RTE_MAX( + /* Ops used plus 1 reference op */ + RTE_MAX((unsigned int)(ad->nb_queues * num_ops + 1), + /* Minimal cache size plus 1 reference op */ + (unsigned int)(1.5 * rte_lcore_count() * + OPS_CACHE_SIZE + 1)), + OPS_POOL_SIZE_MIN)); + + if (org_op_type == RTE_BBDEV_OP_NONE) + op_type = RTE_BBDEV_OP_TURBO_ENC; + + op_type_str = rte_bbdev_op_type_str(op_type); + TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); + + snprintf(pool_name, sizeof(pool_name), "%s_pool_%u", op_type_str, + ad->dev_id); + mp = rte_bbdev_op_pool_create(pool_name, op_type, + ops_pool_size, OPS_CACHE_SIZE, socket_id); + TEST_ASSERT_NOT_NULL(mp, + "ERROR Failed to create %u items ops pool for dev %u on socket %u.", + ops_pool_size, + ad->dev_id, + socket_id); + ad->ops_mempool = mp; + + /* Do not create inputs and outputs mbufs for BaseBand Null Device */ + if (org_op_type == RTE_BBDEV_OP_NONE) + return TEST_SUCCESS; + + /* Inputs */ + mbuf_pool_size = optimal_mempool_size(ops_pool_size * in->nb_segments); + mp = create_mbuf_pool(in, ad->dev_id, socket_id, mbuf_pool_size, "in"); + TEST_ASSERT_NOT_NULL(mp, + "ERROR Failed to create %u items input pktmbuf pool for dev %u on socket %u.", + mbuf_pool_size, + ad->dev_id, + socket_id); + ad->in_mbuf_pool = mp; + + /* Hard outputs */ + mbuf_pool_size = optimal_mempool_size(ops_pool_size * + hard_out->nb_segments); + mp = create_mbuf_pool(hard_out, ad->dev_id, socket_id, mbuf_pool_size, + "hard_out"); + TEST_ASSERT_NOT_NULL(mp, + "ERROR Failed to create %u items hard output pktmbuf pool for dev %u on socket %u.", + mbuf_pool_size, + ad->dev_id, + socket_id); + ad->hard_out_mbuf_pool = mp; + + if (soft_out->nb_segments == 0) + return TEST_SUCCESS; + + /* Soft outputs */ + mbuf_pool_size = optimal_mempool_size(ops_pool_size * + soft_out->nb_segments); + mp = create_mbuf_pool(soft_out, ad->dev_id, socket_id, mbuf_pool_size, + "soft_out"); + TEST_ASSERT_NOT_NULL(mp, + "ERROR Failed to create %uB soft output pktmbuf pool for dev %u on socket %u.", + mbuf_pool_size, + ad->dev_id, + socket_id); + ad->soft_out_mbuf_pool = mp; + + return 0; +} + +static int +add_bbdev_dev(uint8_t dev_id, struct rte_bbdev_info *info, + struct test_bbdev_vector *vector) +{ + int ret; + unsigned int queue_id; + struct rte_bbdev_queue_conf qconf; + struct active_device *ad = &active_devs[nb_active_devs]; + unsigned int nb_queues; + enum rte_bbdev_op_type op_type = vector->op_type; + + nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues); + /* setup device */ + ret = rte_bbdev_setup_queues(dev_id, nb_queues, info->socket_id); + if (ret < 0) { + printf("rte_bbdev_setup_queues(%u, %u, %d) ret %i\n", + dev_id, nb_queues, info->socket_id, ret); + return TEST_FAILED; + } + + /* configure interrupts if needed */ + if (intr_enabled) { + ret = rte_bbdev_intr_enable(dev_id); + if (ret < 0) { + printf("rte_bbdev_intr_enable(%u) ret %i\n", dev_id, + ret); + return TEST_FAILED; + } + } + + /* setup device queues */ + qconf.socket = info->socket_id; + qconf.queue_size = info->drv.default_queue_conf.queue_size; + qconf.priority = 0; + qconf.deferred_start = 0; + qconf.op_type = op_type; + + for (queue_id = 0; queue_id < nb_queues; ++queue_id) { + ret = rte_bbdev_queue_configure(dev_id, queue_id, &qconf); + if (ret != 0) { + printf( + "Allocated all queues (id=%u) at prio%u on dev%u\n", + queue_id, qconf.priority, dev_id); + qconf.priority++; + ret = rte_bbdev_queue_configure(ad->dev_id, queue_id, + &qconf); + } + if (ret != 0) { + printf("All queues on dev %u allocated: %u\n", + dev_id, queue_id); + break; + } + ad->queue_ids[queue_id] = queue_id; + } + TEST_ASSERT(queue_id != 0, + "ERROR Failed to configure any queues on dev %u", + dev_id); + ad->nb_queues = queue_id; + + set_avail_op(ad, op_type); + + return TEST_SUCCESS; +} + +static int +add_active_device(uint8_t dev_id, struct rte_bbdev_info *info, + struct test_bbdev_vector *vector) +{ + int ret; + + active_devs[nb_active_devs].driver_name = info->drv.driver_name; + active_devs[nb_active_devs].dev_id = dev_id; + + ret = add_bbdev_dev(dev_id, info, vector); + if (ret == TEST_SUCCESS) + ++nb_active_devs; + return ret; +} + +static uint8_t +populate_active_devices(void) +{ + int ret; + uint8_t dev_id; + uint8_t nb_devs_added = 0; + struct rte_bbdev_info info; + + RTE_BBDEV_FOREACH(dev_id) { + rte_bbdev_info_get(dev_id, &info); + + if (check_dev_cap(&info)) { + printf( + "Device %d (%s) does not support specified capabilities\n", + dev_id, info.dev_name); + continue; + } + + ret = add_active_device(dev_id, &info, &test_vector); + if (ret != 0) { + printf("Adding active bbdev %s skipped\n", + info.dev_name); + continue; + } + nb_devs_added++; + } + + return nb_devs_added; +} + +static int +read_test_vector(void) +{ + int ret; + + memset(&test_vector, 0, sizeof(test_vector)); + printf("Test vector file = %s\n", get_vector_filename()); + ret = test_bbdev_vector_read(get_vector_filename(), &test_vector); + TEST_ASSERT_SUCCESS(ret, "Failed to parse file %s\n", + get_vector_filename()); + + return TEST_SUCCESS; +} + +static int +testsuite_setup(void) +{ + TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); + + if (populate_active_devices() == 0) { + printf("No suitable devices found!\n"); + return TEST_SKIPPED; + } + + return TEST_SUCCESS; +} + +static int +interrupt_testsuite_setup(void) +{ + TEST_ASSERT_SUCCESS(read_test_vector(), "Test suite setup failed\n"); + + /* Enable interrupts */ + intr_enabled = true; + + /* Special case for NULL device (RTE_BBDEV_OP_NONE) */ + if (populate_active_devices() == 0 || + test_vector.op_type == RTE_BBDEV_OP_NONE) { + intr_enabled = false; + printf("No suitable devices found!\n"); + return TEST_SKIPPED; + } + + return TEST_SUCCESS; +} + +static void +testsuite_teardown(void) +{ + uint8_t dev_id; + + /* Unconfigure devices */ + RTE_BBDEV_FOREACH(dev_id) + rte_bbdev_close(dev_id); + + /* Clear active devices structs. */ + memset(active_devs, 0, sizeof(active_devs)); + nb_active_devs = 0; +} + +static int +ut_setup(void) +{ + uint8_t i, dev_id; + + for (i = 0; i < nb_active_devs; i++) { + dev_id = active_devs[i].dev_id; + /* reset bbdev stats */ + TEST_ASSERT_SUCCESS(rte_bbdev_stats_reset(dev_id), + "Failed to reset stats of bbdev %u", dev_id); + /* start the device */ + TEST_ASSERT_SUCCESS(rte_bbdev_start(dev_id), + "Failed to start bbdev %u", dev_id); + } + + return TEST_SUCCESS; +} + +static void +ut_teardown(void) +{ + uint8_t i, dev_id; + struct rte_bbdev_stats stats; + + for (i = 0; i < nb_active_devs; i++) { + dev_id = active_devs[i].dev_id; + /* read stats and print */ + rte_bbdev_stats_get(dev_id, &stats); + /* Stop the device */ + rte_bbdev_stop(dev_id); + } +} + +static int +init_op_data_objs(struct rte_bbdev_op_data *bufs, + struct op_data_entries *ref_entries, + struct rte_mempool *mbuf_pool, const uint16_t n, + enum op_data_type op_type, uint16_t min_alignment) +{ + int ret; + unsigned int i, j; + + for (i = 0; i < n; ++i) { + char *data; + struct op_data_buf *seg = &ref_entries->segments[0]; + struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool); + TEST_ASSERT_NOT_NULL(m_head, + "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", + op_type, n * ref_entries->nb_segments, + mbuf_pool->size); + + bufs[i].data = m_head; + bufs[i].offset = 0; + bufs[i].length = 0; + + if (op_type == DATA_INPUT) { + data = rte_pktmbuf_append(m_head, seg->length); + TEST_ASSERT_NOT_NULL(data, + "Couldn't append %u bytes to mbuf from %d data type mbuf pool", + seg->length, op_type); + + TEST_ASSERT(data == RTE_PTR_ALIGN(data, min_alignment), + "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", + data, min_alignment); + rte_memcpy(data, seg->addr, seg->length); + bufs[i].length += seg->length; + + + for (j = 1; j < ref_entries->nb_segments; ++j) { + struct rte_mbuf *m_tail = + rte_pktmbuf_alloc(mbuf_pool); + TEST_ASSERT_NOT_NULL(m_tail, + "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)", + op_type, + n * ref_entries->nb_segments, + mbuf_pool->size); + seg += 1; + + data = rte_pktmbuf_append(m_tail, seg->length); + TEST_ASSERT_NOT_NULL(data, + "Couldn't append %u bytes to mbuf from %d data type mbuf pool", + seg->length, op_type); + + TEST_ASSERT(data == RTE_PTR_ALIGN(data, + min_alignment), + "Data addr in mbuf (%p) is not aligned to device min alignment (%u)", + data, min_alignment); + rte_memcpy(data, seg->addr, seg->length); + bufs[i].length += seg->length; + + ret = rte_pktmbuf_chain(m_head, m_tail); + TEST_ASSERT_SUCCESS(ret, + "Couldn't chain mbufs from %d data type mbuf pool", + op_type); + } + } + } + + return 0; +} + +static int +allocate_buffers_on_socket(struct rte_bbdev_op_data **buffers, const int len, + const int socket) +{ + int i; + + *buffers = rte_zmalloc_socket(NULL, len, 0, socket); + if (*buffers == NULL) { + printf("WARNING: Failed to allocate op_data on socket %d\n", + socket); + /* try to allocate memory on other detected sockets */ + for (i = 0; i < socket; i++) { + *buffers = rte_zmalloc_socket(NULL, len, 0, i); + if (*buffers != NULL) + break; + } + } + + return (*buffers == NULL) ? TEST_FAILED : TEST_SUCCESS; +} + +static void +limit_input_llr_val_range(struct rte_bbdev_op_data *input_ops, + uint16_t n, int8_t max_llr_modulus) +{ + uint16_t i, byte_idx; + + for (i = 0; i < n; ++i) { + struct rte_mbuf *m = input_ops[i].data; + while (m != NULL) { + int8_t *llr = rte_pktmbuf_mtod_offset(m, int8_t *, + input_ops[i].offset); + for (byte_idx = 0; byte_idx < input_ops[i].length; + ++byte_idx) + llr[byte_idx] = round((double)max_llr_modulus * + llr[byte_idx] / INT8_MAX); + + m = m->next; + } + } +} + +static int +fill_queue_buffers(struct test_op_params *op_params, + struct rte_mempool *in_mp, struct rte_mempool *hard_out_mp, + struct rte_mempool *soft_out_mp, uint16_t queue_id, + const struct rte_bbdev_op_cap *capabilities, + uint16_t min_alignment, const int socket_id) +{ + int ret; + enum op_data_type type; + const uint16_t n = op_params->num_to_process; + + struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = { + in_mp, + soft_out_mp, + hard_out_mp, + }; + + struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = { + &op_params->q_bufs[socket_id][queue_id].inputs, + &op_params->q_bufs[socket_id][queue_id].soft_outputs, + &op_params->q_bufs[socket_id][queue_id].hard_outputs, + }; + + for (type = DATA_INPUT; type < DATA_NUM_TYPES; ++type) { + struct op_data_entries *ref_entries = + &test_vector.entries[type]; + if (ref_entries->nb_segments == 0) + continue; + + ret = allocate_buffers_on_socket(queue_ops[type], + n * sizeof(struct rte_bbdev_op_data), + socket_id); + TEST_ASSERT_SUCCESS(ret, + "Couldn't allocate memory for rte_bbdev_op_data structs"); + + ret = init_op_data_objs(*queue_ops[type], ref_entries, + mbuf_pools[type], n, type, min_alignment); + TEST_ASSERT_SUCCESS(ret, + "Couldn't init rte_bbdev_op_data structs"); + } + + if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) + limit_input_llr_val_range(*queue_ops[DATA_INPUT], n, + capabilities->cap.turbo_dec.max_llr_modulus); + + return 0; +} + +static void +free_buffers(struct active_device *ad, struct test_op_params *op_params) +{ + unsigned int i, j; + + rte_mempool_free(ad->ops_mempool); + rte_mempool_free(ad->in_mbuf_pool); + rte_mempool_free(ad->hard_out_mbuf_pool); + rte_mempool_free(ad->soft_out_mbuf_pool); + + for (i = 0; i < rte_lcore_count(); ++i) { + for (j = 0; j < RTE_MAX_NUMA_NODES; ++j) { + rte_free(op_params->q_bufs[j][i].inputs); + rte_free(op_params->q_bufs[j][i].hard_outputs); + rte_free(op_params->q_bufs[j][i].soft_outputs); + } + } +} + +static void +copy_reference_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n, + unsigned int start_idx, + struct rte_bbdev_op_data *inputs, + struct rte_bbdev_op_data *hard_outputs, + struct rte_bbdev_op_data *soft_outputs, + struct rte_bbdev_dec_op *ref_op) +{ + unsigned int i; + struct rte_bbdev_op_turbo_dec *turbo_dec = &ref_op->turbo_dec; + + for (i = 0; i < n; ++i) { + if (turbo_dec->code_block_mode == 0) { + ops[i]->turbo_dec.tb_params.ea = + turbo_dec->tb_params.ea; + ops[i]->turbo_dec.tb_params.eb = + turbo_dec->tb_params.eb; + ops[i]->turbo_dec.tb_params.k_pos = + turbo_dec->tb_params.k_pos; + ops[i]->turbo_dec.tb_params.k_neg = + turbo_dec->tb_params.k_neg; + ops[i]->turbo_dec.tb_params.c = + turbo_dec->tb_params.c; + ops[i]->turbo_dec.tb_params.c_neg = + turbo_dec->tb_params.c_neg; + ops[i]->turbo_dec.tb_params.cab = + turbo_dec->tb_params.cab; + } else { + ops[i]->turbo_dec.cb_params.e = turbo_dec->cb_params.e; + ops[i]->turbo_dec.cb_params.k = turbo_dec->cb_params.k; + } + + ops[i]->turbo_dec.ext_scale = turbo_dec->ext_scale; + ops[i]->turbo_dec.iter_max = turbo_dec->iter_max; + ops[i]->turbo_dec.iter_min = turbo_dec->iter_min; + ops[i]->turbo_dec.op_flags = turbo_dec->op_flags; + ops[i]->turbo_dec.rv_index = turbo_dec->rv_index; + ops[i]->turbo_dec.num_maps = turbo_dec->num_maps; + ops[i]->turbo_dec.code_block_mode = turbo_dec->code_block_mode; + + ops[i]->turbo_dec.hard_output = hard_outputs[start_idx + i]; + ops[i]->turbo_dec.input = inputs[start_idx + i]; + if (soft_outputs != NULL) + ops[i]->turbo_dec.soft_output = + soft_outputs[start_idx + i]; + } +} + +static void +copy_reference_enc_op(struct rte_bbdev_enc_op **ops, unsigned int n, + unsigned int start_idx, + struct rte_bbdev_op_data *inputs, + struct rte_bbdev_op_data *outputs, + struct rte_bbdev_enc_op *ref_op) +{ + unsigned int i; + struct rte_bbdev_op_turbo_enc *turbo_enc = &ref_op->turbo_enc; + for (i = 0; i < n; ++i) { + if (turbo_enc->code_block_mode == 0) { + ops[i]->turbo_enc.tb_params.ea = + turbo_enc->tb_params.ea; + ops[i]->turbo_enc.tb_params.eb = + turbo_enc->tb_params.eb; + ops[i]->turbo_enc.tb_params.k_pos = + turbo_enc->tb_params.k_pos; + ops[i]->turbo_enc.tb_params.k_neg = + turbo_enc->tb_params.k_neg; + ops[i]->turbo_enc.tb_params.c = + turbo_enc->tb_params.c; + ops[i]->turbo_enc.tb_params.c_neg = + turbo_enc->tb_params.c_neg; + ops[i]->turbo_enc.tb_params.cab = + turbo_enc->tb_params.cab; + ops[i]->turbo_enc.tb_params.ncb_pos = + turbo_enc->tb_params.ncb_pos; + ops[i]->turbo_enc.tb_params.ncb_neg = + turbo_enc->tb_params.ncb_neg; + ops[i]->turbo_enc.tb_params.r = turbo_enc->tb_params.r; + } else { + ops[i]->turbo_enc.cb_params.e = turbo_enc->cb_params.e; + ops[i]->turbo_enc.cb_params.k = turbo_enc->cb_params.k; + ops[i]->turbo_enc.cb_params.ncb = + turbo_enc->cb_params.ncb; + } + ops[i]->turbo_enc.rv_index = turbo_enc->rv_index; + ops[i]->turbo_enc.op_flags = turbo_enc->op_flags; + ops[i]->turbo_enc.code_block_mode = turbo_enc->code_block_mode; + + ops[i]->turbo_enc.output = outputs[start_idx + i]; + ops[i]->turbo_enc.input = inputs[start_idx + i]; + } +} + +static int +check_dec_status_and_ordering(struct rte_bbdev_dec_op *op, + unsigned int order_idx, const int expected_status) +{ + TEST_ASSERT(op->status == expected_status, + "op_status (%d) != expected_status (%d)", + op->status, expected_status); + + TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, + "Ordering error, expected %p, got %p", + (void *)(uintptr_t)order_idx, op->opaque_data); + + return TEST_SUCCESS; +} + +static int +check_enc_status_and_ordering(struct rte_bbdev_enc_op *op, + unsigned int order_idx, const int expected_status) +{ + TEST_ASSERT(op->status == expected_status, + "op_status (%d) != expected_status (%d)", + op->status, expected_status); + + TEST_ASSERT((void *)(uintptr_t)order_idx == op->opaque_data, + "Ordering error, expected %p, got %p", + (void *)(uintptr_t)order_idx, op->opaque_data); + + return TEST_SUCCESS; +} + +static inline int +validate_op_chain(struct rte_bbdev_op_data *op, + struct op_data_entries *orig_op) +{ + uint8_t i; + struct rte_mbuf *m = op->data; + uint8_t nb_dst_segments = orig_op->nb_segments; + + TEST_ASSERT(nb_dst_segments == m->nb_segs, + "Number of segments differ in original (%u) and filled (%u) op", + nb_dst_segments, m->nb_segs); + + for (i = 0; i < nb_dst_segments; ++i) { + /* Apply offset to the first mbuf segment */ + uint16_t offset = (i == 0) ? op->offset : 0; + uint16_t data_len = m->data_len - offset; + + TEST_ASSERT(orig_op->segments[i].length == data_len, + "Length of segment differ in original (%u) and filled (%u) op", + orig_op->segments[i].length, data_len); + TEST_ASSERT_BUFFERS_ARE_EQUAL(orig_op->segments[i].addr, + rte_pktmbuf_mtod_offset(m, uint32_t *, offset), + data_len, + "Output buffers (CB=%u) are not equal", i); + m = m->next; + } + + return TEST_SUCCESS; +} + +static int +validate_dec_buffers(struct rte_bbdev_dec_op *ref_op, struct test_buffers *bufs, + const uint16_t num_to_process) +{ + int i; + + struct op_data_entries *hard_data_orig = + &test_vector.entries[DATA_HARD_OUTPUT]; + struct op_data_entries *soft_data_orig = + &test_vector.entries[DATA_SOFT_OUTPUT]; + + for (i = 0; i < num_to_process; i++) { + TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i], + hard_data_orig), + "Hard output buffers are not equal"); + if (ref_op->turbo_dec.op_flags & + RTE_BBDEV_TURBO_SOFT_OUTPUT) + TEST_ASSERT_SUCCESS(validate_op_chain( + &bufs->soft_outputs[i], + soft_data_orig), + "Soft output buffers are not equal"); + } + + return TEST_SUCCESS; +} + +static int +validate_enc_buffers(struct test_buffers *bufs, const uint16_t num_to_process) +{ + int i; + + struct op_data_entries *hard_data_orig = + &test_vector.entries[DATA_HARD_OUTPUT]; + + for (i = 0; i < num_to_process; i++) + TEST_ASSERT_SUCCESS(validate_op_chain(&bufs->hard_outputs[i], + hard_data_orig), ""); + + return TEST_SUCCESS; +} + +static int +validate_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n, + struct rte_bbdev_dec_op *ref_op, const int vector_mask) +{ + unsigned int i; + int ret; + struct op_data_entries *hard_data_orig = + &test_vector.entries[DATA_HARD_OUTPUT]; + struct op_data_entries *soft_data_orig = + &test_vector.entries[DATA_SOFT_OUTPUT]; + struct rte_bbdev_op_turbo_dec *ops_td; + struct rte_bbdev_op_data *hard_output; + struct rte_bbdev_op_data *soft_output; + struct rte_bbdev_op_turbo_dec *ref_td = &ref_op->turbo_dec; + + for (i = 0; i < n; ++i) { + ops_td = &ops[i]->turbo_dec; + hard_output = &ops_td->hard_output; + soft_output = &ops_td->soft_output; + + if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT) + TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count, + "Returned iter_count (%d) > expected iter_count (%d)", + ops_td->iter_count, ref_td->iter_count); + ret = check_dec_status_and_ordering(ops[i], i, ref_op->status); + TEST_ASSERT_SUCCESS(ret, + "Checking status and ordering for decoder failed"); + + TEST_ASSERT_SUCCESS(validate_op_chain(hard_output, + hard_data_orig), + "Hard output buffers (CB=%u) are not equal", + i); + + if (ref_op->turbo_dec.op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) + TEST_ASSERT_SUCCESS(validate_op_chain(soft_output, + soft_data_orig), + "Soft output buffers (CB=%u) are not equal", + i); + } + + return TEST_SUCCESS; +} + +static int +validate_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n, + struct rte_bbdev_enc_op *ref_op) +{ + unsigned int i; + int ret; + struct op_data_entries *hard_data_orig = + &test_vector.entries[DATA_HARD_OUTPUT]; + + for (i = 0; i < n; ++i) { + ret = check_enc_status_and_ordering(ops[i], i, ref_op->status); + TEST_ASSERT_SUCCESS(ret, + "Checking status and ordering for encoder failed"); + TEST_ASSERT_SUCCESS(validate_op_chain( + &ops[i]->turbo_enc.output, + hard_data_orig), + "Output buffers (CB=%u) are not equal", + i); + } + + return TEST_SUCCESS; +} + +static void +create_reference_dec_op(struct rte_bbdev_dec_op *op) +{ + unsigned int i; + struct op_data_entries *entry; + + op->turbo_dec = test_vector.turbo_dec; + entry = &test_vector.entries[DATA_INPUT]; + for (i = 0; i < entry->nb_segments; ++i) + op->turbo_dec.input.length += + entry->segments[i].length; +} + +static void +create_reference_enc_op(struct rte_bbdev_enc_op *op) +{ + unsigned int i; + struct op_data_entries *entry; + + op->turbo_enc = test_vector.turbo_enc; + entry = &test_vector.entries[DATA_INPUT]; + for (i = 0; i < entry->nb_segments; ++i) + op->turbo_enc.input.length += + entry->segments[i].length; +} + +static int +init_test_op_params(struct test_op_params *op_params, + enum rte_bbdev_op_type op_type, const int expected_status, + const int vector_mask, struct rte_mempool *ops_mp, + uint16_t burst_sz, uint16_t num_to_process, uint16_t num_lcores) +{ + int ret = 0; + if (op_type == RTE_BBDEV_OP_TURBO_DEC) + ret = rte_bbdev_dec_op_alloc_bulk(ops_mp, + &op_params->ref_dec_op, 1); + else + ret = rte_bbdev_enc_op_alloc_bulk(ops_mp, + &op_params->ref_enc_op, 1); + + TEST_ASSERT_SUCCESS(ret, "rte_bbdev_op_alloc_bulk() failed"); + + op_params->mp = ops_mp; + op_params->burst_sz = burst_sz; + op_params->num_to_process = num_to_process; + op_params->num_lcores = num_lcores; + op_params->vector_mask = vector_mask; + if (op_type == RTE_BBDEV_OP_TURBO_DEC) + op_params->ref_dec_op->status = expected_status; + else if (op_type == RTE_BBDEV_OP_TURBO_ENC) + op_params->ref_enc_op->status = expected_status; + + return 0; +} + +static int +run_test_case_on_device(test_case_function *test_case_func, uint8_t dev_id, + struct test_op_params *op_params) +{ + int t_ret, f_ret, socket_id = SOCKET_ID_ANY; + unsigned int i; + struct active_device *ad; + unsigned int burst_sz = get_burst_sz(); + enum rte_bbdev_op_type op_type = test_vector.op_type; + const struct rte_bbdev_op_cap *capabilities = NULL; + + ad = &active_devs[dev_id]; + + /* Check if device supports op_type */ + if (!is_avail_op(ad, test_vector.op_type)) + return TEST_SUCCESS; + + struct rte_bbdev_info info; + rte_bbdev_info_get(ad->dev_id, &info); + socket_id = GET_SOCKET(info.socket_id); + + f_ret = create_mempools(ad, socket_id, op_type, + get_num_ops()); + if (f_ret != TEST_SUCCESS) { + printf("Couldn't create mempools"); + goto fail; + } + if (op_type == RTE_BBDEV_OP_NONE) + op_type = RTE_BBDEV_OP_TURBO_ENC; + + f_ret = init_test_op_params(op_params, test_vector.op_type, + test_vector.expected_status, + test_vector.mask, + ad->ops_mempool, + burst_sz, + get_num_ops(), + get_num_lcores()); + if (f_ret != TEST_SUCCESS) { + printf("Couldn't init test op params"); + goto fail; + } + + if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) { + /* Find Decoder capabilities */ + const struct rte_bbdev_op_cap *cap = info.drv.capabilities; + while (cap->type != RTE_BBDEV_OP_NONE) { + if (cap->type == RTE_BBDEV_OP_TURBO_DEC) { + capabilities = cap; + break; + } + } + TEST_ASSERT_NOT_NULL(capabilities, + "Couldn't find Decoder capabilities"); + + create_reference_dec_op(op_params->ref_dec_op); + } else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) + create_reference_enc_op(op_params->ref_enc_op); + + for (i = 0; i < ad->nb_queues; ++i) { + f_ret = fill_queue_buffers(op_params, + ad->in_mbuf_pool, + ad->hard_out_mbuf_pool, + ad->soft_out_mbuf_pool, + ad->queue_ids[i], + capabilities, + info.drv.min_alignment, + socket_id); + if (f_ret != TEST_SUCCESS) { + printf("Couldn't init queue buffers"); + goto fail; + } + } + + /* Run test case function */ + t_ret = test_case_func(ad, op_params); + + /* Free active device resources and return */ + free_buffers(ad, op_params); + return t_ret; + +fail: + free_buffers(ad, op_params); + return TEST_FAILED; +} + +/* Run given test function per active device per supported op type + * per burst size. + */ +static int +run_test_case(test_case_function *test_case_func) +{ + int ret = 0; + uint8_t dev; + + /* Alloc op_params */ + struct test_op_params *op_params = rte_zmalloc(NULL, + sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE); + TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params", + RTE_ALIGN(sizeof(struct test_op_params), + RTE_CACHE_LINE_SIZE)); + + /* For each device run test case function */ + for (dev = 0; dev < nb_active_devs; ++dev) + ret |= run_test_case_on_device(test_case_func, dev, op_params); + + rte_free(op_params); + + return ret; +} + +static void +dequeue_event_callback(uint16_t dev_id, + enum rte_bbdev_event_type event, void *cb_arg, + void *ret_param) +{ + int ret; + uint16_t i; + uint64_t total_time; + uint16_t deq, burst_sz, num_to_process; + uint16_t queue_id = INVALID_QUEUE_ID; + struct rte_bbdev_dec_op *dec_ops[MAX_BURST]; + struct rte_bbdev_enc_op *enc_ops[MAX_BURST]; + struct test_buffers *bufs; + struct rte_bbdev_info info; + + /* Input length in bytes, million operations per second, + * million bits per second. + */ + double in_len; + + struct thread_params *tp = cb_arg; + RTE_SET_USED(ret_param); + queue_id = tp->queue_id; + + /* Find matching thread params using queue_id */ + for (i = 0; i < MAX_QUEUES; ++i, ++tp) + if (tp->queue_id == queue_id) + break; + + if (i == MAX_QUEUES) { + printf("%s: Queue_id from interrupt details was not found!\n", + __func__); + return; + } + + if (unlikely(event != RTE_BBDEV_EVENT_DEQUEUE)) { + rte_atomic16_set(&tp->processing_status, TEST_FAILED); + printf( + "Dequeue interrupt handler called for incorrect event!\n"); + return; + } + + burst_sz = tp->op_params->burst_sz; + num_to_process = tp->op_params->num_to_process; + + if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) + deq = rte_bbdev_dequeue_dec_ops(dev_id, queue_id, dec_ops, + burst_sz); + else + deq = rte_bbdev_dequeue_enc_ops(dev_id, queue_id, enc_ops, + burst_sz); + + if (deq < burst_sz) { + printf( + "After receiving the interrupt all operations should be dequeued. Expected: %u, got: %u\n", + burst_sz, deq); + rte_atomic16_set(&tp->processing_status, TEST_FAILED); + return; + } + + if (rte_atomic16_read(&tp->nb_dequeued) + deq < num_to_process) { + rte_atomic16_add(&tp->nb_dequeued, deq); + return; + } + + total_time = rte_rdtsc_precise() - tp->start_time; + + rte_bbdev_info_get(dev_id, &info); + + bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; + + ret = TEST_SUCCESS; + if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) + ret = validate_dec_buffers(tp->op_params->ref_dec_op, bufs, + num_to_process); + else if (test_vector.op_type == RTE_BBDEV_OP_TURBO_ENC) + ret = validate_enc_buffers(bufs, num_to_process); + + if (ret) { + printf("Buffers validation failed\n"); + rte_atomic16_set(&tp->processing_status, TEST_FAILED); + } + + switch (test_vector.op_type) { + case RTE_BBDEV_OP_TURBO_DEC: + in_len = tp->op_params->ref_dec_op->turbo_dec.input.length; + break; + case RTE_BBDEV_OP_TURBO_ENC: + in_len = tp->op_params->ref_enc_op->turbo_enc.input.length; + break; + case RTE_BBDEV_OP_NONE: + in_len = 0.0; + break; + default: + printf("Unknown op type: %d\n", test_vector.op_type); + rte_atomic16_set(&tp->processing_status, TEST_FAILED); + return; + } + + tp->mops = ((double)num_to_process / 1000000.0) / + ((double)total_time / (double)rte_get_tsc_hz()); + tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / + ((double)total_time / (double)rte_get_tsc_hz()); + + rte_atomic16_add(&tp->nb_dequeued, deq); +} + +static int +throughput_intr_lcore_dec(void *arg) +{ + struct thread_params *tp = arg; + unsigned int enqueued; + struct rte_bbdev_dec_op *ops[MAX_BURST]; + const uint16_t queue_id = tp->queue_id; + const uint16_t burst_sz = tp->op_params->burst_sz; + const uint16_t num_to_process = tp->op_params->num_to_process; + struct test_buffers *bufs = NULL; + unsigned int allocs_failed = 0; + struct rte_bbdev_info info; + int ret; + + TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), + "BURST_SIZE should be <= %u", MAX_BURST); + + TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), + "Failed to enable interrupts for dev: %u, queue_id: %u", + tp->dev_id, queue_id); + + rte_bbdev_info_get(tp->dev_id, &info); + bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; + + rte_atomic16_clear(&tp->processing_status); + rte_atomic16_clear(&tp->nb_dequeued); + + while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) + rte_pause(); + + tp->start_time = rte_rdtsc_precise(); + for (enqueued = 0; enqueued < num_to_process;) { + + uint16_t num_to_enq = burst_sz; + + if (unlikely(num_to_process - enqueued < num_to_enq)) + num_to_enq = num_to_process - enqueued; + + ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops, + num_to_enq); + if (ret != 0) { + allocs_failed++; + continue; + } + + if (test_vector.op_type != RTE_BBDEV_OP_NONE) + copy_reference_dec_op(ops, num_to_enq, enqueued, + bufs->inputs, + bufs->hard_outputs, + bufs->soft_outputs, + tp->op_params->ref_dec_op); + + enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, queue_id, ops, + num_to_enq); + + rte_bbdev_dec_op_free_bulk(ops, num_to_enq); + } + + if (allocs_failed > 0) + printf("WARNING: op allocations failed: %u times\n", + allocs_failed); + + return TEST_SUCCESS; +} + +static int +throughput_intr_lcore_enc(void *arg) +{ + struct thread_params *tp = arg; + unsigned int enqueued; + struct rte_bbdev_enc_op *ops[MAX_BURST]; + const uint16_t queue_id = tp->queue_id; + const uint16_t burst_sz = tp->op_params->burst_sz; + const uint16_t num_to_process = tp->op_params->num_to_process; + struct test_buffers *bufs = NULL; + unsigned int allocs_failed = 0; + struct rte_bbdev_info info; + int ret; + + TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), + "BURST_SIZE should be <= %u", MAX_BURST); + + TEST_ASSERT_SUCCESS(rte_bbdev_queue_intr_enable(tp->dev_id, queue_id), + "Failed to enable interrupts for dev: %u, queue_id: %u", + tp->dev_id, queue_id); + + rte_bbdev_info_get(tp->dev_id, &info); + bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; + + rte_atomic16_clear(&tp->processing_status); + rte_atomic16_clear(&tp->nb_dequeued); + + while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) + rte_pause(); + + tp->start_time = rte_rdtsc_precise(); + for (enqueued = 0; enqueued < num_to_process;) { + + uint16_t num_to_enq = burst_sz; + + if (unlikely(num_to_process - enqueued < num_to_enq)) + num_to_enq = num_to_process - enqueued; + + ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, ops, + num_to_enq); + if (ret != 0) { + allocs_failed++; + continue; + } + + if (test_vector.op_type != RTE_BBDEV_OP_NONE) + copy_reference_enc_op(ops, num_to_enq, enqueued, + bufs->inputs, + bufs->hard_outputs, + tp->op_params->ref_enc_op); + + enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, queue_id, ops, + num_to_enq); + + rte_bbdev_enc_op_free_bulk(ops, num_to_enq); + } + + if (allocs_failed > 0) + printf("WARNING: op allocations failed: %u times\n", + allocs_failed); + + return TEST_SUCCESS; +} + +static int +throughput_pmd_lcore_dec(void *arg) +{ + struct thread_params *tp = arg; + unsigned int enqueued, dequeued; + struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; + uint64_t total_time, start_time; + const uint16_t queue_id = tp->queue_id; + const uint16_t burst_sz = tp->op_params->burst_sz; + const uint16_t num_to_process = tp->op_params->num_to_process; + struct rte_bbdev_dec_op *ref_op = tp->op_params->ref_dec_op; + struct test_buffers *bufs = NULL; + unsigned int allocs_failed = 0; + int ret; + struct rte_bbdev_info info; + + /* Input length in bytes, million operations per second, million bits + * per second. + */ + double in_len; + + TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), + "BURST_SIZE should be <= %u", MAX_BURST); + + rte_bbdev_info_get(tp->dev_id, &info); + bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; + + while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) + rte_pause(); + + start_time = rte_rdtsc_precise(); + for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) { + uint16_t deq; + + if (likely(enqueued < num_to_process)) { + + uint16_t num_to_enq = burst_sz; + + if (unlikely(num_to_process - enqueued < num_to_enq)) + num_to_enq = num_to_process - enqueued; + + ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, + ops_enq, num_to_enq); + if (ret != 0) { + allocs_failed++; + goto do_dequeue; + } + + if (test_vector.op_type != RTE_BBDEV_OP_NONE) + copy_reference_dec_op(ops_enq, num_to_enq, + enqueued, + bufs->inputs, + bufs->hard_outputs, + bufs->soft_outputs, + ref_op); + + enqueued += rte_bbdev_enqueue_dec_ops(tp->dev_id, + queue_id, ops_enq, num_to_enq); + } +do_dequeue: + deq = rte_bbdev_dequeue_dec_ops(tp->dev_id, queue_id, ops_deq, + burst_sz); + dequeued += deq; + rte_bbdev_dec_op_free_bulk(ops_enq, deq); + } + total_time = rte_rdtsc_precise() - start_time; + + if (allocs_failed > 0) + printf("WARNING: op allocations failed: %u times\n", + allocs_failed); + + TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)", + enqueued, dequeued); + + if (test_vector.op_type != RTE_BBDEV_OP_NONE) { + ret = validate_dec_buffers(ref_op, bufs, num_to_process); + TEST_ASSERT_SUCCESS(ret, "Buffers validation failed"); + } + + in_len = ref_op->turbo_dec.input.length; + tp->mops = ((double)num_to_process / 1000000.0) / + ((double)total_time / (double)rte_get_tsc_hz()); + tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / + ((double)total_time / (double)rte_get_tsc_hz()); + + return TEST_SUCCESS; +} + +static int +throughput_pmd_lcore_enc(void *arg) +{ + struct thread_params *tp = arg; + unsigned int enqueued, dequeued; + struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; + uint64_t total_time, start_time; + const uint16_t queue_id = tp->queue_id; + const uint16_t burst_sz = tp->op_params->burst_sz; + const uint16_t num_to_process = tp->op_params->num_to_process; + struct rte_bbdev_enc_op *ref_op = tp->op_params->ref_enc_op; + struct test_buffers *bufs = NULL; + unsigned int allocs_failed = 0; + int ret; + struct rte_bbdev_info info; + + /* Input length in bytes, million operations per second, million bits + * per second. + */ + double in_len; + + TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), + "BURST_SIZE should be <= %u", MAX_BURST); + + rte_bbdev_info_get(tp->dev_id, &info); + bufs = &tp->op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; + + while (rte_atomic16_read(&tp->op_params->sync) == SYNC_WAIT) + rte_pause(); + + start_time = rte_rdtsc_precise(); + for (enqueued = 0, dequeued = 0; dequeued < num_to_process;) { + uint16_t deq; + + if (likely(enqueued < num_to_process)) { + + uint16_t num_to_enq = burst_sz; + + if (unlikely(num_to_process - enqueued < num_to_enq)) + num_to_enq = num_to_process - enqueued; + + ret = rte_bbdev_enc_op_alloc_bulk(tp->op_params->mp, + ops_enq, num_to_enq); + if (ret != 0) { + allocs_failed++; + goto do_dequeue; + } + + if (test_vector.op_type != RTE_BBDEV_OP_NONE) + copy_reference_enc_op(ops_enq, num_to_enq, + enqueued, + bufs->inputs, + bufs->hard_outputs, + ref_op); + + enqueued += rte_bbdev_enqueue_enc_ops(tp->dev_id, + queue_id, ops_enq, num_to_enq); + } +do_dequeue: + deq = rte_bbdev_dequeue_enc_ops(tp->dev_id, queue_id, ops_deq, + burst_sz); + dequeued += deq; + rte_bbdev_enc_op_free_bulk(ops_enq, deq); + } + total_time = rte_rdtsc_precise() - start_time; + + if (allocs_failed > 0) + printf("WARNING: op allocations failed: %u times\n", + allocs_failed); + + TEST_ASSERT(enqueued == dequeued, "enqueued (%u) != dequeued (%u)", + enqueued, dequeued); + + if (test_vector.op_type != RTE_BBDEV_OP_NONE) { + ret = validate_enc_buffers(bufs, num_to_process); + TEST_ASSERT_SUCCESS(ret, "Buffers validation failed"); + } + + in_len = ref_op->turbo_enc.input.length; + + tp->mops = ((double)num_to_process / 1000000.0) / + ((double)total_time / (double)rte_get_tsc_hz()); + tp->mbps = ((double)num_to_process * in_len * 8 / 1000000.0) / + ((double)total_time / (double)rte_get_tsc_hz()); + + return TEST_SUCCESS; +} +static void +print_throughput(struct thread_params *t_params, unsigned int used_cores) +{ + unsigned int lcore_id, iter = 0; + double total_mops = 0, total_mbps = 0; + + RTE_LCORE_FOREACH(lcore_id) { + if (iter++ >= used_cores) + break; + printf("\tlcore_id: %u, throughput: %.8lg MOPS, %.8lg Mbps\n", + lcore_id, t_params[lcore_id].mops, t_params[lcore_id].mbps); + total_mops += t_params[lcore_id].mops; + total_mbps += t_params[lcore_id].mbps; + } + printf( + "\n\tTotal stats for %u cores: throughput: %.8lg MOPS, %.8lg Mbps\n", + used_cores, total_mops, total_mbps); +} + +/* + * Test function that determines how long an enqueue + dequeue of a burst + * takes on available lcores. + */ +static int +throughput_test(struct active_device *ad, + struct test_op_params *op_params) +{ + int ret; + unsigned int lcore_id, used_cores = 0; + struct thread_params t_params[MAX_QUEUES]; + struct rte_bbdev_info info; + lcore_function_t *throughput_function; + struct thread_params *tp; + uint16_t num_lcores; + const char *op_type_str; + + rte_bbdev_info_get(ad->dev_id, &info); + + op_type_str = rte_bbdev_op_type_str(test_vector.op_type); + TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", + test_vector.op_type); + + printf( + "Throughput test: dev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u, op type: %s, int mode: %s, GHz: %lg\n", + info.dev_name, ad->nb_queues, op_params->burst_sz, + op_params->num_to_process, op_params->num_lcores, + op_type_str, + intr_enabled ? "Interrupt mode" : "PMD mode", + (double)rte_get_tsc_hz() / 1000000000.0); + + /* Set number of lcores */ + num_lcores = (ad->nb_queues < (op_params->num_lcores)) + ? ad->nb_queues + : op_params->num_lcores; + + if (intr_enabled) { + if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) + throughput_function = throughput_intr_lcore_dec; + else + throughput_function = throughput_intr_lcore_enc; + + /* Dequeue interrupt callback registration */ + ret = rte_bbdev_callback_register(ad->dev_id, + RTE_BBDEV_EVENT_DEQUEUE, dequeue_event_callback, + &t_params); + if (ret < 0) + return ret; + } else { + if (test_vector.op_type == RTE_BBDEV_OP_TURBO_DEC) + throughput_function = throughput_pmd_lcore_dec; + else + throughput_function = throughput_pmd_lcore_enc; + } + + rte_atomic16_set(&op_params->sync, SYNC_WAIT); + + t_params[rte_lcore_id()].dev_id = ad->dev_id; + t_params[rte_lcore_id()].op_params = op_params; + t_params[rte_lcore_id()].queue_id = + ad->queue_ids[used_cores++]; + + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (used_cores >= num_lcores) + break; + + t_params[lcore_id].dev_id = ad->dev_id; + t_params[lcore_id].op_params = op_params; + t_params[lcore_id].queue_id = ad->queue_ids[used_cores++]; + + rte_eal_remote_launch(throughput_function, &t_params[lcore_id], + lcore_id); + } + + rte_atomic16_set(&op_params->sync, SYNC_START); + ret = throughput_function(&t_params[rte_lcore_id()]); + + /* Master core is always used */ + used_cores = 1; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (used_cores++ >= num_lcores) + break; + + ret |= rte_eal_wait_lcore(lcore_id); + } + + /* Return if test failed */ + if (ret) + return ret; + + /* Print throughput if interrupts are disabled and test passed */ + if (!intr_enabled) { + if (test_vector.op_type != RTE_BBDEV_OP_NONE) + print_throughput(t_params, num_lcores); + return ret; + } + + /* In interrupt TC we need to wait for the interrupt callback to deqeue + * all pending operations. Skip waiting for queues which reported an + * error using processing_status variable. + * Wait for master lcore operations. + */ + tp = &t_params[rte_lcore_id()]; + while ((rte_atomic16_read(&tp->nb_dequeued) < + op_params->num_to_process) && + (rte_atomic16_read(&tp->processing_status) != + TEST_FAILED)) + rte_pause(); + + ret |= rte_atomic16_read(&tp->processing_status); + + /* Wait for slave lcores operations */ + used_cores = 1; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + tp = &t_params[lcore_id]; + if (used_cores++ >= num_lcores) + break; + + while ((rte_atomic16_read(&tp->nb_dequeued) < + op_params->num_to_process) && + (rte_atomic16_read(&tp->processing_status) != + TEST_FAILED)) + rte_pause(); + + ret |= rte_atomic16_read(&tp->processing_status); + } + + /* Print throughput if test passed */ + if (!ret && test_vector.op_type != RTE_BBDEV_OP_NONE) + print_throughput(t_params, num_lcores); + + return ret; +} + +static int +latency_test_dec(struct rte_mempool *mempool, + struct test_buffers *bufs, struct rte_bbdev_dec_op *ref_op, + int vector_mask, uint16_t dev_id, uint16_t queue_id, + const uint16_t num_to_process, uint16_t burst_sz, + uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) +{ + int ret = TEST_SUCCESS; + uint16_t i, j, dequeued; + struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; + uint64_t start_time = 0, last_time = 0; + + for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { + uint16_t enq = 0, deq = 0; + bool first_time = true; + last_time = 0; + + if (unlikely(num_to_process - dequeued < burst_sz)) + burst_sz = num_to_process - dequeued; + + ret = rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); + TEST_ASSERT_SUCCESS(ret, + "rte_bbdev_dec_op_alloc_bulk() failed"); + if (test_vector.op_type != RTE_BBDEV_OP_NONE) + copy_reference_dec_op(ops_enq, burst_sz, dequeued, + bufs->inputs, + bufs->hard_outputs, + bufs->soft_outputs, + ref_op); + + /* Set counter to validate the ordering */ + for (j = 0; j < burst_sz; ++j) + ops_enq[j]->opaque_data = (void *)(uintptr_t)j; + + start_time = rte_rdtsc_precise(); + + enq = rte_bbdev_enqueue_dec_ops(dev_id, queue_id, &ops_enq[enq], + burst_sz); + TEST_ASSERT(enq == burst_sz, + "Error enqueueing burst, expected %u, got %u", + burst_sz, enq); + + /* Dequeue */ + do { + deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, + &ops_deq[deq], burst_sz - deq); + if (likely(first_time && (deq > 0))) { + last_time = rte_rdtsc_precise() - start_time; + first_time = false; + } + } while (unlikely(burst_sz != deq)); + + *max_time = RTE_MAX(*max_time, last_time); + *min_time = RTE_MIN(*min_time, last_time); + *total_time += last_time; + + if (test_vector.op_type != RTE_BBDEV_OP_NONE) { + ret = validate_dec_op(ops_deq, burst_sz, ref_op, + vector_mask); + TEST_ASSERT_SUCCESS(ret, "Validation failed!"); + } + + rte_bbdev_dec_op_free_bulk(ops_enq, deq); + dequeued += deq; + } + + return i; +} + +static int +latency_test_enc(struct rte_mempool *mempool, + struct test_buffers *bufs, struct rte_bbdev_enc_op *ref_op, + uint16_t dev_id, uint16_t queue_id, + const uint16_t num_to_process, uint16_t burst_sz, + uint64_t *total_time, uint64_t *min_time, uint64_t *max_time) +{ + int ret = TEST_SUCCESS; + uint16_t i, j, dequeued; + struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; + uint64_t start_time = 0, last_time = 0; + + for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { + uint16_t enq = 0, deq = 0; + bool first_time = true; + last_time = 0; + + if (unlikely(num_to_process - dequeued < burst_sz)) + burst_sz = num_to_process - dequeued; + + ret = rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); + TEST_ASSERT_SUCCESS(ret, + "rte_bbdev_enc_op_alloc_bulk() failed"); + if (test_vector.op_type != RTE_BBDEV_OP_NONE) + copy_reference_enc_op(ops_enq, burst_sz, dequeued, + bufs->inputs, + bufs->hard_outputs, + ref_op); + + /* Set counter to validate the ordering */ + for (j = 0; j < burst_sz; ++j) + ops_enq[j]->opaque_data = (void *)(uintptr_t)j; + + start_time = rte_rdtsc_precise(); + + enq = rte_bbdev_enqueue_enc_ops(dev_id, queue_id, &ops_enq[enq], + burst_sz); + TEST_ASSERT(enq == burst_sz, + "Error enqueueing burst, expected %u, got %u", + burst_sz, enq); + + /* Dequeue */ + do { + deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, + &ops_deq[deq], burst_sz - deq); + if (likely(first_time && (deq > 0))) { + last_time += rte_rdtsc_precise() - start_time; + first_time = false; + } + } while (unlikely(burst_sz != deq)); + + *max_time = RTE_MAX(*max_time, last_time); + *min_time = RTE_MIN(*min_time, last_time); + *total_time += last_time; + + if (test_vector.op_type != RTE_BBDEV_OP_NONE) { + ret = validate_enc_op(ops_deq, burst_sz, ref_op); + TEST_ASSERT_SUCCESS(ret, "Validation failed!"); + } + + rte_bbdev_enc_op_free_bulk(ops_enq, deq); + dequeued += deq; + } + + return i; +} + +static int +latency_test(struct active_device *ad, + struct test_op_params *op_params) +{ + int iter; + uint16_t burst_sz = op_params->burst_sz; + const uint16_t num_to_process = op_params->num_to_process; + const enum rte_bbdev_op_type op_type = test_vector.op_type; + const uint16_t queue_id = ad->queue_ids[0]; + struct test_buffers *bufs = NULL; + struct rte_bbdev_info info; + uint64_t total_time, min_time, max_time; + const char *op_type_str; + + total_time = max_time = 0; + min_time = UINT64_MAX; + + TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), + "BURST_SIZE should be <= %u", MAX_BURST); + + rte_bbdev_info_get(ad->dev_id, &info); + bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; + + op_type_str = rte_bbdev_op_type_str(op_type); + TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); + + printf( + "Validation/Latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", + info.dev_name, burst_sz, num_to_process, op_type_str); + + if (op_type == RTE_BBDEV_OP_TURBO_DEC) + iter = latency_test_dec(op_params->mp, bufs, + op_params->ref_dec_op, op_params->vector_mask, + ad->dev_id, queue_id, num_to_process, + burst_sz, &total_time, &min_time, &max_time); + else + iter = latency_test_enc(op_params->mp, bufs, + op_params->ref_enc_op, ad->dev_id, queue_id, + num_to_process, burst_sz, &total_time, + &min_time, &max_time); + + if (iter <= 0) + return TEST_FAILED; + + printf("\toperation latency:\n" + "\t\tavg latency: %lg cycles, %lg us\n" + "\t\tmin latency: %lg cycles, %lg us\n" + "\t\tmax latency: %lg cycles, %lg us\n", + (double)total_time / (double)iter, + (double)(total_time * 1000000) / (double)iter / + (double)rte_get_tsc_hz(), (double)min_time, + (double)(min_time * 1000000) / (double)rte_get_tsc_hz(), + (double)max_time, (double)(max_time * 1000000) / + (double)rte_get_tsc_hz()); + + return TEST_SUCCESS; +} + +#ifdef RTE_BBDEV_OFFLOAD_COST +static int +get_bbdev_queue_stats(uint16_t dev_id, uint16_t queue_id, + struct rte_bbdev_stats *stats) +{ + struct rte_bbdev *dev = &rte_bbdev_devices[dev_id]; + struct rte_bbdev_stats *q_stats; + + if (queue_id >= dev->data->num_queues) + return -1; + + q_stats = &dev->data->queues[queue_id].queue_stats; + + stats->enqueued_count = q_stats->enqueued_count; + stats->dequeued_count = q_stats->dequeued_count; + stats->enqueue_err_count = q_stats->enqueue_err_count; + stats->dequeue_err_count = q_stats->dequeue_err_count; + stats->offload_time = q_stats->offload_time; + + return 0; +} + +static int +offload_latency_test_dec(struct rte_mempool *mempool, struct test_buffers *bufs, + struct rte_bbdev_dec_op *ref_op, uint16_t dev_id, + uint16_t queue_id, const uint16_t num_to_process, + uint16_t burst_sz, struct test_time_stats *time_st) +{ + int i, dequeued, ret; + struct rte_bbdev_dec_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; + uint64_t enq_start_time, deq_start_time; + uint64_t enq_sw_last_time, deq_last_time; + struct rte_bbdev_stats stats; + + for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { + uint16_t enq = 0, deq = 0; + + if (unlikely(num_to_process - dequeued < burst_sz)) + burst_sz = num_to_process - dequeued; + + rte_bbdev_dec_op_alloc_bulk(mempool, ops_enq, burst_sz); + if (test_vector.op_type != RTE_BBDEV_OP_NONE) + copy_reference_dec_op(ops_enq, burst_sz, dequeued, + bufs->inputs, + bufs->hard_outputs, + bufs->soft_outputs, + ref_op); + + /* Start time meas for enqueue function offload latency */ + enq_start_time = rte_rdtsc_precise(); + do { + enq += rte_bbdev_enqueue_dec_ops(dev_id, queue_id, + &ops_enq[enq], burst_sz - enq); + } while (unlikely(burst_sz != enq)); + + ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); + TEST_ASSERT_SUCCESS(ret, + "Failed to get stats for queue (%u) of device (%u)", + queue_id, dev_id); + + enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - + stats.offload_time; + time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, + enq_sw_last_time); + time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, + enq_sw_last_time); + time_st->enq_sw_tot_time += enq_sw_last_time; + + time_st->enq_tur_max_time = RTE_MAX(time_st->enq_tur_max_time, + stats.offload_time); + time_st->enq_tur_min_time = RTE_MIN(time_st->enq_tur_min_time, + stats.offload_time); + time_st->enq_tur_tot_time += stats.offload_time; + + /* ensure enqueue has been completed */ + rte_delay_ms(10); + + /* Start time meas for dequeue function offload latency */ + deq_start_time = rte_rdtsc_precise(); + /* Dequeue one operation */ + do { + deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, + &ops_deq[deq], 1); + } while (unlikely(deq != 1)); + + deq_last_time = rte_rdtsc_precise() - deq_start_time; + time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, + deq_last_time); + time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, + deq_last_time); + time_st->deq_tot_time += deq_last_time; + + /* Dequeue remaining operations if needed*/ + while (burst_sz != deq) + deq += rte_bbdev_dequeue_dec_ops(dev_id, queue_id, + &ops_deq[deq], burst_sz - deq); + + rte_bbdev_dec_op_free_bulk(ops_enq, deq); + dequeued += deq; + } + + return i; +} + +static int +offload_latency_test_enc(struct rte_mempool *mempool, struct test_buffers *bufs, + struct rte_bbdev_enc_op *ref_op, uint16_t dev_id, + uint16_t queue_id, const uint16_t num_to_process, + uint16_t burst_sz, struct test_time_stats *time_st) +{ + int i, dequeued, ret; + struct rte_bbdev_enc_op *ops_enq[MAX_BURST], *ops_deq[MAX_BURST]; + uint64_t enq_start_time, deq_start_time; + uint64_t enq_sw_last_time, deq_last_time; + struct rte_bbdev_stats stats; + + for (i = 0, dequeued = 0; dequeued < num_to_process; ++i) { + uint16_t enq = 0, deq = 0; + + if (unlikely(num_to_process - dequeued < burst_sz)) + burst_sz = num_to_process - dequeued; + + rte_bbdev_enc_op_alloc_bulk(mempool, ops_enq, burst_sz); + if (test_vector.op_type != RTE_BBDEV_OP_NONE) + copy_reference_enc_op(ops_enq, burst_sz, dequeued, + bufs->inputs, + bufs->hard_outputs, + ref_op); + + /* Start time meas for enqueue function offload latency */ + enq_start_time = rte_rdtsc_precise(); + do { + enq += rte_bbdev_enqueue_enc_ops(dev_id, queue_id, + &ops_enq[enq], burst_sz - enq); + } while (unlikely(burst_sz != enq)); + + ret = get_bbdev_queue_stats(dev_id, queue_id, &stats); + TEST_ASSERT_SUCCESS(ret, + "Failed to get stats for queue (%u) of device (%u)", + queue_id, dev_id); + + enq_sw_last_time = rte_rdtsc_precise() - enq_start_time - + stats.offload_time; + time_st->enq_sw_max_time = RTE_MAX(time_st->enq_sw_max_time, + enq_sw_last_time); + time_st->enq_sw_min_time = RTE_MIN(time_st->enq_sw_min_time, + enq_sw_last_time); + time_st->enq_sw_tot_time += enq_sw_last_time; + + time_st->enq_tur_max_time = RTE_MAX(time_st->enq_tur_max_time, + stats.offload_time); + time_st->enq_tur_min_time = RTE_MIN(time_st->enq_tur_min_time, + stats.offload_time); + time_st->enq_tur_tot_time += stats.offload_time; + + /* ensure enqueue has been completed */ + rte_delay_ms(10); + + /* Start time meas for dequeue function offload latency */ + deq_start_time = rte_rdtsc_precise(); + /* Dequeue one operation */ + do { + deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, + &ops_deq[deq], 1); + } while (unlikely(deq != 1)); + + deq_last_time = rte_rdtsc_precise() - deq_start_time; + time_st->deq_max_time = RTE_MAX(time_st->deq_max_time, + deq_last_time); + time_st->deq_min_time = RTE_MIN(time_st->deq_min_time, + deq_last_time); + time_st->deq_tot_time += deq_last_time; + + while (burst_sz != deq) + deq += rte_bbdev_dequeue_enc_ops(dev_id, queue_id, + &ops_deq[deq], burst_sz - deq); + + rte_bbdev_enc_op_free_bulk(ops_enq, deq); + dequeued += deq; + } + + return i; +} +#endif + +static int +offload_cost_test(struct active_device *ad, + struct test_op_params *op_params) +{ +#ifndef RTE_BBDEV_OFFLOAD_COST + RTE_SET_USED(ad); + RTE_SET_USED(op_params); + printf("Offload latency test is disabled.\n"); + printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); + return TEST_SKIPPED; +#else + int iter; + uint16_t burst_sz = op_params->burst_sz; + const uint16_t num_to_process = op_params->num_to_process; + const enum rte_bbdev_op_type op_type = test_vector.op_type; + const uint16_t queue_id = ad->queue_ids[0]; + struct test_buffers *bufs = NULL; + struct rte_bbdev_info info; + const char *op_type_str; + struct test_time_stats time_st; + + memset(&time_st, 0, sizeof(struct test_time_stats)); + time_st.enq_sw_min_time = UINT64_MAX; + time_st.enq_tur_min_time = UINT64_MAX; + time_st.deq_min_time = UINT64_MAX; + + TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), + "BURST_SIZE should be <= %u", MAX_BURST); + + rte_bbdev_info_get(ad->dev_id, &info); + bufs = &op_params->q_bufs[GET_SOCKET(info.socket_id)][queue_id]; + + op_type_str = rte_bbdev_op_type_str(op_type); + TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); + + printf( + "Offload latency test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", + info.dev_name, burst_sz, num_to_process, op_type_str); + + if (op_type == RTE_BBDEV_OP_TURBO_DEC) + iter = offload_latency_test_dec(op_params->mp, bufs, + op_params->ref_dec_op, ad->dev_id, queue_id, + num_to_process, burst_sz, &time_st); + else + iter = offload_latency_test_enc(op_params->mp, bufs, + op_params->ref_enc_op, ad->dev_id, queue_id, + num_to_process, burst_sz, &time_st); + + if (iter <= 0) + return TEST_FAILED; + + printf("\tenq offload cost latency:\n" + "\t\tsoftware avg %lg cycles, %lg us\n" + "\t\tsoftware min %lg cycles, %lg us\n" + "\t\tsoftware max %lg cycles, %lg us\n" + "\t\tturbo avg %lg cycles, %lg us\n" + "\t\tturbo min %lg cycles, %lg us\n" + "\t\tturbo max %lg cycles, %lg us\n", + (double)time_st.enq_sw_tot_time / (double)iter, + (double)(time_st.enq_sw_tot_time * 1000000) / + (double)iter / (double)rte_get_tsc_hz(), + (double)time_st.enq_sw_min_time, + (double)(time_st.enq_sw_min_time * 1000000) / + rte_get_tsc_hz(), (double)time_st.enq_sw_max_time, + (double)(time_st.enq_sw_max_time * 1000000) / + rte_get_tsc_hz(), (double)time_st.enq_tur_tot_time / + (double)iter, + (double)(time_st.enq_tur_tot_time * 1000000) / + (double)iter / (double)rte_get_tsc_hz(), + (double)time_st.enq_tur_min_time, + (double)(time_st.enq_tur_min_time * 1000000) / + rte_get_tsc_hz(), (double)time_st.enq_tur_max_time, + (double)(time_st.enq_tur_max_time * 1000000) / + rte_get_tsc_hz()); + + printf("\tdeq offload cost latency - one op:\n" + "\t\tavg %lg cycles, %lg us\n" + "\t\tmin %lg cycles, %lg us\n" + "\t\tmax %lg cycles, %lg us\n", + (double)time_st.deq_tot_time / (double)iter, + (double)(time_st.deq_tot_time * 1000000) / + (double)iter / (double)rte_get_tsc_hz(), + (double)time_st.deq_min_time, + (double)(time_st.deq_min_time * 1000000) / + rte_get_tsc_hz(), (double)time_st.deq_max_time, + (double)(time_st.deq_max_time * 1000000) / + rte_get_tsc_hz()); + + return TEST_SUCCESS; +#endif +} + +#ifdef RTE_BBDEV_OFFLOAD_COST +static int +offload_latency_empty_q_test_dec(uint16_t dev_id, uint16_t queue_id, + const uint16_t num_to_process, uint16_t burst_sz, + uint64_t *deq_tot_time, uint64_t *deq_min_time, + uint64_t *deq_max_time) +{ + int i, deq_total; + struct rte_bbdev_dec_op *ops[MAX_BURST]; + uint64_t deq_start_time, deq_last_time; + + /* Test deq offload latency from an empty queue */ + + for (i = 0, deq_total = 0; deq_total < num_to_process; + ++i, deq_total += burst_sz) { + deq_start_time = rte_rdtsc_precise(); + + if (unlikely(num_to_process - deq_total < burst_sz)) + burst_sz = num_to_process - deq_total; + rte_bbdev_dequeue_dec_ops(dev_id, queue_id, ops, burst_sz); + + deq_last_time = rte_rdtsc_precise() - deq_start_time; + *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); + *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); + *deq_tot_time += deq_last_time; + } + + return i; +} + +static int +offload_latency_empty_q_test_enc(uint16_t dev_id, uint16_t queue_id, + const uint16_t num_to_process, uint16_t burst_sz, + uint64_t *deq_tot_time, uint64_t *deq_min_time, + uint64_t *deq_max_time) +{ + int i, deq_total; + struct rte_bbdev_enc_op *ops[MAX_BURST]; + uint64_t deq_start_time, deq_last_time; + + /* Test deq offload latency from an empty queue */ + for (i = 0, deq_total = 0; deq_total < num_to_process; + ++i, deq_total += burst_sz) { + deq_start_time = rte_rdtsc_precise(); + + if (unlikely(num_to_process - deq_total < burst_sz)) + burst_sz = num_to_process - deq_total; + rte_bbdev_dequeue_enc_ops(dev_id, queue_id, ops, burst_sz); + + deq_last_time = rte_rdtsc_precise() - deq_start_time; + *deq_max_time = RTE_MAX(*deq_max_time, deq_last_time); + *deq_min_time = RTE_MIN(*deq_min_time, deq_last_time); + *deq_tot_time += deq_last_time; + } + + return i; +} +#endif + +static int +offload_latency_empty_q_test(struct active_device *ad, + struct test_op_params *op_params) +{ +#ifndef RTE_BBDEV_OFFLOAD_COST + RTE_SET_USED(ad); + RTE_SET_USED(op_params); + printf("Offload latency empty dequeue test is disabled.\n"); + printf("Set RTE_BBDEV_OFFLOAD_COST to 'y' to turn the test on.\n"); + return TEST_SKIPPED; +#else + int iter; + uint64_t deq_tot_time, deq_min_time, deq_max_time; + uint16_t burst_sz = op_params->burst_sz; + const uint16_t num_to_process = op_params->num_to_process; + const enum rte_bbdev_op_type op_type = test_vector.op_type; + const uint16_t queue_id = ad->queue_ids[0]; + struct rte_bbdev_info info; + const char *op_type_str; + + deq_tot_time = deq_max_time = 0; + deq_min_time = UINT64_MAX; + + TEST_ASSERT_SUCCESS((burst_sz > MAX_BURST), + "BURST_SIZE should be <= %u", MAX_BURST); + + rte_bbdev_info_get(ad->dev_id, &info); + + op_type_str = rte_bbdev_op_type_str(op_type); + TEST_ASSERT_NOT_NULL(op_type_str, "Invalid op type: %u", op_type); + + printf( + "Offload latency empty dequeue test: dev: %s, burst size: %u, num ops: %u, op type: %s\n", + info.dev_name, burst_sz, num_to_process, op_type_str); + + if (op_type == RTE_BBDEV_OP_TURBO_DEC) + iter = offload_latency_empty_q_test_dec(ad->dev_id, queue_id, + num_to_process, burst_sz, &deq_tot_time, + &deq_min_time, &deq_max_time); + else + iter = offload_latency_empty_q_test_enc(ad->dev_id, queue_id, + num_to_process, burst_sz, &deq_tot_time, + &deq_min_time, &deq_max_time); + + if (iter <= 0) + return TEST_FAILED; + + printf("\tempty deq offload\n" + "\t\tavg. latency: %lg cycles, %lg us\n" + "\t\tmin. latency: %lg cycles, %lg us\n" + "\t\tmax. latency: %lg cycles, %lg us\n", + (double)deq_tot_time / (double)iter, + (double)(deq_tot_time * 1000000) / (double)iter / + (double)rte_get_tsc_hz(), (double)deq_min_time, + (double)(deq_min_time * 1000000) / rte_get_tsc_hz(), + (double)deq_max_time, (double)(deq_max_time * 1000000) / + rte_get_tsc_hz()); + + return TEST_SUCCESS; +#endif +} + +static int +throughput_tc(void) +{ + return run_test_case(throughput_test); +} + +static int +offload_cost_tc(void) +{ + return run_test_case(offload_cost_test); +} + +static int +offload_latency_empty_q_tc(void) +{ + return run_test_case(offload_latency_empty_q_test); +} + +static int +latency_tc(void) +{ + return run_test_case(latency_test); +} + +static int +interrupt_tc(void) +{ + return run_test_case(throughput_test); +} + +static struct unit_test_suite bbdev_throughput_testsuite = { + .suite_name = "BBdev Throughput Tests", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, throughput_tc), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite bbdev_validation_testsuite = { + .suite_name = "BBdev Validation Tests", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite bbdev_latency_testsuite = { + .suite_name = "BBdev Latency Tests", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, latency_tc), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite bbdev_offload_cost_testsuite = { + .suite_name = "BBdev Offload Cost Tests", + .setup = testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, offload_cost_tc), + TEST_CASE_ST(ut_setup, ut_teardown, offload_latency_empty_q_tc), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +static struct unit_test_suite bbdev_interrupt_testsuite = { + .suite_name = "BBdev Interrupt Tests", + .setup = interrupt_testsuite_setup, + .teardown = testsuite_teardown, + .unit_test_cases = { + TEST_CASE_ST(ut_setup, ut_teardown, interrupt_tc), + TEST_CASES_END() /**< NULL terminate unit test array */ + } +}; + +REGISTER_TEST_COMMAND(throughput, bbdev_throughput_testsuite); +REGISTER_TEST_COMMAND(validation, bbdev_validation_testsuite); +REGISTER_TEST_COMMAND(latency, bbdev_latency_testsuite); +REGISTER_TEST_COMMAND(offload, bbdev_offload_cost_testsuite); +REGISTER_TEST_COMMAND(interrupt, bbdev_interrupt_testsuite); diff --git a/dpdk/app/test-bbdev/test_bbdev_vector.c b/dpdk/app/test-bbdev/test_bbdev_vector.c new file mode 100644 index 00000000..81b8ee78 --- /dev/null +++ b/dpdk/app/test-bbdev/test_bbdev_vector.c @@ -0,0 +1,939 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#ifdef RTE_EXEC_ENV_BSDAPP + #define _WITH_GETLINE +#endif +#include +#include +#include + +#include "test_bbdev_vector.h" + +#define VALUE_DELIMITER "," +#define ENTRY_DELIMITER "=" + +const char *op_data_prefixes[] = { + "input", + "soft_output", + "hard_output", +}; + +/* trim leading and trailing spaces */ +static void +trim_space(char *str) +{ + char *start, *end; + + for (start = str; *start; start++) { + if (!isspace((unsigned char) start[0])) + break; + } + + for (end = start + strlen(start); end > start + 1; end--) { + if (!isspace((unsigned char) end[-1])) + break; + } + + *end = 0; + + /* Shift from "start" to the beginning of the string */ + if (start > str) + memmove(str, start, (end - start) + 1); +} + +static bool +starts_with(const char *str, const char *pre) +{ + return strncmp(pre, str, strlen(pre)) == 0; +} + +/* tokenization test values separated by a comma */ +static int +parse_values(char *tokens, uint32_t **data, uint32_t *data_length) +{ + uint32_t n_tokens = 0; + uint32_t data_size = 32; + + uint32_t *values, *values_resized; + char *tok, *error = NULL; + + tok = strtok(tokens, VALUE_DELIMITER); + if (tok == NULL) + return -1; + + values = (uint32_t *) + rte_zmalloc(NULL, sizeof(uint32_t) * data_size, 0); + if (values == NULL) + return -1; + + while (tok != NULL) { + values_resized = NULL; + + if (n_tokens >= data_size) { + data_size *= 2; + + values_resized = (uint32_t *) rte_realloc(values, + sizeof(uint32_t) * data_size, 0); + if (values_resized == NULL) { + rte_free(values); + return -1; + } + values = values_resized; + } + + values[n_tokens] = (uint32_t) strtoul(tok, &error, 0); + if ((error == NULL) || (*error != '\0')) { + printf("Failed with convert '%s'\n", tok); + rte_free(values); + return -1; + } + + *data_length = *data_length + (strlen(tok) - strlen("0x"))/2; + + tok = strtok(NULL, VALUE_DELIMITER); + if (tok == NULL) + break; + + n_tokens++; + } + + values_resized = (uint32_t *) rte_realloc(values, + sizeof(uint32_t) * (n_tokens + 1), 0); + + if (values_resized == NULL) { + rte_free(values); + return -1; + } + + *data = values_resized; + + return 0; +} + +/* convert turbo decoder flag from string to unsigned long int*/ +static int +op_decoder_flag_strtoul(char *token, uint32_t *op_flag_value) +{ + if (!strcmp(token, "RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE")) + *op_flag_value = RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE; + else if (!strcmp(token, "RTE_BBDEV_TURBO_CRC_TYPE_24B")) + *op_flag_value = RTE_BBDEV_TURBO_CRC_TYPE_24B; + else if (!strcmp(token, "RTE_BBDEV_TURBO_EQUALIZER")) + *op_flag_value = RTE_BBDEV_TURBO_EQUALIZER; + else if (!strcmp(token, "RTE_BBDEV_TURBO_SOFT_OUT_SATURATE")) + *op_flag_value = RTE_BBDEV_TURBO_SOFT_OUT_SATURATE; + else if (!strcmp(token, "RTE_BBDEV_TURBO_HALF_ITERATION_EVEN")) + *op_flag_value = RTE_BBDEV_TURBO_HALF_ITERATION_EVEN; + else if (!strcmp(token, "RTE_BBDEV_TURBO_CONTINUE_CRC_MATCH")) + *op_flag_value = RTE_BBDEV_TURBO_CONTINUE_CRC_MATCH; + else if (!strcmp(token, "RTE_BBDEV_TURBO_SOFT_OUTPUT")) + *op_flag_value = RTE_BBDEV_TURBO_SOFT_OUTPUT; + else if (!strcmp(token, "RTE_BBDEV_TURBO_EARLY_TERMINATION")) + *op_flag_value = RTE_BBDEV_TURBO_EARLY_TERMINATION; + else if (!strcmp(token, "RTE_BBDEV_TURBO_POS_LLR_1_BIT_IN")) + *op_flag_value = RTE_BBDEV_TURBO_POS_LLR_1_BIT_IN; + else if (!strcmp(token, "RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN")) + *op_flag_value = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN; + else if (!strcmp(token, "RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT")) + *op_flag_value = RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT; + else if (!strcmp(token, "RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT")) + *op_flag_value = RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; + else if (!strcmp(token, "RTE_BBDEV_TURBO_MAP_DEC")) + *op_flag_value = RTE_BBDEV_TURBO_MAP_DEC; + else if (!strcmp(token, "RTE_BBDEV_TURBO_DEC_SCATTER_GATHER")) + *op_flag_value = RTE_BBDEV_TURBO_DEC_SCATTER_GATHER; + else if (!strcmp(token, "RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP")) + *op_flag_value = RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP; + else { + printf("The given value is not a turbo decoder flag\n"); + return -1; + } + + return 0; +} + +/* convert turbo encoder flag from string to unsigned long int*/ +static int +op_encoder_flag_strtoul(char *token, uint32_t *op_flag_value) +{ + if (!strcmp(token, "RTE_BBDEV_TURBO_RV_INDEX_BYPASS")) + *op_flag_value = RTE_BBDEV_TURBO_RV_INDEX_BYPASS; + else if (!strcmp(token, "RTE_BBDEV_TURBO_RATE_MATCH")) + *op_flag_value = RTE_BBDEV_TURBO_RATE_MATCH; + else if (!strcmp(token, "RTE_BBDEV_TURBO_CRC_24B_ATTACH")) + *op_flag_value = RTE_BBDEV_TURBO_CRC_24B_ATTACH; + else if (!strcmp(token, "RTE_BBDEV_TURBO_CRC_24A_ATTACH")) + *op_flag_value = RTE_BBDEV_TURBO_CRC_24A_ATTACH; + else if (!strcmp(token, "RTE_BBDEV_TURBO_ENC_SCATTER_GATHER")) + *op_flag_value = RTE_BBDEV_TURBO_ENC_SCATTER_GATHER; + else { + printf("The given value is not a turbo encoder flag\n"); + return -1; + } + + return 0; +} + +/* tokenization turbo decoder/encoder flags values separated by a comma */ +static int +parse_turbo_flags(char *tokens, uint32_t *op_flags, + enum rte_bbdev_op_type op_type) +{ + char *tok = NULL; + uint32_t op_flag_value = 0; + + tok = strtok(tokens, VALUE_DELIMITER); + if (tok == NULL) + return -1; + + while (tok != NULL) { + trim_space(tok); + if (op_type == RTE_BBDEV_OP_TURBO_DEC) { + if (op_decoder_flag_strtoul(tok, &op_flag_value) == -1) + return -1; + } else if (op_type == RTE_BBDEV_OP_TURBO_ENC) { + if (op_encoder_flag_strtoul(tok, &op_flag_value) == -1) + return -1; + } else { + return -1; + } + + *op_flags = *op_flags | op_flag_value; + + tok = strtok(NULL, VALUE_DELIMITER); + if (tok == NULL) + break; + } + + return 0; +} + +/* convert turbo encoder/decoder op_type from string to enum*/ +static int +op_turbo_type_strtol(char *token, enum rte_bbdev_op_type *op_type) +{ + trim_space(token); + if (!strcmp(token, "RTE_BBDEV_OP_TURBO_DEC")) + *op_type = RTE_BBDEV_OP_TURBO_DEC; + else if (!strcmp(token, "RTE_BBDEV_OP_TURBO_ENC")) + *op_type = RTE_BBDEV_OP_TURBO_ENC; + else if (!strcmp(token, "RTE_BBDEV_OP_NONE")) + *op_type = RTE_BBDEV_OP_NONE; + else { + printf("Not valid turbo op_type: '%s'\n", token); + return -1; + } + + return 0; +} + +/* tokenization expected status values separated by a comma */ +static int +parse_expected_status(char *tokens, int *status, enum rte_bbdev_op_type op_type) +{ + char *tok = NULL; + bool status_ok = false; + + tok = strtok(tokens, VALUE_DELIMITER); + if (tok == NULL) + return -1; + + while (tok != NULL) { + trim_space(tok); + if (!strcmp(tok, "OK")) + status_ok = true; + else if (!strcmp(tok, "DMA")) + *status = *status | (1 << RTE_BBDEV_DRV_ERROR); + else if (!strcmp(tok, "FCW")) + *status = *status | (1 << RTE_BBDEV_DATA_ERROR); + else if (!strcmp(tok, "CRC")) { + if (op_type == RTE_BBDEV_OP_TURBO_DEC) + *status = *status | (1 << RTE_BBDEV_CRC_ERROR); + else { + printf( + "CRC is only a valid value for turbo decoder\n"); + return -1; + } + } else { + printf("Not valid status: '%s'\n", tok); + return -1; + } + + tok = strtok(NULL, VALUE_DELIMITER); + if (tok == NULL) + break; + } + + if (status_ok && *status != 0) { + printf( + "Not valid status values. Cannot be OK and ERROR at the same time.\n"); + return -1; + } + + return 0; +} + +/* parse ops data entry (there can be more than 1 input entry, each will be + * contained in a separate op_data_buf struct) + */ +static int +parse_data_entry(const char *key_token, char *token, + struct test_bbdev_vector *vector, enum op_data_type type, + const char *prefix) +{ + int ret; + uint32_t data_length = 0; + uint32_t *data = NULL; + unsigned int id; + struct op_data_buf *op_data; + unsigned int *nb_ops; + + if (type >= DATA_NUM_TYPES) { + printf("Unknown op type: %d!\n", type); + return -1; + } + + op_data = vector->entries[type].segments; + nb_ops = &vector->entries[type].nb_segments; + + if (*nb_ops >= RTE_BBDEV_MAX_CODE_BLOCKS) { + printf("Too many segments (code blocks defined): %u, max %d!\n", + *nb_ops, RTE_BBDEV_MAX_CODE_BLOCKS); + return -1; + } + + if (sscanf(key_token + strlen(prefix), "%u", &id) != 1) { + printf("Missing ID of %s\n", prefix); + return -1; + } + if (id != *nb_ops) { + printf( + "Please order data entries sequentially, i.e. %s0, %s1, ...\n", + prefix, prefix); + return -1; + } + + /* Clear new op data struct */ + memset(op_data + *nb_ops, 0, sizeof(struct op_data_buf)); + + ret = parse_values(token, &data, &data_length); + if (!ret) { + op_data[*nb_ops].addr = data; + op_data[*nb_ops].length = data_length; + ++(*nb_ops); + } + + return ret; +} + +/* parses turbo decoder parameters and assigns to global variable */ +static int +parse_decoder_params(const char *key_token, char *token, + struct test_bbdev_vector *vector) +{ + int ret = 0, status = 0; + uint32_t op_flags = 0; + char *err = NULL; + + struct rte_bbdev_op_turbo_dec *turbo_dec = &vector->turbo_dec; + + /* compare keys */ + if (starts_with(key_token, op_data_prefixes[DATA_INPUT])) + ret = parse_data_entry(key_token, token, vector, + DATA_INPUT, op_data_prefixes[DATA_INPUT]); + + else if (starts_with(key_token, op_data_prefixes[DATA_SOFT_OUTPUT])) + ret = parse_data_entry(key_token, token, vector, + DATA_SOFT_OUTPUT, + op_data_prefixes[DATA_SOFT_OUTPUT]); + + else if (starts_with(key_token, op_data_prefixes[DATA_HARD_OUTPUT])) + ret = parse_data_entry(key_token, token, vector, + DATA_HARD_OUTPUT, + op_data_prefixes[DATA_HARD_OUTPUT]); + else if (!strcmp(key_token, "e")) { + vector->mask |= TEST_BBDEV_VF_E; + turbo_dec->cb_params.e = (uint32_t) strtoul(token, &err, 0); + } else if (!strcmp(key_token, "ea")) { + vector->mask |= TEST_BBDEV_VF_EA; + turbo_dec->tb_params.ea = (uint32_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "eb")) { + vector->mask |= TEST_BBDEV_VF_EB; + turbo_dec->tb_params.eb = (uint32_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "k")) { + vector->mask |= TEST_BBDEV_VF_K; + turbo_dec->cb_params.k = (uint16_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "k_pos")) { + vector->mask |= TEST_BBDEV_VF_K_POS; + turbo_dec->tb_params.k_pos = (uint16_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "k_neg")) { + vector->mask |= TEST_BBDEV_VF_K_NEG; + turbo_dec->tb_params.k_neg = (uint16_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "c")) { + vector->mask |= TEST_BBDEV_VF_C; + turbo_dec->tb_params.c = (uint16_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "c_neg")) { + vector->mask |= TEST_BBDEV_VF_C_NEG; + turbo_dec->tb_params.c_neg = (uint16_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "cab")) { + vector->mask |= TEST_BBDEV_VF_CAB; + turbo_dec->tb_params.cab = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "rv_index")) { + vector->mask |= TEST_BBDEV_VF_RV_INDEX; + turbo_dec->rv_index = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "iter_max")) { + vector->mask |= TEST_BBDEV_VF_ITER_MAX; + turbo_dec->iter_max = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "iter_min")) { + vector->mask |= TEST_BBDEV_VF_ITER_MIN; + turbo_dec->iter_min = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "expected_iter_count")) { + vector->mask |= TEST_BBDEV_VF_EXPECTED_ITER_COUNT; + turbo_dec->iter_count = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "ext_scale")) { + vector->mask |= TEST_BBDEV_VF_EXT_SCALE; + turbo_dec->ext_scale = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "num_maps")) { + vector->mask |= TEST_BBDEV_VF_NUM_MAPS; + turbo_dec->num_maps = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "code_block_mode")) { + vector->mask |= TEST_BBDEV_VF_CODE_BLOCK_MODE; + turbo_dec->code_block_mode = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "op_flags")) { + vector->mask |= TEST_BBDEV_VF_OP_FLAGS; + ret = parse_turbo_flags(token, &op_flags, + vector->op_type); + if (!ret) + turbo_dec->op_flags = op_flags; + } else if (!strcmp(key_token, "expected_status")) { + vector->mask |= TEST_BBDEV_VF_EXPECTED_STATUS; + ret = parse_expected_status(token, &status, vector->op_type); + if (!ret) + vector->expected_status = status; + } else { + printf("Not valid dec key: '%s'\n", key_token); + return -1; + } + + if (ret != 0) { + printf("Failed with convert '%s\t%s'\n", key_token, token); + return -1; + } + + return 0; +} + +/* parses turbo encoder parameters and assigns to global variable */ +static int +parse_encoder_params(const char *key_token, char *token, + struct test_bbdev_vector *vector) +{ + int ret = 0, status = 0; + uint32_t op_flags = 0; + char *err = NULL; + + + struct rte_bbdev_op_turbo_enc *turbo_enc = &vector->turbo_enc; + + if (starts_with(key_token, op_data_prefixes[DATA_INPUT])) + ret = parse_data_entry(key_token, token, vector, + DATA_INPUT, op_data_prefixes[DATA_INPUT]); + else if (starts_with(key_token, "output")) + ret = parse_data_entry(key_token, token, vector, + DATA_HARD_OUTPUT, "output"); + else if (!strcmp(key_token, "e")) { + vector->mask |= TEST_BBDEV_VF_E; + turbo_enc->cb_params.e = (uint32_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "ea")) { + vector->mask |= TEST_BBDEV_VF_EA; + turbo_enc->tb_params.ea = (uint32_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "eb")) { + vector->mask |= TEST_BBDEV_VF_EB; + turbo_enc->tb_params.eb = (uint32_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "k")) { + vector->mask |= TEST_BBDEV_VF_K; + turbo_enc->cb_params.k = (uint16_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "k_neg")) { + vector->mask |= TEST_BBDEV_VF_K_NEG; + turbo_enc->tb_params.k_neg = (uint16_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "k_pos")) { + vector->mask |= TEST_BBDEV_VF_K_POS; + turbo_enc->tb_params.k_pos = (uint16_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "c_neg")) { + vector->mask |= TEST_BBDEV_VF_C_NEG; + turbo_enc->tb_params.c_neg = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "c")) { + vector->mask |= TEST_BBDEV_VF_C; + turbo_enc->tb_params.c = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "cab")) { + vector->mask |= TEST_BBDEV_VF_CAB; + turbo_enc->tb_params.cab = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "rv_index")) { + vector->mask |= TEST_BBDEV_VF_RV_INDEX; + turbo_enc->rv_index = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "ncb")) { + vector->mask |= TEST_BBDEV_VF_NCB; + turbo_enc->cb_params.ncb = (uint16_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "ncb_neg")) { + vector->mask |= TEST_BBDEV_VF_NCB_NEG; + turbo_enc->tb_params.ncb_neg = + (uint16_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "ncb_pos")) { + vector->mask |= TEST_BBDEV_VF_NCB_POS; + turbo_enc->tb_params.ncb_pos = + (uint16_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "r")) { + vector->mask |= TEST_BBDEV_VF_R; + turbo_enc->tb_params.r = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "code_block_mode")) { + vector->mask |= TEST_BBDEV_VF_CODE_BLOCK_MODE; + turbo_enc->code_block_mode = (uint8_t) strtoul(token, &err, 0); + ret = ((err == NULL) || (*err != '\0')) ? -1 : 0; + } else if (!strcmp(key_token, "op_flags")) { + vector->mask |= TEST_BBDEV_VF_OP_FLAGS; + ret = parse_turbo_flags(token, &op_flags, + vector->op_type); + if (!ret) + turbo_enc->op_flags = op_flags; + } else if (!strcmp(key_token, "expected_status")) { + vector->mask |= TEST_BBDEV_VF_EXPECTED_STATUS; + ret = parse_expected_status(token, &status, vector->op_type); + if (!ret) + vector->expected_status = status; + } else { + printf("Not valid enc key: '%s'\n", key_token); + return -1; + } + + if (ret != 0) { + printf("Failed with convert '%s\t%s'\n", key_token, token); + return -1; + } + + return 0; +} + +/* checks the type of key and assigns data */ +static int +parse_entry(char *entry, struct test_bbdev_vector *vector) +{ + int ret = 0; + char *token, *key_token; + enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_NONE; + + if (entry == NULL) { + printf("Expected entry value\n"); + return -1; + } + + /* get key */ + token = strtok(entry, ENTRY_DELIMITER); + key_token = token; + /* get values for key */ + token = strtok(NULL, ENTRY_DELIMITER); + + if (key_token == NULL || token == NULL) { + printf("Expected 'key = values' but was '%.40s'..\n", entry); + return -1; + } + trim_space(key_token); + + /* first key_token has to specify type of operation */ + if (vector->op_type == RTE_BBDEV_OP_NONE) { + if (!strcmp(key_token, "op_type")) { + ret = op_turbo_type_strtol(token, &op_type); + if (!ret) + vector->op_type = op_type; + return (!ret) ? 0 : -1; + } + printf("First key_token (%s) does not specify op_type\n", + key_token); + return -1; + } + + /* compare keys */ + if (vector->op_type == RTE_BBDEV_OP_TURBO_DEC) { + if (parse_decoder_params(key_token, token, vector) == -1) + return -1; + } else if (vector->op_type == RTE_BBDEV_OP_TURBO_ENC) { + if (parse_encoder_params(key_token, token, vector) == -1) + return -1; + } + + return 0; +} + +static int +check_decoder_segments(struct test_bbdev_vector *vector) +{ + unsigned char i; + struct rte_bbdev_op_turbo_dec *turbo_dec = &vector->turbo_dec; + + if (vector->entries[DATA_INPUT].nb_segments == 0) + return -1; + + for (i = 0; i < vector->entries[DATA_INPUT].nb_segments; i++) + if (vector->entries[DATA_INPUT].segments[i].addr == NULL) + return -1; + + if (vector->entries[DATA_HARD_OUTPUT].nb_segments == 0) + return -1; + + for (i = 0; i < vector->entries[DATA_HARD_OUTPUT].nb_segments; + i++) + if (vector->entries[DATA_HARD_OUTPUT].segments[i].addr == NULL) + return -1; + + if ((turbo_dec->op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT) && + (vector->entries[DATA_SOFT_OUTPUT].nb_segments == 0)) + return -1; + + for (i = 0; i < vector->entries[DATA_SOFT_OUTPUT].nb_segments; + i++) + if (vector->entries[DATA_SOFT_OUTPUT].segments[i].addr == NULL) + return -1; + + return 0; +} + +static int +check_decoder_llr_spec(struct test_bbdev_vector *vector) +{ + struct rte_bbdev_op_turbo_dec *turbo_dec = &vector->turbo_dec; + + /* Check input LLR sign formalism specification */ + if ((turbo_dec->op_flags & RTE_BBDEV_TURBO_POS_LLR_1_BIT_IN) && + (turbo_dec->op_flags & + RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN)) { + printf( + "Both positive and negative LLR input flags were set!\n"); + return -1; + } + if (!(turbo_dec->op_flags & RTE_BBDEV_TURBO_POS_LLR_1_BIT_IN) && + !(turbo_dec->op_flags & + RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN)) { + printf( + "WARNING: input LLR sign formalism was not specified and will be set to negative LLR for '1' bit\n"); + turbo_dec->op_flags |= RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN; + } + + if (!(turbo_dec->op_flags & RTE_BBDEV_TURBO_SOFT_OUTPUT)) + return 0; + + /* Check output LLR sign formalism specification */ + if ((turbo_dec->op_flags & RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT) && + (turbo_dec->op_flags & + RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT)) { + printf( + "Both positive and negative LLR output flags were set!\n"); + return -1; + } + if (!(turbo_dec->op_flags & RTE_BBDEV_TURBO_POS_LLR_1_BIT_SOFT_OUT) && + !(turbo_dec->op_flags & + RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT)) { + printf( + "WARNING: soft output LLR sign formalism was not specified and will be set to negative LLR for '1' bit\n"); + turbo_dec->op_flags |= + RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT; + } + + return 0; +} + +/* checks decoder parameters */ +static int +check_decoder(struct test_bbdev_vector *vector) +{ + struct rte_bbdev_op_turbo_dec *turbo_dec = &vector->turbo_dec; + const int mask = vector->mask; + + if (check_decoder_segments(vector) < 0) + return -1; + + if (check_decoder_llr_spec(vector) < 0) + return -1; + + /* Check which params were set */ + if (!(mask & TEST_BBDEV_VF_CODE_BLOCK_MODE)) { + printf( + "WARNING: code_block_mode was not specified in vector file and will be set to 1 (0 - TB Mode, 1 - CB mode)\n"); + turbo_dec->code_block_mode = 1; + } + if (turbo_dec->code_block_mode == 0) { + if (!(mask & TEST_BBDEV_VF_EA)) + printf( + "WARNING: ea was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_EB)) + printf( + "WARNING: eb was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_K_NEG)) + printf( + "WARNING: k_neg was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_K_POS)) + printf( + "WARNING: k_pos was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_C_NEG)) + printf( + "WARNING: c_neg was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_C)) { + printf( + "WARNING: c was not specified in vector file and will be set to 1\n"); + turbo_dec->tb_params.c = 1; + } + if (!(mask & TEST_BBDEV_VF_CAB)) + printf( + "WARNING: cab was not specified in vector file and will be set to 0\n"); + } else { + if (!(mask & TEST_BBDEV_VF_E)) + printf( + "WARNING: e was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_K)) + printf( + "WARNING: k was not specified in vector file and will be set to 0\n"); + } + if (!(mask & TEST_BBDEV_VF_RV_INDEX)) + printf( + "WARNING: rv_index was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_ITER_MIN)) + printf( + "WARNING: iter_min was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_ITER_MAX)) + printf( + "WARNING: iter_max was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)) + printf( + "WARNING: expected_iter_count was not specified in vector file and iter_count will not be validated\n"); + if (!(mask & TEST_BBDEV_VF_EXT_SCALE)) + printf( + "WARNING: ext_scale was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_OP_FLAGS)) { + printf( + "WARNING: op_flags was not specified in vector file and capabilities will not be validated\n"); + turbo_dec->num_maps = 0; + } else if (!(turbo_dec->op_flags & RTE_BBDEV_TURBO_MAP_DEC) && + mask & TEST_BBDEV_VF_NUM_MAPS) { + printf( + "WARNING: RTE_BBDEV_TURBO_MAP_DEC was not set in vector file and num_maps will be set to 0\n"); + turbo_dec->num_maps = 0; + } + if (!(mask & TEST_BBDEV_VF_EXPECTED_STATUS)) + printf( + "WARNING: expected_status was not specified in vector file and will be set to 0\n"); + return 0; +} + +/* checks encoder parameters */ +static int +check_encoder(struct test_bbdev_vector *vector) +{ + unsigned char i; + const int mask = vector->mask; + + if (vector->entries[DATA_INPUT].nb_segments == 0) + return -1; + + for (i = 0; i < vector->entries[DATA_INPUT].nb_segments; i++) + if (vector->entries[DATA_INPUT].segments[i].addr == NULL) + return -1; + + if (vector->entries[DATA_HARD_OUTPUT].nb_segments == 0) + return -1; + + for (i = 0; i < vector->entries[DATA_HARD_OUTPUT].nb_segments; i++) + if (vector->entries[DATA_HARD_OUTPUT].segments[i].addr == NULL) + return -1; + + if (!(mask & TEST_BBDEV_VF_CODE_BLOCK_MODE)) { + printf( + "WARNING: code_block_mode was not specified in vector file and will be set to 1\n"); + vector->turbo_enc.code_block_mode = 1; + } + if (vector->turbo_enc.code_block_mode == 0) { + if (!(mask & TEST_BBDEV_VF_EA) && (vector->turbo_enc.op_flags & + RTE_BBDEV_TURBO_RATE_MATCH)) + printf( + "WARNING: ea was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_EB) && (vector->turbo_enc.op_flags & + RTE_BBDEV_TURBO_RATE_MATCH)) + printf( + "WARNING: eb was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_K_NEG)) + printf( + "WARNING: k_neg was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_K_POS)) + printf( + "WARNING: k_pos was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_C_NEG)) + printf( + "WARNING: c_neg was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_C)) { + printf( + "WARNING: c was not specified in vector file and will be set to 1\n"); + vector->turbo_enc.tb_params.c = 1; + } + if (!(mask & TEST_BBDEV_VF_CAB) && (vector->turbo_enc.op_flags & + RTE_BBDEV_TURBO_RATE_MATCH)) + printf( + "WARNING: cab was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_NCB_NEG)) + printf( + "WARNING: ncb_neg was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_NCB_POS)) + printf( + "WARNING: ncb_pos was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_R)) + printf( + "WARNING: r was not specified in vector file and will be set to 0\n"); + } else { + if (!(mask & TEST_BBDEV_VF_E) && (vector->turbo_enc.op_flags & + RTE_BBDEV_TURBO_RATE_MATCH)) + printf( + "WARNING: e was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_K)) + printf( + "WARNING: k was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_NCB)) + printf( + "WARNING: ncb was not specified in vector file and will be set to 0\n"); + } + if (!(mask & TEST_BBDEV_VF_RV_INDEX)) + printf( + "WARNING: rv_index was not specified in vector file and will be set to 0\n"); + if (!(mask & TEST_BBDEV_VF_OP_FLAGS)) + printf( + "WARNING: op_flags was not specified in vector file and capabilities will not be validated\n"); + if (!(mask & TEST_BBDEV_VF_EXPECTED_STATUS)) + printf( + "WARNING: expected_status was not specified in vector file and will be set to 0\n"); + + return 0; +} + +static int +bbdev_check_vector(struct test_bbdev_vector *vector) +{ + if (vector->op_type == RTE_BBDEV_OP_TURBO_DEC) { + if (check_decoder(vector) == -1) + return -1; + } else if (vector->op_type == RTE_BBDEV_OP_TURBO_ENC) { + if (check_encoder(vector) == -1) + return -1; + } else if (vector->op_type != RTE_BBDEV_OP_NONE) { + printf("Vector was not filled\n"); + return -1; + } + + return 0; +} + +int +test_bbdev_vector_read(const char *filename, + struct test_bbdev_vector *vector) +{ + int ret = 0; + size_t len = 0; + + FILE *fp = NULL; + char *line = NULL; + char *entry = NULL; + + fp = fopen(filename, "r"); + if (fp == NULL) { + printf("File %s does not exist\n", filename); + return -1; + } + + while (getline(&line, &len, fp) != -1) { + + /* ignore comments and new lines */ + if (line[0] == '#' || line[0] == '/' || line[0] == '\n' + || line[0] == '\r') + continue; + + trim_space(line); + + /* buffer for multiline */ + entry = realloc(entry, strlen(line) + 1); + if (entry == NULL) { + printf("Fail to realloc %zu bytes\n", strlen(line) + 1); + ret = -ENOMEM; + goto exit; + } + + strcpy(entry, line); + + /* check if entry ends with , or = */ + if (entry[strlen(entry) - 1] == ',' + || entry[strlen(entry) - 1] == '=') { + while (getline(&line, &len, fp) != -1) { + trim_space(line); + + /* extend entry about length of new line */ + char *entry_extended = realloc(entry, + strlen(line) + + strlen(entry) + 1); + + if (entry_extended == NULL) { + printf("Fail to allocate %zu bytes\n", + strlen(line) + + strlen(entry) + 1); + ret = -ENOMEM; + goto exit; + } + + entry = entry_extended; + /* entry has been allocated accordingly */ + strcpy(&entry[strlen(entry)], line); + + if (entry[strlen(entry) - 1] != ',') + break; + } + } + ret = parse_entry(entry, vector); + if (ret != 0) { + printf("An error occurred while parsing!\n"); + goto exit; + } + } + ret = bbdev_check_vector(vector); + if (ret != 0) + printf("An error occurred while checking!\n"); + +exit: + fclose(fp); + free(line); + free(entry); + + return ret; +} diff --git a/dpdk/app/test-bbdev/test_bbdev_vector.h b/dpdk/app/test-bbdev/test_bbdev_vector.h new file mode 100644 index 00000000..476aae13 --- /dev/null +++ b/dpdk/app/test-bbdev/test_bbdev_vector.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation + */ + +#ifndef TEST_BBDEV_VECTOR_H_ +#define TEST_BBDEV_VECTOR_H_ + +#include + +/* Flags which are set when specific parameter is define in vector file */ +enum { + TEST_BBDEV_VF_E = (1ULL << 0), + TEST_BBDEV_VF_EA = (1ULL << 1), + TEST_BBDEV_VF_EB = (1ULL << 2), + TEST_BBDEV_VF_K = (1ULL << 3), + TEST_BBDEV_VF_K_NEG = (1ULL << 4), + TEST_BBDEV_VF_K_POS = (1ULL << 5), + TEST_BBDEV_VF_C_NEG = (1ULL << 6), + TEST_BBDEV_VF_C = (1ULL << 7), + TEST_BBDEV_VF_CAB = (1ULL << 8), + TEST_BBDEV_VF_RV_INDEX = (1ULL << 9), + TEST_BBDEV_VF_ITER_MAX = (1ULL << 10), + TEST_BBDEV_VF_ITER_MIN = (1ULL << 11), + TEST_BBDEV_VF_EXPECTED_ITER_COUNT = (1ULL << 12), + TEST_BBDEV_VF_EXT_SCALE = (1ULL << 13), + TEST_BBDEV_VF_NUM_MAPS = (1ULL << 14), + TEST_BBDEV_VF_NCB = (1ULL << 15), + TEST_BBDEV_VF_NCB_NEG = (1ULL << 16), + TEST_BBDEV_VF_NCB_POS = (1ULL << 17), + TEST_BBDEV_VF_R = (1ULL << 18), + TEST_BBDEV_VF_CODE_BLOCK_MODE = (1ULL << 19), + TEST_BBDEV_VF_OP_FLAGS = (1ULL << 20), + TEST_BBDEV_VF_EXPECTED_STATUS = (1ULL << 21), +}; + +enum op_data_type { + DATA_INPUT = 0, + DATA_SOFT_OUTPUT, + DATA_HARD_OUTPUT, + DATA_NUM_TYPES, +}; + +struct op_data_buf { + uint32_t *addr; + uint32_t length; +}; + +struct op_data_entries { + struct op_data_buf segments[RTE_BBDEV_MAX_CODE_BLOCKS]; + unsigned int nb_segments; +}; + +struct test_bbdev_vector { + enum rte_bbdev_op_type op_type; + int expected_status; + int mask; + union { + struct rte_bbdev_op_turbo_dec turbo_dec; + struct rte_bbdev_op_turbo_enc turbo_enc; + }; + /* Additional storage for op data entries */ + struct op_data_entries entries[DATA_NUM_TYPES]; +}; + +/* fills test vector parameters based on test file */ +int +test_bbdev_vector_read(const char *filename, + struct test_bbdev_vector *vector); + + +#endif /* TEST_BBDEV_VECTOR_H_ */ diff --git a/dpdk/app/test-bbdev/test_vectors/bbdev_null.data b/dpdk/app/test-bbdev/test_vectors/bbdev_null.data new file mode 100644 index 00000000..c9a9abe9 --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/bbdev_null.data @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_NONE \ No newline at end of file diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k40_r0_e17280_sbd_negllr.data b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k40_r0_e17280_sbd_negllr.data new file mode 100644 index 00000000..d98210ff --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k40_r0_e17280_sbd_negllr.data @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_DEC + +input0 = +0x7f007f00, 0x7f817f00, 0x767f8100, 0x817f8100, 0x81008100, 0x7f818100, 0x81817f00, 0x7f818100, +0x86007f00, 0x7f818100, 0x887f8100, 0x81815200, 0x81008100, 0x817f7f00, 0x7f7f8100, 0x9e817f00, +0x7f7f0000, 0xb97f0000, 0xa7810000, 0x7f7f4a7f, 0x7f810000, 0x7f7f7f7f, 0x81720000, 0x40658181, +0x84810000, 0x817f0000, 0x81810000, 0x7f818181, 0x7f810000, 0x81815a81, 0x817f0000, 0x7a867f7b, +0x817f0000, 0x6b7f0000, 0x7f810000, 0x81818181, 0x817f0000, 0x7f7f817f, 0x7f7f0000, 0xab7f4f7f, +0x817f0000, 0x817f6c00, 0x81810000, 0x817f8181, 0x7f810000, 0x81816981, 0x7f7f0000, 0x007f8181 + +hard_output0 = +0xa7d6732e, 0x61 + +soft_output0 = +0x7f7f7f7f, 0x81817f7f, 0x7f817f81, 0x817f7f81, 0x81817f81, 0x81817f81, 0x8181817f, 0x7f81817f, +0x7f81817f, 0x7f817f7f, 0x81817f7f, 0x817f8181, 0x81818181, 0x817f7f7f, 0x7f818181, 0x817f817f, +0x81818181, 0x81817f7f, 0x7f817f81, 0x7f81817f, 0x817f7f7f, 0x817f7f7f, 0x7f81817f, 0x817f817f, +0x81817f7f, 0x81817f7f, 0x81817f7f, 0x7f817f7f, 0x817f7f81, 0x7f7f8181, 0x81817f81, 0x817f7f7f, +0x7f7f8181 + +e = +17280 + +k = +40 + +rv_index = +1 + +iter_max = +8 + +iter_min = +4 + +expected_iter_count = +8 + +ext_scale = +15 + +num_maps = +0 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_SOFT_OUTPUT, RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE, +RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN, RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e10376_crc24b_sbd_negllr_high_snr.data b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e10376_crc24b_sbd_negllr_high_snr.data new file mode 100644 index 00000000..3472c992 --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e10376_crc24b_sbd_negllr_high_snr.data @@ -0,0 +1,643 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_DEC + +input0 = +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0xCE000000, 0x2F00F6D9, 0xCC1AF942, 0x22F8F4E1, 0xBED8FAFF, 0x43DC19B5, 0x35E91CC0, 0x5B070D30, +0xF9DACDFD, 0x170121DA, 0x012AEF53, 0x39E7D90E, 0xD5FBEFDD, 0xCCF0FDC8, 0xD62B0D54, 0xEAE601BE, +0xE5001328, 0x0CF00DC7, 0x3EE0E4B8, 0xC2D8EB00, 0xBE1EEBF7, 0xCDF5E7E4, 0x4FDFF4FA, 0xF402DA2A, +0xFEFEE32B, 0xD62CDA55, 0x17DE0106, 0xF92411B5, 0x301B2142, 0xE50C09E3, 0x51030EDB, 0x48DB2A4C, +0x03E420BC, 0x1AE1DCF8, 0xB6EE0D3A, 0xC2FADF22, 0xBF211508, 0x3228E8AF, 0xE0170BEE, 0x26F8091F, +0xDBFA02D2, 0xC810FE19, 0xE4DE1005, 0x18FF0D29, 0xE723F04B, 0xFDF2F1E7, 0xCF05DBD3, 0x0412F616, +0x00DC234C, 0x1501EDD7, 0xD9EB4FED, 0x1C03BC2B, 0xE2FDF6D4, 0xE9EC10C4, 0x2426034D, 0xEDE33B45, +0x05DC2304, 0xEDE81411, 0x11E416BD, 0x03D1DAA8, 0x04F72431, 0x30F15718, 0xF3F33534, 0xED043AD3, +0x1CE8BC11, 0xEDE1EBF7, 0x242C0454, 0x0118D740, 0x07F3D01B, 0x0205D623, 0x15233E05, 0xF2EBE5ED, +0xED2D3BAC, 0xEBF83CD1, 0xFCF9D421, 0x0B0934E0, 0x0A09E1E0, 0x1AEB0EC4, 0xF8FEDF26, 0xFCE8D53F, +0x20E10846, 0xE327F5B1, 0x18E340F5, 0xF71ED0BB, 0x0EF5CBE3, 0xF4F11C36, 0x141BC343, 0xF4E7E5F1, +0x30205809, 0x10EAC713, 0x22DBB603, 0x250AFC1E, 0xE4F3BB1A, 0x2316FA12, 0xDEE105F7, 0x12FDC7D4, +0xFC212507, 0xDB290300, 0x1500ED00, 0x1EABBA27, 0xF14ECA2D, 0xF2D93625, 0x23F20502, 0xF0B9C8E6, +0xEAD83EE1, 0x0C331CFF, 0x23D205F5, 0x0EC61906, 0xF128C9EF, 0xEF3CC800, 0x0B9DCDED, 0xF3C2CC3C, +0x23EB04E9, 0x0B51CD13, 0xEADEC3D7, 0x222D06FB, 0xF0331904, 0x16C93F0C, 0xDAE5FE0F, 0x1F14B80C, +0x001B28ED, 0x3F3B67F3, 0xEEEDE9EC, 0x1CE94514, 0x1D4145EF, 0xD8BA5019, 0xFC06D41E, 0x17BDC0DF, +0x16D712E4, 0xE8041001, 0xEC2AEDDB, 0x11BE3803, 0xEBD2ECE5, 0xEF5A17F9, 0x2BFF52CD, 0xE958C126, +0xEE1A1531, 0x0EBF1BF2, 0x18BC401A, 0x2AC2521C, 0xEE4817EA, 0x07482E20, 0x141314E0, 0xF41DCD15, +0xDB10B40C, 0xC23700E9, 0x4221EB10, 0x4BE7E607, 0xEEF9DC0E, 0x62AC16DF, 0xE2E1C5D3, 0x27E4F70A, +0x3213020D, 0x4ABE0A14, 0xACC1DE1B, 0xD636D416, 0xDFAF020E, 0xCE230828, 0x2615F6FB, 0xEA1002EE, +0xD1351117, 0xB71C07F3, 0x03C9DF0C, 0x2B52DB0F, 0xD9AB02D7, 0xD20702D3, 0x48EEFB21, 0x3AE5E116, +0xE728EE0E, 0x33C50F00, 0x363EF5EC, 0xD4FBF217, 0xB00FFDDE, 0x2333281A, 0x3624040C, 0x02DB0E04, +0x3FDE27FD, 0x173716FB, 0xF20BEFF1, 0x3EF0191E, 0xB6B01719, 0x1BC7DD27, 0xE80AF3EE, 0x20BE101E, +0xF42EF8E6, 0x40531BF9, 0x4646172B, 0x44DA1E1D, 0xC7F5E503, 0xF3E8F0E4, 0xA4E2E5EF, 0x42B935F6, +0xEB1D1A1E, 0xE500ED0B, 0x2909F2CF, 0xCE0E0037, 0xD1320AA6, 0xEB01F9D9, 0x45E7ECC0, 0x4CFEE3D7, +0x0EE024B8, 0xD3F31B1A, 0xE846056E, 0xF522EFFB, 0x5F1AE443, 0xCAF93620, 0x2CE9F23E, 0xDDE604F2, +0x3BDDFC04, 0xC8EBEE13, 0xC11FF0F6, 0x18EBE9EC, 0xFAEEF1C7, 0xB6EFDDEA, 0xF8E123F7, 0x4EF4E1CC, +0x430C271C, 0x3BE01CB8, 0xABE813EF, 0xCAEED439, 0xC1FAF3D1, 0x4E2F1656, 0x11E4DB0C, 0xDAF816DF, +0x44E3FE0C, 0x2B1D1C0B, 0x3300FD28, 0xC320F4F9, 0x2A1DEB0A, 0xEA2502B4, 0x41061322, 0xC81518C2, +0x1F2510FD, 0xD1E1F7F7, 0xBD1906F1, 0xF3C9E55F, 0xB3D91A4F, 0x0D0D26CC, 0xBD071B20, 0x41141BC4, +0x2A1B19F2, 0x4902FDD5, 0xD6F9DE2E, 0x19190341, 0x09E80F11, 0x31FC1E2C, 0xEC1DF8F4, 0xCC0B14E2, +0x51DFF3B7, 0x1AF028C8, 0x0F0E0E37, 0x2B041A25, 0x1A03FD25, 0xC5030FD5, 0xBCFAED22, 0x9A1F1CB9, +0xCCE8C2F0, 0xEF1BF4BD, 0x21EC163C, 0xBAED07C6, 0x03F7E3D0, 0xC503DB2A, 0xC20C13E4, 0xD4F8EA30, +0x391B0544, 0x59F3EECB, 0xAD0A311D, 0xFE0C2C33, 0x250CDBE4, 0xC5E80341, 0x3B0914CE, 0xC0F2ED36, +0xC204182B, 0x3B0C16E5, 0xFFFF1429, 0xC316D8C3, 0xE408EB1F, 0x3713F4C4, 0xE4F2F01A, 0x0A280C01, +0xEC0FE318, 0xCB16153E, 0x5C04F2D4, 0x281F35F8, 0xE4D80051, 0x4E0A0CE3, 0xB107DB21, 0xD1DED9FB, +0x380AFACE, 0x50CC105C, 0xD9D80100, 0xF40ECCE6, 0xDADC03B4, 0x0AE5E243, 0x0C22E4FA, 0x08CD1F5A, +0xE2D1F7A9, 0x12CE39A5, 0xFFDDD7FB, 0x032C2BAC, 0xF1D9E700, 0x14003B28, 0xFC08D420, 0x08093131, +0xFEFFD7D7, 0xBB219348, 0x07DC2E4C, 0xDF0DB8CB, 0x22EB06EC, 0x0BE034F8, 0xF10FC819, 0x1623EEFA, +0x0D1DE6BA, 0xDA1AB142, 0x1C16F4EE, 0x27F500E2, 0x1137C85E, 0x1C0BF434, 0xF004C924, 0x02EBD9ED, +0x1BF943D2, 0xE7F80F30, 0xEFFE39D9, 0x2CFA5323, 0x04242DFB, 0xF420E347, 0x15F43C33, 0xE9F41134, +0x20EBB8C2, 0x0205D523, 0xECE03DB8, 0x10F3E935, 0x180BC11E, 0x12E2C7BA, 0xE70B4233, 0xEED8EAB0, +0x110E3936, 0xFEE327F5, 0x12E60017, 0xFE4026F3, 0xED26C518, 0xE5E9F2FF, 0x200C08EE, 0xEF1439E4, +0xE92010ED, 0xE1B80908, 0x18584021, 0x24FBFC31, 0x00AE2823, 0x040CDC2A, 0xDE34FBE5, 0x1439C30D, +0x0DB7E6EF, 0xE6090F21, 0xECAF3C20, 0x10DFE8D8, 0x0CC0CCF9, 0x15061318, 0x043723DD, 0xEA3A3DF0, +0xEC18C413, 0xFE2BD611, 0xE3BC45FE, 0x16FCC11D, 0x120E39DC, 0xEC00141B, 0x13F03B27, 0x24D2FC18, +0x0917E1FB, 0x113BEAEE, 0xE9FE11EE, 0x2C505427, 0x0D0B3529, 0x15E73E1E, 0x0FD91AF1, 0xDFF8B6FF, +0xF413CCE1, 0x1EEFB9EA, 0xDCD04C17, 0xCEC25AF8, 0x1AB2BE17, 0xD8BEAFDA, 0xE6CEBD19, 0xFCC42CF6, +0xDE19FA15, 0x250203F2, 0xA90042DB, 0x242A2F1A, 0x004D0401, 0x0B4C2825, 0xACB61DDD, 0x07422CDE, +0xCDFEDFE7, 0xAFF8F527, 0x35052820, 0x0B34F2DD, 0xD648E2F3, 0x3DDF02E0, 0xFE42EBF8, 0xC1F6251A, +0xBA0DE9E2, 0x442FE2E5, 0xD014E308, 0x2DB409EC, 0x50F8FADC, 0x264BD8E0, 0x2E0FFEDC, 0x1E2D051A, +0x0FD20AFB, 0xCE1CE7FA, 0x35CDF6F4, 0xFC49F3F4, 0x642FDCDF, 0x3625C507, 0xDF140FFC, 0xDB21F913, +0xC7D5FDF9, 0xCEE112FE, 0xD1D4F7F8, 0x5D410704, 0xD914CC19, 0xCCF800EC, 0xE3E00C1F, 0xCD05F408, +0xC952F4DD, 0x444EF02A, 0x1E15E526, 0xD73909ED, 0xFA1900EF, 0xFFB6210F, 0x06EB27DE, 0x0B5BDF13, +0x1B4F1D33, 0xCDF5F4D9, 0x00F40C1D, 0xC6031CDC, 0xF91C13F4, 0x061321C6, 0xBFDD2206, 0xEE13193C, +0xFF1C17BB, 0xA91E27F5, 0x25ECD13C, 0x0CF90321, 0xBA1FE5B8, 0x50151F3D, 0xE935285D, 0x330A10CE, +0x3EE6F542, 0xE80AEA1E, 0x45251103, 0xC014E3EC, 0x2D2C1854, 0x9F0E0536, 0x2E14C7C3, 0xECE50642, +0xE6E214F7, 0x1021F208, 0xBCDC19FB, 0x4B281BFF, 0x0CDFDEB7, 0x21E91CEE, 0xC22FF856, 0xA2F3EAE4, +0x5810CB18, 0xE416D03E, 0x25E3F40C, 0xF4180340, 0xFE0EE4C9, 0x433C2763, 0x38F11BE7, 0xE5FC11D4, +0x1E11F4C6, 0x491D0ABB, 0xC0E72140, 0xB226E84F, 0x122E26AB, 0xEAF116E7, 0xD5E4EE0B, 0xF4FB0323, +0xDCE61D42, 0x05F1FCE7, 0x0DDCDDB4, 0xF9E42000, 0x181610EE, 0xE1F246CA, 0xEDEDC53B, 0x14DEC5B6, +0x1E02F629, 0xDEE749F0, 0x1C02BDD7, 0x1EF3F5CB, 0x11E8C640, 0xFA272301, 0xF3FC352C, 0xFDD52553, +0x09191FF1, 0xDB11FEE9, 0xF7EB1FEE, 0xFFF9292E, 0x0A2032B8, 0x0C1BCDBC, 0xDDFFFBD9, 0x1BEABDEF, +0x19F84130, 0xECE5ECF3, 0x271302EB, 0xE7EB423D, 0x1E2146B7, 0x14E13CF7, 0x16FE112B, 0xEB211308, +0xDAD802B0, 0x0017D811, 0xFE26D901, 0x0D0ACBE2, 0xF2ED1A3B, 0xFBDED206, 0x18F0F1C8, 0xD1EFA9C6, +0xE5E1BDF7, 0x1514C314, 0xF9FCD1D3, 0x02EB2BED, 0x0EEB1BC2, 0x2427B5FF, 0x0B2233FA, 0xFE0FDBE8, +0xF7DDCE04, 0x03F42BE3, 0x0525D202, 0xF9B20021, 0xEF0FE926, 0xF505E318, 0xF7DF1F23, 0xEADF12F8, +0xF12C3807, 0xDF01FA04, 0x1D23F426, 0xF1231A05, 0x20F549FB, 0xE0F0F9E3, 0xDB67B418, 0xC6E5623F, +0x212AF90D, 0xE6C7BD02, 0x24CA4C11, 0xE249F6F1, 0xE8383F21, 0xFC212310, 0xF7561F07, 0x25C4032E, +0x070EDE14, 0x1437151A, 0xE93C4010, 0xF10CE615, 0xDB0D4D1C, 0xFAF6221B, 0xF5F3E21E, 0xE2470B1B, +0xE71BC01F, 0x07B0220C, 0xF311CAD8, 0xFF052816, 0x1DC20BDC, 0xE7BB40EB, 0xF5DEE21E, 0x013527FB, +0x0FF519F2, 0x1AE9BDE4, 0x2B0F53F0, 0x13C6C5E8, 0xF12636ED, 0x10BD38FE, 0xDD0606E4, 0xDA5C4FDE, +0x1329C5CC, 0x0DC8CA00, 0xD33956F0, 0x0700D711, 0x31B1DE02, 0x210C09D9, 0x0C2EF81B, 0xF5FFE406, +0x27C1E2DA, 0xCB1801E8, 0x38C70EF0, 0x0D1EF0EF, 0x142E1C0B, 0x0B00EB06, 0x5D441DD8, 0x1E4035E5, +0xFD3C0AE8, 0x3732DCEC, 0xC8B5F00A, 0xB9BF1023, 0x0AF22018, 0xC6DDE21A, 0x18EF1306, 0x1AC8F1E9, +0x1F020EF0, 0xEA0209DB, 0x41CFEEDA, 0x21E31AF8, 0x4BBD070A, 0xEECFDD1A, 0xD4DB15F8, 0x430FFC02, +0x15361A18, 0xCF17ECF1, 0xE42009EF, 0xC21CF4F8, 0x3DDFEAF3, 0x3E0916F9, 0xF028151F, 0x03CA1700, +0x983B26F1, 0xDE2AC112, 0x209FF9FD, 0x1935F7C6, 0xED18F1F3, 0xBFD0EA0F, 0xC5071908, 0x1F4E13DE, +0x17F00926, 0xF80DEE19, 0x4DF01FE6, 0xB7D2DB18, 0x0AF5E0F9, 0x4043E21D, 0x4AD517E5, 0xB0CFDE03, +0xFB22D90A, 0xDF5023F9, 0xDABDF828, 0x1911011A, 0xBDF30FE8, 0xCEAA1A1A, 0xB0D2F62D, 0x3AAC28F9, +0x42D6ED2D, 0xA83D1B02, 0x1831D1EA, 0xF11D10F7, 0x1D201AF5, 0x394F0C07, 0xB327F0D9, 0x3254DA01, +0xEBF40AD5, 0x26A413E4, 0x5113FECC, 0xD4F32915, 0xDDB3FD1A, 0xC0D2FB25, 0x02DEE7FB, 0xC33DDB05, +0x01021615, 0x0B1D27DA, 0xCBC2E2F6, 0x193CF3E9, 0x27E2F1EC, 0x5A1BFF0A, 0xE8CF320D, 0xBD4DF0F8, +0x0F291A26, 0x1817E7FF, 0xEFCB10F0, 0xD40CE9F3, 0x0606041B, 0xFABC2322, 0xE0E9DEE5, 0x1BD80710, +0xC015F301, 0x4718E8EE, 0x69D81E11, 0x19F6BF00, 0xF60F1E00, 0x3002E225, 0x29EA08C1, 0xF4EA00EF, +0x3910E5E7, 0xDB0B12E4, 0x2A3E0265, 0x3BE8FE11, 0xBBEFEEE9, 0x151BE2BE, 0x1E2314FB, 0x4C2DF754, +0xD5F823E0, 0x10200247, 0x01DB194D, 0x1B17D93F, 0x5806F42E, 0xFA19D040, 0x040BDF1D, 0xFCCB25A4, +0x39D9DC00, 0x2C0EEFE7, 0x20F3FDCB, 0xC2E70841, 0x3EEFEAE9, 0x32EDEAC5, 0xAC1BF70D, 0x0C15D4C3, +0x4CE7E5BF, 0x26F024E9, 0x061AFE0D, 0xDE13DEC5, 0x09DE0706, 0xFC0C1FE5, 0xD41CDC0B, 0xCC1DFBBB, +0xC6020CDB, 0xCD16133E, 0x3FF30CCC, 0x0B2D16AB, 0x3800E3D9, 0x3AFCF0DC, 0xE4FFED27, 0xFCF9F3E0, +0x05FF24D9, 0xE1F9DE2F, 0x391908F2, 0xC8F211CB, 0xEAEF003D, 0xDC1C040D, 0x25F74ECF, 0xE41F0CBA, +0xDF1708EE, 0xE016F83F, 0x1D124515, 0xF325344D, 0xE114F63C, 0x261FFEB9, 0x02F02739, 0xF12018F8, +0xD9124F3A, 0x1E1A45F3, 0x1504142C, 0xDDF506E3, 0xEAEAC113, 0xDE1AFB42, 0xF113C93B, 0x1721C207, +0xE3D4BB54, 0x070A2FCE, 0x0C0AE41D, 0xF90D3035, 0xE6DCF2B4, 0xFB15D4C3, 0xFE2425FC, 0x03002B28, +0xE50AF433, 0x07EC2FC4, 0x07E921EF, 0xFE09DAE1, 0xE21EF6B9, 0xF3E6360D, 0x17EE3F39, 0xE7FA0E22, +0xF5151DED, 0x1EF945DE, 0x0D1236EB, 0x0AFD32D6, 0x0928CFFF, 0xDCDAFD02, 0xFB272D00, 0xCE025B25, +0xEA0D3ECB, 0xE3E2BBF5, 0x06D32354, 0xF4DDCCB6, 0xE4002245, 0x213AB706, 0xECCCC3ED, 0x3CED9C0B, +0x15D23C14, 0x0BE7E3F9, 0x1FE70A0E, 0xF30D350F, 0x1928F01B, 0xD50AAE01, 0x120AC6E1, 0x0ED4C91E, +0xFFF7D804, 0x0D0CCA1F, 0x0433DBE4, 0xEA0DEFF5, 0x36FB5E1C, 0xEB11ECDC, 0x23C0FB17, 0xEBC63C17, +0x272800EF, 0xFC34D4FF, 0x0AB8E2F5, 0x01FB2AE0, 0x090630DD, 0xFE0AD522, 0x1B10BD1D, 0xDF520719, +0xF12436D5, 0x1D20BAFC, 0xE442BCF8, 0x05E0DDE6, 0x183FF1F8, 0xF816E118, 0xE3C9F4EE, 0xF2F51AF0, +0xFE5126E4, 0x1E31F7D7, 0xF9362F09, 0x15E43CF2, 0x0D3EE50C, 0xF74931EA, 0x23EEB621, 0x02B9DAE9, +0xDFE5B71F, 0xFD46D50D, 0x103E181F, 0xDF580717, 0x00C3EC31, 0x0CF714EC, 0x4ACCE41E, 0xF2E0DEF4, +0xB0431A08, 0xBD38D8E6, 0xEED6E510, 0x41B8EA02, 0xEC36E61F, 0xBEE9EBF3, 0xBE28E712, 0xF6051BFF, +0x2FE5E222, 0x2ADCFAF2, 0x3834FF04, 0x2AD1100D, 0xF4C502F8, 0x4DE7E414, 0x07A5DA10, 0xD1A92133, +0xB6ACFAD2, 0xCF51222C, 0xCD1BF8D7, 0x3F16F50E, 0xEEF4EA12, 0xA52F161D, 0xC4B3CE07, 0xB622EDDC, +0xDE2ADE06, 0x4FC8FB02, 0x6EDED9F1, 0xFC19B9F9, 0xD5B4DDF1, 0xDFC30423, 0xDFFBF9EA, 0x012C07DD, +0x12AF2805, 0x1A2EEAD6, 0xF8A1F207, 0xC0BA21CA, 0xFFDCE9E3, 0x36BFDA03, 0x05DD0E19, 0xF905DDFB, +0x32AEDF23, 0xFDFB09D5, 0x45D5DA23, 0x4B3E1CFD, 0x0905DDEA, 0xDFE2DE00, 0xB1FB07DD, 0x33EA2813, +0x02ECF5C3, 0xED0CD91B, 0x0D1BEABD, 0x4A17E5C0, 0xF707DFDF, 0xEE191F41, 0x19EC15C5, 0x4112F1EB, +0xD4F01AE9, 0xBF21FC06, 0x19DCE605, 0x2F0C10E4, 0x181FF947, 0xC00AEF1E, 0xF72EE8AB, 0x42F8E2E1, +0xA3EFE7C7, 0x31133514, 0x1EF608E2, 0x0DECF6EC, 0x0CECE5C5, 0x3DD31D54, 0xECE21545, 0xAE0CECE5, +0x49202B08, 0x0EFE2026, 0x13F41B35, 0x1B0BEB32, 0x1719F30F, 0xECDAF0FF, 0xECE9ECC0, 0xF212EC3B, +0x3DE8E6EF, 0x13F2EB1B, 0xB008EBE0, 0xBAE22845, 0x1A111DEA, 0xDBFC0EDD, 0x3F29FE50, 0xCDF817E0, +0x4B030CD5, 0xEADCDE05, 0xC31312C6, 0x272EEAA9, 0x340201DA, 0x17F400EF, 0xED1F1608, 0x24D24B55, +0xF4091DCE, 0xF51633C2, 0x03D32B56, 0x2325FA4D, 0xDEDA05FD, 0x01F72831, 0x04F12C19, 0x2716B1C3, +0x1B19F3C0, 0x10E2C909, 0x050C2233, 0x03F22519, 0x1C06F422, 0xE7D70EAF, 0x03D725AF, 0x13EC14EB, +0xF6F631E3, 0xF900DE28, 0x1FE2F7F6, 0xFBE9D2C1, 0xFBEFDDE9, 0xE110F7C8, 0x1C290C50, 0x27D9FFFF, +0x07E421BB, 0x3BE89CF0, 0xF50733D0, 0x2514B2EC, 0xF5EB1DED, 0x0619D241, 0x1FE8B840, 0xEEEA3B3E, +0xED1F1648, 0xFF15D73E, 0x0123D805, 0x0310D5E8, 0x010ED819, 0xECE53B43, 0x03E1D547, 0x01D9D6FF, +0xDFE3B745, 0x1C14443C, 0xDF0908E1, 0x03FE2ADA, 0x1306C5D2, 0xDD001004, 0x1F3AF818, 0x1A3BF3ED, +0x1EBC47EE, 0xF83E1F1B, 0xE21445EA, 0x2101B713, 0xE81AF126, 0x1930BFF3, 0x1533C408, 0xEDBFC60C, +0x060F2EE8, 0x06BDD11A, 0x04F524E4, 0xD94C001E, 0x0334DCDD, 0x06F1D30C, 0xFBDBD219, 0x0EBE1A03, +0x08D230E5, 0x250B4DFA, 0x0903311C, 0x14233C25, 0x1ABAF205, 0x30C5581D, 0x02FAD712, 0xDF2108DE, +0x02DDDB07, 0x1D47F606, 0x0EEF191F, 0x1B3AF2E9, 0xF9E52E12, 0xF0B1C8F4, 0x290CAF28, 0x0013D8E5, +0xE1B50AEC, 0x1A224322, 0xFDE8DB06, 0xF8A531EF, 0x0DD1CCCD, 0xE81AC007, 0x201048F3, 0x1800F1E8, +0x000BD8D9, 0xEC2A13E2, 0xF3EE3502, 0x282F0117, 0x08F8E006, 0xFE542620, 0x19E9C0D3, 0xDD140412, +0x1119C6EB, 0xF6C2E10E, 0x1EC50BEB, 0x15BD13ED, 0x143DC51B, 0xE5F2BE14, 0xF552E41A, 0xE10FF62A, +0x2A0B5218, 0xE33A0CE2, 0x02D7D612, 0xE1A7B800, 0xD8C10031, 0x0810E0E9, 0xD93AB0E8, 0xF8113011, +0x15B73C17, 0xFE46D520, 0xF6C51EE3, 0xD51A52ED, 0x1A0ABD0E, 0xFBF8DEE2, 0xE245BA20, 0x2225FB1E, +0xEC3A3CFD, 0x19340EEE, 0x08FE20F4, 0x1F154727, 0x05F8D213, 0x16F2EEE1, 0x22B6FA1A, 0xF404CCDF, +0xE22E4624, 0xFB3FDDFB, 0xE6F7BF17, 0xF0B6E71F, 0xF013E723, 0x17081114, 0x24EBFCE0, 0xFBE52DEC, +0xE9C310F3, 0x26F04E15, 0xF145E8E8, 0x122B17E2, 0x0D40CCFD, 0x43CA96E8, 0x421D0F00, 0x27DCE70B, +0x233B00FC, 0xCC44FAEE, 0xE0BC0BE3, 0x1BE7071C, 0x01C7F20F, 0xB29B28F0, 0xFB02DBC4, 0x32092326, +0xE648F5E2, 0x4707F2DF, 0xDF36E2E0, 0x1CC308F2, 0xBBA50DEB, 0x1DE71D34, 0xCE36F5F1, 0xCDF90AF2, +0x0FDE0CE0, 0xCF0C1906, 0xE0E509E4, 0xBC37F9F3, 0xDA06E510, 0x1FD3FDDF, 0xDB09F705, 0xF445031E, +0x24311CE2, 0x55EEFDF8, 0x3CEDD216, 0xAFF01516, 0xCA3D29E8, 0xCCAEF1EA, 0xE0F60CD5, 0x4CECF7E2, +0xDCD5DBEB, 0xAEC2FC02, 0x4825D7EA, 0x252D2004, 0x24F604FB, 0xD9D7051F, 0x1AF201FF, 0x3DE70EE6, +0x43BCEC0F, 0x43D8E5E4, 0x20EB1BFF, 0x2AAFF814, 0x021A02D8, 0xD80C26F2, 0xDB00001C, 0x2BF2FCE6, +0x181F0347, 0xAFE8F0F1, 0xE31D2A0B, 0xD015F4C2, 0xEF02072B, 0x361717EF, 0xFA25F2B3, 0xEEFADE22, +0xEE21EBF8, 0xD6F5EA1D, 0xBF160212, 0x03E6190F, 0x452126B7, 0x32D91DFF, 0x2D000A28, 0x25F205CA, +0xDEED043B, 0x36DCFA04, 0x13EAF112, 0x141915F1, 0xF2F0EC38, 0xBC20E6F8, 0x0BF2E519, 0xD0F11D19, +0xC7F6F932, 0xBFE2EFB9, 0xA4F919DF, 0xFFEACCEE, 0x3BDBDAFC, 0x2B1E1245, 0x5CEBFC14, 0xDB10CD18, +0xB718FE40, 0x0BE92012, 0x2414E2C5, 0x3B09FCE0, 0x17DF1306, 0x212F12A9, 0xC71407C4, 0x0901EFD7, +0x22EAE23F, 0xE7E805C1, 0xC6F21037, 0x4A111218, 0x1AEC22EC, 0x27E30EF6, 0x0C000134, 0x1BF10D37, +0xFD1F2A09, 0x1AEFF3E9, 0x160F3EC8, 0xEC173B3F, 0xE8F711E1, 0x050A22CD, 0x15263DB1, 0x07012127, +0x1D22BBB7, 0x34E3A5BB, 0x19210FB8, 0x17D41254, 0xD822B1FA, 0xE6DE0E06, 0x1519ED40, 0xDDE54CBE, +0x1EFC0BD4, 0xFE0D2A35, 0xE8EF40C7, 0x09E5E1F3, 0xF60631DE, 0xD6EDADC5, 0xE61A0FBE, 0x091C30BC, +0x0514DE3C, 0x1D1A45BF, 0x03E0DC08, 0xF4FE1C25, 0x1D260BFF, 0xF6FD3126, 0x1013C8EA, 0xFE18DB3F, +0xDF114916, 0x1806BF2D, 0x0E0535DC, 0xE5ED0DEB, 0x1EE445F5, 0x16DC12B4, 0x23FA4B2F, 0xFFD6DAAF, +0xECF714E2, 0x273D0066, 0x1B15F313, 0x1D0CF533, 0x17003E28, 0x1FD3F654, 0x00EFB117, 0x1C13BBD9, +0xF5C21D15, 0xF1CF3715, 0x244A0309, 0x09321FDD, 0xEFF81609, 0xE1ED091F, 0xFDFA2A16, 0xF742D022, +0xDCECB51B, 0x16FAEE14, 0x140C3C21, 0xEA0AED1B, 0x24E94CE2, 0x17E03F12, 0x1E01BAF9, 0xE82E4128, +0xEE311506, 0xF7FEE1F6, 0x0661DFD9, 0xDFB1FAC8, 0xFF282928, 0xEA3412FF, 0x104FC80C, 0x00ABD927, +0xEE32EA2D, 0x153B3D09, 0xFCB82DEC, 0xE9E8C021, 0x09BECEF0, 0xFCCA2D19, 0xDF09F90F, 0x013F2AE1, +0xFC44D4E9, 0xFD24D6E5, 0x1C0CF3FB, 0x2601021C, 0x1AB6BED9, 0x04B7DCDD, 0x102BC8E0, 0x09331EFC, +0xE8ACC1F5, 0x222E4BD4, 0xF715E1FA, 0x23484BED, 0xEB3913E0, 0xDFE4B711, 0x0C411B0D, 0x18FEE800, +0x42231127, 0x493B19FC, 0x39BCDE12, 0xFB46121B, 0x452DDDE2, 0xEA0DE3FB, 0x2109EF1A, 0x1B3CF81E, +0xE228F2EC, 0x48ADF6FF, 0xC2C0202C, 0x9F3FEB19, 0x192BC8E9, 0x2B0F0F04, 0x4F2DFD19, 0x1045D9FB, +0x0B3FE8E2, 0xF2FBE316, 0xBC0EE623, 0xB301E419, 0x1F2226D9, 0xBE0009FA, 0x27121A27, 0xF95002EA, +0x4714DED8, 0xBAE6E1ED, 0xC5191EF2, 0x3B2E14F1, 0xFB0FEC06, 0xDAB123E7, 0x3DDBFFD9, 0xC318ECFD, +0xDFD014EF, 0xB23D0608, 0x11E326EB, 0xC4CAE90B, 0xD847EC0D, 0xDFC3FFE2, 0x55BD0815, 0x17F9D3E5, +0x3EE91121, 0xF9C41712, 0x22AFE0ED, 0xCDE7FBD7, 0x1BC9F5F0, 0x0422F4F0, 0x200EDC06, 0xC3F700E6, +0x3B20EBF7, 0xD115EDEE, 0xC3ED08EA, 0x5AFEEC29, 0xC21F3247, 0x3315163E, 0xE330F659, 0xEAF5F5CC, +0x1716113E, 0x4CE611BF, 0xD517DBC2, 0xB2EF043A, 0x45DD25B5, 0xC0151E13, 0x131CE843, 0xFFE4EB44, +0x5FFB27DC, 0x381336EB, 0xB8EFF017, 0xECF720CF, 0x200B131D, 0x0622F805, 0xBA07DD30, 0xD9061FDE, +0x30EA013D, 0x40300858, 0x9AD917B2, 0x270FC1C9, 0x1209FFCF, 0x201716EE, 0x2A050723, 0x1BEF0239, +0xBF18F310, 0xA83719A0, 0xA9DDD105, 0xE70F3037, 0xDE0D0E1B, 0xE1D4F955, 0x0CF00AC8, 0xE80FE4C9, +0x1BE2100A, 0x40FB0EDC, 0xDD1A17BD, 0x4E1604C2, 0x5CE7270F, 0x2BF635E3, 0x00D80350, 0xFD0027DB, +0xDD22B54A, 0xEAE11247, 0xF705CE22, 0x01F2D736, 0x1D17443E, 0x1D0145D8, 0x18181111, 0x04FEDCDA, +0x01EAD7C3, 0x05DC23B5, 0x1FCFB959, 0x1A0A431E, 0xFCDC2CFC, 0xD61452EC, 0xECF23DCB, 0x31D6A652, +0x261BFE0D, 0x0CD73552, 0xEAFB11D3, 0x10291952, 0xF7F41F34, 0xEC1E3CF6, 0x180A41E2, 0x03FBDB23, +0x2026F9B3, 0x0C02E42A, 0x08F030C9, 0x0E19E70E, 0x0F1519C4, 0xDCE703F2, 0x100CC833, 0x0105272C, +0x01252903, 0x0DCCCB5C, 0xD21AAA0E, 0x2917AFC0, 0x1BEA0E3F, 0x2A1B52BE, 0xF2E8CAC0, 0x1DDDBB4C, +0x0215D73D, 0x0DF7E5E1, 0x08E230F5, 0x0EDFCA4A, 0x19FE0ED9, 0xE6EC42C4, 0xFB1D23F4, 0x22090632, +0xEA073F21, 0x06ED2115, 0x180BBFCC, 0xE6E20EF6, 0xE512F23A, 0xE21BF7F4, 0xF9F22E37, 0xF82B2F53, +0x1CF94421, 0x16D8C2B1, 0xD9F4FF1C, 0x23EC4B14, 0xF5F03339, 0xFE172B40, 0x1C2F0D57, 0x230A0632, +0xECDA3CFD, 0x1306C52D, 0x01D3D856, 0x1D2846FF, 0xF9D82150, 0x1B0CF31D, 0xEE03C626, 0x1AFFBED8, +0xD605AEDD, 0x2403FD25, 0x01EC28C3, 0xF1E1C8F7, 0x06E7D140, 0x09CF325A, 0xF2151AC3, 0xE7F8F120, +0x0A2BE353, 0x23E5FBBD, 0x1C1645C1, 0xF716D011, 0xF6ED1D15, 0xFB2123B7, 0xE2070A2E, 0x16F7C2E0, +0xE6100D18, 0xFB19D440, 0x2FF15619, 0x1F0FB938, 0x0101272A, 0xF8F42F35, 0xF3DFE649, 0x21E549F4, +0x0D430000, 0x400EE51B, 0x452E181A, 0xF1411DFB, 0xF4DC1919, 0xF247E3FC, 0x33141AE1, 0xF81CF515, +0x3115200D, 0x3FCFF713, 0xE9DA17F6, 0x0BF31102, 0x17D1E3E5, 0x0E9712FA, 0x2EAFE6C0, 0x5900F928, +0x50D8CE27, 0xCDACD9FF, 0x41BB0B2B, 0x2AC9181E, 0x13E1020F, 0x0708EC08, 0x09D220E1, 0x1EC0E206, +0x1F2BF6E9, 0xD23809FD, 0x46D3FBF0, 0xF617E2FB, 0xF4F61EEF, 0x31D71BE2, 0x1AD4F7FE, 0xB534F103, +0xDE15DD0C, 0xC7FBFA12, 0x1931F0DC, 0x4AD8F1F7, 0x4937DEFF, 0xF908210F, 0x35D121E0, 0xECCC0EFA, +0x34CBEDF5, 0x062A0CF3, 0xE4B12103, 0xD73DF427, 0xDCE8FE15, 0xB4FC04F0, 0x0B06DC23, 0x16391D22, +0x23051310, 0xE6FCFADC, 0x1C95F2DB, 0xE3DE0DBD, 0xDB270A05, 0xB4EEFC00, 0x1845DD16, 0x0149F11D, +0xC6D52621, 0x1E34EE03, 0x17D20AF4, 0xA10611FA, 0x20F937DE, 0x5D17F8DF, 0xA00435EF, 0x3ECEC7DC, +0xF9D8EB0A, 0xFC332200, 0x1DE9DBF5, 0x2E580CEE, 0x3E1AF9D1, 0xC64715F2, 0xD7CB121F, 0xEFF400F2, +0x084CEAE4, 0x064DE0DD, 0x2B10DE25, 0xC1FB0218, 0xD4F8EADE, 0x32D6FBE1, 0x0F65F602, 0xEC32E83C, +0xD2F613F5, 0xC526F9E1, 0xF0F51302, 0xF5B017E3, 0xFA0FE4D7, 0xFACBDFE8, 0xC0F222F2, 0xD630E81A, +0xD6DEFEF8, 0x33A602F9, 0xC826F633, 0x10E51002, 0x0308E9F3, 0xE011251F, 0xBAEAF8EA, 0x26F21E12, +0x0000021A, 0xE00E08CB, 0xE0ED0715, 0xE71CC0F4, 0x0A1ECE0A, 0xDB260303, 0xE7E2BEF5, 0xDDE705F0, +0xDA0BFEE3, 0xF6D83250, 0xF4FA1CD3, 0x2805FFDC, 0x1DDBBAB4, 0x1611C218, 0x18E13F09, 0x1C1D0DF5, +0xDC25B403, 0x2BF6AE32, 0x180F4136, 0x39FC61D3, 0xF3DE1BFA, 0x1AD94100, 0xE8F8BFDF, 0x0826E0FE, +0x1AE142B8, 0xF80F20C8, 0x0116D9EE, 0x22E84BF1, 0xEB22C3B6, 0xEE01EBD7, 0x0EE8CA10, 0xD8FA00D1, +0xEB073DD1, 0x1EE3BBBB, 0x12EFC639, 0x3708A121, 0xDFDB49B3, 0x28D85000, 0x0C003429, 0x000D28CA, +0x12D1C6A9, 0x0D061B2D, 0xEBDCC304, 0xEF05C8D4, 0xDACD02A5, 0xF3F7CAE2, 0xE5144313, 0xFC222B4A, +0x23F4B51C, 0x18270FFE, 0xEDDC3CB4, 0xE9213F49, 0x1EF309CB, 0xE3E5BB44, 0x0AF6E333, 0xE1EF473A, +0x080DD036, 0x0FEB38EE, 0xFE0629DE, 0xDAFCB224, 0x0DF9CB21, 0x07D3DFAC, 0x06F822D0, 0x1E1ABAF1, +0x00F32835, 0xE00AF71E, 0x08E2D009, 0xE003F9D5, 0xD5F2AECA, 0xF4EFCCEA, 0x0B0E33E5, 0x0D271A00, +0x1232EB59, 0x07DBDFB4, 0xD2ECAAED, 0x17FC40D4, 0xDAEB0313, 0x07E8E010, 0x07F5201E, 0x13173CC1, +0xDBDCFDFD, 0x24ECFBC3, 0xF2D9E6FE, 0x420695DE, 0xFC172C11, 0xDF0F06C9, 0xF413CC3C, 0x1B11F217, +0xE01148C7, 0x1EEEBA3B, 0xDDDCFCB3, 0x24E1B4F6, 0xF4DA3303, 0xF1FC192C, 0xEAFC11D4, 0xEAD41354, +0x22ED06EC, 0x23090000, 0x0BE7FBE1, 0xE0BFE4F0, 0xF2C4F818, 0xE5BEE6EC, 0xDD3E0DE7, 0xEB360516, +0xFBC6ECF2, 0xFCEC23EE, 0x0A20DD13, 0x12211FF8, 0xF23A1706, 0xCC36E6EF, 0x1C26F5F2, 0x2F05F303, +0xF540FA23, 0xFE4FE3E9, 0x23CED927, 0x18C2FA0A, 0xD1A411EA, 0x1A4306CB, 0x474E0EE6, 0x21331FDB, +0x2C33070B, 0xC60804F6, 0x515A12DF, 0xE8FDD7CE, 0xD7171126, 0x1043FF11, 0x48F8E81C, 0xCF48E021, +0xF9CA09E0, 0x2B26210F, 0x1F10FD02, 0xC9ECF8E7, 0xF9D30FED, 0x33222205, 0xC44E0A05, 0x41381425, +0xCEC21811, 0xCFFFF5EA, 0xC0E7F727, 0xF5481810, 0xE247E3E1, 0x11150AE0, 0x40C116EE, 0xFDEC18E9, +0xCF0F2415, 0x46E5F6E6, 0x430D1DF3, 0x11EEE61A, 0xA45017EA, 0xF1BBCB28, 0x35DC19E3, 0xECBC0CFB, +0x3631ED1D, 0x202C0EF8, 0xE7D2F8FC, 0x441910FA, 0xBED4E4F2, 0xCE3DE6FD, 0xD8CB0B15, 0xDE3C01F2, +0xD7F1F914, 0xFDCE00E7, 0xBDEF240A, 0x2CC5E616, 0x1C16FCED, 0xE805F412, 0xE4F11023, 0x14110DE8, +0xE2ED1418, 0xD7F7F6EC, 0xF8E9001E, 0x18C3E111, 0xAA4BF1EA, 0x2BB02DDE, 0xEB18FDD9, 0xEECFED11, +0xAEF8E9F7, 0xB0D129E0, 0x1952D9FA, 0x07D40ED6, 0xB13EDF03, 0xC74C28E9, 0xD7DB12DC, 0x0B11FFFD, +0xCEFC1D17, 0xC34CF623, 0x4A011523, 0x0410DDDA, 0xF2072317, 0x33101B21, 0x35F7F4E7, 0xF2CBF3E0, +0xAACDE6F4, 0x00002F0B, 0xE51344C5, 0x1211E938, 0x091EE1F6, 0x26CD02A4, 0xE028F84F, 0x2522B206, +0x1BFDF3D5, 0x0C23CC06, 0xE809C031, 0x19D30F54, 0x1B0DBD1C, 0x2EE256F5, 0xE11A080E, 0x17DF11B8, +0x16E4C20C, 0xEECA3B5F, 0x05EA2D11, 0x140AC432, 0xE0F6481D, 0xE024094B, 0x16F6121D, 0x14083C20, +0x200A481F, 0xE1F308E5, 0xFB0623D1, 0xDE1EF946, 0xEE211606, 0x15EC3D13, 0xF402E4DA, 0xF1D8E700, +0xE6D0F358, 0xEEF9C621, 0xE3F3BB36, 0x0A171E3F, 0x0D0ACBE1, 0x09071F21, 0x04DDDDB5, 0x051422C4, +0x07F2211B, 0xFD1BD5F3, 0xEFFE3926, 0x261602C1, 0xF40F3438, 0xEE07EB21, 0xF4DA1CB2, 0x241EFCBA, +0x2600B2D8, 0x11DC174C, 0x1A0542D4, 0xDF1349C6, 0x19E10F47, 0x09131F16, 0x010A27E2, 0x0A12E2C5, +0x01E229F5, 0xE3090AE1, 0xDEFEB629, 0x150E1319, 0x221D050B, 0xE3F4F6CD, 0x21F6071D, 0x23FC4C24, +0x1312C6EA, 0x12F01738, 0xEFFF3AD7, 0x1FFD48D5, 0x1C0B0CE4, 0xF5F71C30, 0x03F725E0, 0x0D38349F, +0xEB0614DE, 0xFB03DCD5, 0xEAC9EDA2, 0xDEF8B6E1, 0xDD0B0532, 0x17ECC2C3, 0x27E1FEB9, 0xEF14EA14, +0xF9FC2F2C, 0x160AC232, 0x1C0445DD, 0xE92311FB, 0x06162F3D, 0x03F12538, 0x0AFE1E26, 0x06E72141, +0xE1E6F841, 0x3221A5B7, 0x00E6D70D, 0xF8F8E030, 0x24FF4CD8, 0xFB02DCDB, 0x162313FB, 0xEAF1C2CA, +0x190CBFCC, 0xEFF4E834, 0x60370000, 0x25C139F2, 0xF731FCEA, 0xF734E009, 0xF10AE00B, 0xD2C81AE2, +0x26590710, 0x1ACE0231, 0xF41D0EF5, 0xD62BE4F6, 0xEF340303, 0x0AB9EA0C, 0xBA251EE1, 0xCE6BE204, +0xC1530A42, 0xC411E82A, 0xD4CD15E9, 0x35D4040C, 0xBEBE0C04, 0x01D4E7E5, 0xE9CAD9FD, 0x58F1EFF2, +0xAC46D1E6, 0x21DED4E2, 0x2C240706, 0xE0330304, 0x4931F80B, 0xF44C2109, 0xB74A1DDB, 0x2AC320DE, +0xBC2D02EC, 0x07EEE4FB, 0x2DCB20E9, 0xC910FBF4, 0xAF52F217, 0xD720D72B, 0xDBCC02F8, 0x1342FDF3, +0x26B315E7, 0x3D450325, 0xC7C0151E, 0xE9471219, 0x42E4111F, 0x4AA3E60B, 0x5537DF36, 0xB4E92E0F, +0xDCFEDC10, 0x18F1FCDA, 0xBFFA10E7, 0xD706E721, 0x2BD20022, 0x16500305, 0x4CC2EF29, 0xD71F2417, +0x20120009, 0xE7EC08EA, 0xE520F1EC, 0x42C8F3F8, 0xF23A19F0, 0x34111BEE, 0xCBF00DE8, 0x02F50DE8, +0xC3F0DA1C, 0x67CC1518, 0x09463FF4, 0xFCF7E11D, 0x01F6DC20, 0x2F1D27E1, 0x2816F9F4, 0xAA1800EE, +0x2E592DF0, 0xF4DC07CE, 0xCF49E504, 0xD33BF6E0, 0x17040514, 0xC445EE25, 0x4C33EB1D, 0xBB39250C, +0x4BAFE310, 0x58F8222A, 0xDD34CF1F, 0x4CCBFBF4, 0x32FDDC0D, 0xCF230ADC, 0x2D13F7FC, 0x49D6FA15, +0x09E2E002, 0x4425E20A, 0x170D1C02, 0x47DDF0E4, 0x2A441F05, 0xC0B502E4, 0x40B71823, 0xCFEE19DF, +0xD34A0917, 0xE7370521, 0x0000F1F1, 0x1CFDBD25, 0x1B080D30, 0xC61F61F7, 0x05122C3A, 0xDEF606CF, +0xF805D023, 0xD91D50F6, 0x1E0D0AE6, 0xE111F7E9, 0x1B1F42F7, 0xE9221106, 0xF7101EC8, 0x061C2FBD, +0xE8E711F2, 0xFB0E2335, 0x0DFDE6D5, 0x10E7E742, 0xF1FFE62A, 0x04F424CC, 0xF82B1F54, 0x10F8E821, +0xF40B1C33, 0xE2F3BA1B, 0xFBFBD32C, 0xF6E531BD, 0x12F7171E, 0x13073BDF, 0xDDEFFAC8, 0xDC1DFB0B, +0x0D00E4D9, 0x120AC7E2, 0xFEE1D546, 0xD9EFFEEA, 0x1EF0F737, 0x021225E9, 0xD4DE54FA, 0xF3FC1B2D, +0xF80DD1CC, 0x1EE1470A, 0x01E6D7F2, 0xE1E6F70D, 0xF9F0DEE9, 0xFB14D2C5, 0xE603F2D5, 0xD82750FF, +0x011AD941, 0xE2040A23, 0xE425BD4C, 0xFC28DC01, 0x22D94BFF, 0x0AE6E242, 0x16EDC2C5, 0xD90B011C, +0x050AD232, 0xDB114E18, 0xF2EB3512, 0x1AE642F3, 0xCFF1A7E7, 0xF220C9F8, 0x171FC148, 0x0735225D, +0xF2FF1926, 0x14D914FF, 0x0DFC36DC, 0x191BF00E, 0xE50CF3CB, 0x07F5221C, 0x2305FAD4, 0x1FD9094F, +0x25FFB4D6, 0x170FC1CA, 0x0FF3C91A, 0x02E82BF0, 0xDFEA07ED, 0xECE9143F, 0xE2E44645, 0xF2F6E71E, +0x26FAFE21, 0xFD17DB3F, 0xF0F6181E, 0xF818D0F0, 0xDD24B5B5, 0x00E8D8BF, 0xFE08DA20, 0x1104C6DC, +0x22FCB5D4, 0xE5FFF3D7, 0xE4CEF5A6, 0xF1EBE6EC, 0xEC06EBD3, 0xF5E91DC1, 0xE70540D4, 0xFC1A2D0E, +0x08E7E10F, 0xFAFE22D9, 0x06F82130, 0x09E80000, 0xF4F51F10, 0x49EE1CE2, 0xEEFADF17, 0x44ECE9DE, +0x0ECEE415, 0x354C1AF6, 0xCC020DDC, 0x50100C25, 0x1F362818, 0xCCB6090D, 0xE5C30C21, 0xFDDA0DEC, +0x350DDA01, 0x131A0C1A, 0x1AAB15F2, 0x2B590DD2, 0xC4A80331, 0xC517EC30, 0xCED3EEEF, 0xCC44F505, +0xF40DF5E3, 0xE2341BE5, 0x20C50AF3, 0xF9D308ED, 0xE345DF06, 0x3DF10CE4, 0x203D15E6, 0x1F4BF8EB, +0x1D37F8DD, 0xFEDC0BF1, 0xC14726FC, 0x0EDD1820, 0xCC221A05, 0xC1CB0D06, 0xC1CAE90C, 0x45EC180F, +0x9A3AE3EB, 0x15D23EED, 0xABCDEE06, 0x5EB52DF4, 0x19D33523, 0x0C41F1FB, 0xE8F8E4E8, 0xCD24F01F, +0x1239F604, 0xE6EA1611, 0xF23E0E11, 0x14F01A16, 0x450F14E8, 0x44BD1DE8, 0x3DE3E41C, 0x13E316F5, +0xEABEECF6, 0xEE00121B, 0x3AEEEB28, 0xE33AED15, 0x229AF5ED, 0xBF26FAC1, 0xE7351A02, 0xA6D80FF3, +0x5303CEFF, 0xCF082ADB, 0xB7BFF6E1, 0x3521E0E6, 0x07120DFA, 0x25E9E0EA, 0xB423FDEE, 0x0E012305, +0x1DF21B28, 0xCE1AF6E6, 0x2AF50AF2, 0xF80602E4, 0x0AB71FDF, 0x1F2A1E21, 0x0316F8FE, 0x0F4F2512, +0xEBCE1A27, 0xC0B3EC0A, 0x23E818DA, 0xEB0AFBEF, 0x164F13E3, 0xCDE61326, 0xC4DF0BF3, 0xBE3C14FA, +0x51E8E714, 0x10F2D710, 0xD3DB191B, 0xEA1F06FD, 0x344311F7, 0x21D60C1A, 0x301207FE, 0x2FF3F9EA, +0xEED1F9E4, 0xD335EAF9, 0xFA0505F3, 0x4B62DDDD, 0xD130DC3A, 0xD1D60708, 0x21D508FD, 0x4112FAFE, +0xEE431816, 0xAE0BEAE4, 0x03E0D51D, 0xD0AEDC07, 0x092BF72A, 0x07C92003, 0x20BDDEF1, 0xDC45F9E5, +0xCC1905E4, 0x0AE50CF2, 0x53D41E0D, 0xC928D505, 0x48B90F00, 0xF9CCDF1F, 0x3D4D21F4, 0xCB0B15DB, +0xF0050D1D, 0xE1F2E8DE, 0x36E5091A, 0x3EF20E0C, 0xD8E6161A, 0x241D00F3, 0xC1E7FD0C, 0xCFB8EA0F, +0xE33CF6DF, 0xF1AA0CEB, 0x45F11A2E, 0x3BFDE3E7, 0x28971425, 0x2DDC00BF, 0xE9DFFC03, 0x05B6EE07, +0xF1DE23DD, 0xB6E01807, 0x1F31DDF7, 0xEB390A0A, 0xD44B13F0, 0x11E2FC22, 0x5021E80A, 0x072E28FA, +0xEEBBDF07, 0xEC30E9E3, 0xD2F2EDF8, 0x3AE8061B, 0x4CB6EF10, 0xCBD2DC23, 0x302F0D07, 0xB73308FA, +0xAFE821F6, 0xD63FD6F0, 0x4CF20216, 0xBFEC23E6, 0x17DB19ED, 0xC9E8EF03, 0x3A34F10F, 0x404711F3, +0x1814181E, 0xEF17F1EB, 0x551F17EE, 0xD7D62E09, 0xF519FFFD, 0xFC16E4F1, 0x28E62412, 0xDD2600F2, +0xFD220501, 0x5DE52407, 0xB9B6CC0E, 0x180220DE, 0x27351026, 0xFA13010D, 0x050222EC, 0x2BE0DDDA, +0xCEB2FC08, 0x0CD00A27, 0xC3FBE509, 0xF6F51422, 0xEEAC1EE3, 0x10CFEAD5, 0x1FDBE9F7, 0x47C20AFE, +0x3AC8E2E9, 0xC80DEEEF, 0xECD9F1E5, 0x0FCEEC00, 0x1F2CE70B, 0xE5BCF8FC, 0x37B10DE4, 0x4200F1DA, +0xFB081A27, 0x385C231F, 0xC7210FCB, 0xFEF610F9, 0x26E30000, 0xEBF8132F, 0x051ADEBF, 0xEDE13CBA, +0x1819C0BF, 0xFE0F2AE7, 0xFC1D24F5, 0xDBE90311, 0x02F02918, 0x190FF037, 0x1CE10B46, 0x0AD0CF58, +0x170410DD, 0xF72231FA, 0x1D15BBED, 0x16053EDD, 0x0A1D1D45, 0xF412E316, 0x191510EE, 0xE3DE0BF9, +0x10113838, 0x02D92AB2, 0xFF07D7E0, 0x21ECF83C, 0xEDEBC4C2, 0xE8EDC13B, 0xF9042025, 0xDB35B4A4, +0x1AE3BF45, 0xFEEB2514, 0xEDFAC522, 0x0B08E3E0, 0x13F2151A, 0xF3D7E551, 0x1D110BE8, 0x25170312, +0x19C9F1A2, 0xEA0C1234, 0xE3FBF52D, 0xF3DC1BFD, 0x11F518CC, 0xFDF92521, 0xF61E31F6, 0xE0EA47EF, +0x09E6E142, 0x09F8E031, 0xDDF9B52F, 0xDAE34EBA, 0x160111D8, 0x1524C34C, 0xF01C170C, 0xDDE14BB9, +0xE7EEC0EB, 0x34005CD8, 0x17E1C2F7, 0x0F1B1943, 0xFD11DCEA, 0xD5EF5417, 0x08D73052, 0x1CDAF4B2, +0x162AED52, 0xFBE9D3EF, 0xF31C1CBC, 0xF225CA03, 0xEB231405, 0x13FFEC2A, 0xDAF4B135, 0xDC21FDB8, +0xDBF90420, 0x0F23184B, 0xE40945E1, 0xFE03DBD4, 0xDF1B48BE, 0x0CE01C49, 0x1200C728, 0xEFF016C7, +0xFC12D43A, 0x0AE9CDC1, 0x15101318, 0xDFE949C1, 0x2314B6C4, 0xFADF2349, 0xFCE4D5F4, 0x1EFA462E, +0x03DEDBFA, 0x19230F05, 0x210D061B, 0xFF2327FB, 0xDA21B207, 0x1302ECD6, 0xEF14E93D, 0xE4010D27, +0xEEE7E942, 0x1B1DF30B, 0x1A180D3F, 0xFF17DAC1, 0x0000F2FA, 0xB1351ADD, 0x4FE626F3, 0x2A12D80E, +0x1434FEEA, 0x36F2EC0D, 0x42C6F2E7, 0xB8C1E512, 0xE23821E9, 0x0E3B0AF0, 0xFC3E1BEE, 0xB631DB15, +0x0623DDF8, 0xD81B2306, 0x28B5FFF3, 0xEF270023, 0x1718E9FF, 0x33EB110F, 0x30F0F5ED, 0x1426F8E7, +0xC727ECFD, 0xB34010FF, 0x3CFFDBE9, 0x44141326, 0x5EA2E4EC, 0x3325CBC9, 0xD8130BFC, 0x0C1E0016, +0xF92EE4F7, 0x1109DEFA, 0x2632E9E0, 0x520EFEF6, 0x342ED619, 0x3229F4F9, 0x0D0DF700, 0x1D111BE5, +0x15BC0C18, 0xA728ED1B, 0x54283000, 0x47C32B00, 0xCAEE1FEB, 0x11EEF2E9, 0xC00C1717, 0xB215181C, +0x030CDA13, 0x480325E4, 0xEDCAE126, 0xD8D915F2, 0x23DAFF01, 0x16240503, 0x374AED04, 0x2904F1DE, +0x3D3DFF25, 0xB3CF1516, 0xDE0A25F8, 0x3306F9E2, 0xCDB9F622, 0xBCD0F5E0, 0xEC291DF8, 0xA8DE14FE, +0x07FC2F06, 0xCB3CE0DC, 0xFBC4F213, 0x2ED0DE15, 0x35DEFB08, 0x38C8F305, 0xCF36F010, 0x3ACE0A0E, +0xEC3A1209, 0x2152EDEF, 0x40440729, 0xCC04E91C, 0x0C56F424, 0xB04A1C2E, 0x28CAD721, 0xCDD8FF0F, +0xCC1CF400, 0xE6BDF3F3, 0x0F020E1C, 0xFBFD19D9, 0x3DC12326, 0x00D8EA17, 0xCB2ED900, 0x5008F3FA, +0xEF2B28DF, 0xA726E9FE, 0xEAF5CFFD, 0x3FE4EEE3, 0x3F27180B, 0x3C47EA01, 0xC8B5EBE0, 0xBEC1F023, +0xD7231A16, 0xE15501FA, 0x40AAF82E, 0x4511E72D, 0xF8DAE317, 0x20030000, 0x0BF7E232, 0x0BF1CDE6, +0xCFF25835, 0xFAE12DF7, 0xCFEAA613, 0x22FFF928, 0xE8E93FC1, 0xE327BAFE, 0x1E030A24, 0xDFDFFAF8, +0x06F2D3E6, 0x00FED8D6, 0xE7DFF2F8, 0x10FB39DD, 0xDBD74CB0, 0x072BDEAE, 0x1EE90A11, 0x26DB4DFD, +0x16E8EEC0, 0x17FEC126, 0xEB03C4DB, 0xFCF924D0, 0xDBD8FC00, 0xE510F3E9, 0xEBF6C4E2, 0xFF2127F8, +0x0DE335F6, 0xE1E7B840, 0x1724C203, 0x0D23E5FB, 0xFD1ADB0F, 0x2C0C5434, 0x22EEFBC6, 0xDB130316, +0x1EE60A0F, 0x27E94F3F, 0xF7E21F46, 0xF1D837FF, 0xECFB3B22, 0x330FA638, 0x0CE21CF7, 0x22FFB728, +0x281AB042, 0xF714CE13, 0x11E0C707, 0x11D9E9FE, 0xD9EBFF3E, 0xF8043023, 0x1C15BCC3, 0xCAC95DA0, +0x13ECEB15, 0xDFEDB73B, 0x201CF8BD, 0xF033185B, 0x160B12CD, 0x01092631, 0x0E06192D, 0x0E001BD8, +0xEC0FEC37, 0xE4E143B9, 0xDD2405B4, 0x1EDAF501, 0xEFFF16D6, 0xD8F70130, 0xF1EF3738, 0xFBEB2E3D, +0x1F0147D6, 0xFAF52EE4, 0x00D92800, 0xE31B440D, 0x02112617, 0xF1DCE6B4, 0x17E3C144, 0xE117B9C1, +0x10F1E719, 0x0C15E3EE, 0xE315F53C, 0xE5260D4E, 0x02F8DB31, 0xEA14EF3C, 0x4B178DC1, 0xDC070320, +0x0F1CC8F4, 0xEAFB122D, 0xE71042C8, 0xEBE8EC40, 0x0EEDE6C5, 0xEDEF1618, 0xFFDE2AFA, 0x260B4DCE, +0xF3D9E600, 0xF816DFC2, 0xEA26EE4E, 0x14F23CCA, 0xD51AAD43, 0x0000133C, 0xFF5116EB, 0xCEF7D9D7, +0x3EFCF7E2, 0xE13E16DC, 0x39D5F7EA, 0x583312FD, 0x3FEB2FF6, 0x35B3E9EE, 0xDD01F424, 0x26230528, +0x5EB4FFFB, 0xB91E3623, 0x0AE11E09, 0xF6201F09, 0x4CC6E208, 0x1CBA2413, 0xC4370D1E, 0xD1001310, +0x35150627, 0x122AF4ED, 0x0BF71502, 0xC618E3E1, 0xF202ED10, 0xF2C31A25, 0xB3F71BEB, 0x2312DB1F, +0xC20804EA, 0x11FAEBDF, 0xA8DE18DE, 0x28113006, 0xE9CC0016, 0x4021120C, 0xEFC41908, 0xEB51E9EB, +0x331113D7, 0x5ACBF5E8, 0x10C431F4, 0x0927E914, 0xB403E0FE, 0xFBE925DA, 0x1B24DD10, 0xB43B0CFB, +0xE7C52413, 0xB94A0F13, 0x1DD7E1DE, 0xE5D10A01, 0xAE560C07, 0x290E292E, 0xF3F3001B, 0x5A131B1B, +0xB4E0CE15, 0xBAE6DDF7, 0xC801E20D, 0xEEC61028, 0x16F1EB11, 0x2FC5EEE6, 0x4D2FF814, 0x2DB526FA, +0x15B105DD, 0x2932EDD9, 0x0FD1FEF6, 0x2DD719F9, 0x29F30601, 0x3AFFFFE5, 0x44BFEFDA, 0xA437E5E8, +0x0E38350F, 0xDF07E510, 0x06AA0821, 0x0052222E, 0xEFEB27D5, 0x1ED11814, 0x0B0E0908, 0x2DA11D1A, +0xEB3405C9, 0x161C140C, 0xC2FBEF0D, 0x02E8E923, 0xD5BADAF1, 0x24C303E3, 0x374F04EA, 0xE8F10FD8, +0x21B01118, 0xB536F8D9, 0x3BF1DDF2, 0xFD0012E6, 0x2F4FDB28, 0x0D2CF826, 0xBEBC1C04, 0xC64C1A1B, +0xC62911DB, 0xF3AAEEFF, 0x40D5E52E, 0xA21C18FC, 0x18ED36F3, 0x00000F15, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 + +hard_output0 = +0x7FE75E62, 0xB98D686A, 0x16FBEDEE, 0xB22861F2, 0x0A1180B7, 0x666C4407, 0x1D4C462A, 0xB81A3416, +0xC6948E41, 0x3CCAE97E, 0x55352D5A, 0x85474DAC, 0xB8DBF013, 0xB9AF7B16, 0xEABEE865, 0x1EB02EF7, +0xBDE86A3E, 0x24EDAAC9, 0x88A3D4B9, 0x8CA15C44, 0x42487190, 0xF4BF8BFF, 0x05EA0214, 0x4708DC42, +0x222993B3, 0x24AAA18B, 0x4D49B2EA, 0xBBA8E2B3, 0x40426630, 0x5676818F, 0x1B79CC4B, 0x1595A8CD, +0x9088385F, 0x351D2A23, 0x0EDC4446, 0x8F17D03E, 0xEA8B3464, 0x8799E718, 0x79E11F5A, 0xE771403F, +0x38C65263, 0xA969930A, 0x59942FBB, 0x74138F31, 0x80518035, 0xB42D7BE0, 0x0C4A7E00, 0x4ACA8CB5, +0x76C89CCB, 0xB56F2806, 0x1C8CF4E5, 0xA192BBF1, 0x1139DA94, 0x46B1C7C9, 0x8B630748, 0xA882704B, +0x69DA8BE2, 0xF2508EA5, 0x05E4AEAF, 0x5AA0A9C8, 0x6177DDCE, 0x8D06A0E1, 0xECD023CA, 0x34FC0800, +0x5AE69AAD, 0xFE8E2F72, 0x5ED3DC49, 0x8B46EB93, 0xC0C3D55F, 0xAC8C8B89, 0x54A20A93, 0x81F6659B, +0x412B7086, 0x09D572AB, 0x85891AB0, 0x96DC6C57, 0xD7117C38, 0xFBA06F16, 0x9EAAFCE5, 0x44AD2185, +0x618D398E, 0xD9AC89E7, 0x031F1D0B, 0xE7D05975, 0x9E500DF3, 0x06BFEF20, 0x42888C91, 0xD6AF7FA6, +0xC839E4C7, 0x0E7B7724, 0x48693BD9, 0xD17B6519, 0x48004F1A, 0xF5A147D5, 0x521C8AD6, 0x26E19AE7, +0xAA17116F, 0x7196631D, 0x89643E51, 0x8CE6B02E, 0x128B9CF6, 0x2F45C206, 0x4EB6C731, 0xCFB17835, +0xCFE85026, 0xA4B9B9B0, 0xD2C152AC, 0xA4F89035, 0x79ED0C5F, 0xBC030479, 0x51D91130, 0x570C019C, +0x4E0306B7, 0x27978CCD, 0x5F8DA612, 0x33C121FE, 0x4407BB22, 0x6FF06E2F, 0x5D7F378B, 0x491B9A93, +0xA0BDE688, 0x0EA89618, 0xE91C3661, 0x27C69F6D, 0xC46463FD, 0x67649FE9, 0x8F6E077F, 0x4D456A5F, +0x396B7218, 0x0C26C824, 0xA4E8FD71, 0xDCA1C865, 0x6F1CC581, 0x97C974D3, 0x38D0F6FC, 0x04190419, +0xED8C7FC5, 0x794BA619, 0x0090798D, 0x3044F72D, 0xBB77EEEF, 0xF9BCFDE1, 0x2C1580F7, 0xC0292305, +0x86827E71, 0x356C7D12, 0xA84DBECC, 0xCA53E3D9, 0x236E73F0, 0x9FD2AFBA, 0x652E7A66, 0x744A1147, +0xBFE9AF5B, 0x59F2552F, 0xBD77D3C0, 0xCD726524, 0x1D56A0FB, 0x85207831, 0xB6F1861B, 0x369CF946, +0x71CA18BD, 0x5EE3F7CF, 0x2C3FC902, 0x4301214D, 0x53A64F16, 0x2956B34F, 0x3DF40618, 0x41A36C68, +0xF7BB2B5F, 0x0C5CD46D, 0x3E4354AC, 0xC29AFEF7, 0xC1A310AF, 0xFFEF128F, 0xBE1C8D14, 0x8B123664, +0x854B7E00, 0xAE175861, 0xE4067FE2, 0xA264F00A, 0x988634EF, 0xEC9E38E7, 0xFE2968A6, 0x2B30719D, +0xFE3DC88D, 0x09E222C8, 0x6B4F1B50, 0x436C7137, 0x161DFE10, 0xA68C403E, 0x12A2856F, 0xAAB8DFF0 + +k = +6144 + +e = +10376 + +iter_max = +8 + +iter_min = +4 + +expected_iter_count = +8 + +ext_scale = +15 + +num_maps = +0 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE, RTE_BBDEV_TURBO_CRC_TYPE_24B, +RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN + +expected_status = +OK \ No newline at end of file diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e10376_crc24b_sbd_negllr_low_snr.data b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e10376_crc24b_sbd_negllr_low_snr.data new file mode 100644 index 00000000..8ff71aac --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e10376_crc24b_sbd_negllr_low_snr.data @@ -0,0 +1,643 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_DEC + +input0 = +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x16000000, 0x081112e9, 0xcd2d1fac, 0xf9fdf4db, 0x110edf1a, 0x0dd9e84f, 0xdbe9e611, 0x0fde0206, +0x43df19f9, 0x071ee4f7, 0x1f1322c5, 0xd4e5f6f3, 0xe3fafcde, 0xd002f52a, 0x4427f801, 0xbadce403, +0xce101dc9, 0x09fff729, 0xd5f91f2e, 0xf3f0fee8, 0x4f061cde, 0x4a2b2753, 0x1d24dffb, 0xd3040bdb, +0xcaf2fbe6, 0x1cd9f350, 0x00fff42a, 0x49d02759, 0x4005de2d, 0xd3fb1923, 0x36cbfc5c, 0x020a0de1, +0x39072621, 0x3b0defe5, 0xb4ebec13, 0xddee2416, 0x24f206e5, 0xd1e1fcb9, 0x0e150713, 0x21f6191f, +0x0d0bf9e3, 0xf0061bd2, 0xae2717ff, 0x0de9d740, 0x13031a2b, 0x4c0215d6, 0x4534dc5c, 0xa1f41c1c, +0x001b3842, 0xeaf83ecf, 0x2519b4be, 0x1d220afb, 0x05162312, 0xee00ea29, 0x25ef4d39, 0x0cc03398, +0x08f5e0ce, 0xb7e68f43, 0x14f7ec31, 0xe7f5f11e, 0xfd29d6ae, 0x0af133c9, 0x03fdd5db, 0x0bca1d5f, +0x00172840, 0x061add0e, 0x0516ddee, 0xe10cf735, 0x1709101f, 0xdf0908d0, 0x06fe2dd6, 0x18ed11c6, +0xd315ac14, 0x06f0d3c8, 0x1ed9474f, 0xef223806, 0x28f3ffe5, 0x0eeb1aee, 0x080de0cb, 0x29f452e4, +0xdbd702b0, 0x041b2cbd, 0x1df30b34, 0x1927f102, 0x2010b719, 0xf0fde8d5, 0xe9eaef3e, 0xf10b36ce, +0xdff4fa1b, 0xfd0cd5e4, 0x04f923d0, 0x110f3936, 0x06fe2e26, 0x00e42844, 0x11e839c0, 0x13ecea13, +0xf621e349, 0xff332600, 0xe604f1f5, 0x0533d425, 0xf240c9f5, 0x0b5933e8, 0xdf1bf931, 0x02c929f2, +0xeee6eb0f, 0x12a2160f, 0xfdde26ca, 0xf016c8f9, 0xdd03b5ee, 0xe2ee09da, 0xfdf4db17, 0xdb0603e4, +0xffda2821, 0xf6f9cefd, 0xd5ca5421, 0x12f03af2, 0x182f0fe9, 0x10423808, 0x0c19e41a, 0x27af02f0, +0xfc14d42a, 0x08c9d114, 0x274700f2, 0xe8c110e1, 0x0e0f19e9, 0xeb58c3e7, 0xe20fbbd1, 0xe52e0e1a, +0xf145e607, 0xe316bc1d, 0xe3e04511, 0xd93c0009, 0x27db01ed, 0x0a30cd03, 0x23434af8, 0xe13f09e6, +0xe4d00de9, 0x07202208, 0xdb3ffd07, 0x0955cfea, 0xeae8112c, 0x041324f1, 0x1011e815, 0xf6d01f17, +0xe5eb0cf9, 0x17be00ee, 0xa2dbeee6, 0xb2e73603, 0x08ebda0f, 0x250fe0ed, 0xe61a0218, 0xf1dbf2f3, +0x420b18fd, 0xa1e9e5e4, 0xd9e1caf0, 0xe7cffe08, 0xeba8f009, 0xc45213d0, 0x20a1ed29, 0x28fa0737, +0x1ad900dd, 0x1bc10dfe, 0xab360c18, 0x023bd3f1, 0xacfd2514, 0x4d382c25, 0xf25adbf1, 0x1eda1a32, +0xf11e0902, 0x23a5e70a, 0x1f32fcce, 0x0ef0f8f5, 0x2029e6e9, 0xdfda0701, 0x1860fafe, 0xd8e5f139, +0x5612010d, 0xf2572d15, 0xba23192f, 0x2ac1e2fb, 0xe91e02e8, 0x44d2100b, 0x08f9e307, 0x5edddfe0, +0x240fc906, 0xc1c3fbe7, 0x0f231614, 0xaa111806, 0x1e2d2eea, 0xc6ba0a06, 0xedc9121e, 0x1ffc150e, +0xd11b0a23, 0xe800f80d, 0x2f27f101, 0x06110638, 0xd01fdef8, 0x15ee0717, 0xba0e131a, 0xcde51f43, +0xd21df6bc, 0x051006c8, 0x2616ddc3, 0x3ef8fee0, 0xf3cde9a4, 0x3bd81b4f, 0x25001229, 0xf6eb023e, +0xd81ee3b9, 0x1cecff14, 0x29fef4d9, 0x9f2d0255, 0x121e38f5, 0x18eb16c2, 0x271a1041, 0xf7f700cf, +0x28f3e1e4, 0x49e90011, 0x4b0fde1a, 0xd9e723f1, 0x0de402bd, 0x261f1b0a, 0xbe0afe1e, 0x0823e606, +0x3d041f2c, 0xf1f21436, 0xec1ae7f1, 0x02ebebc3, 0x52f1da1a, 0xc9e5d60d, 0xa9dc0e04, 0xc504d22d, +0x09fd1325, 0x3706e02e, 0x26dc0f4b, 0x2012fe3b, 0xeef109e7, 0xb509161f, 0xf01dde45, 0x13d51853, +0xc823ebfb, 0x2d23f0fc, 0xcbf6fbcd, 0xd505f22d, 0xee0afd1e, 0xa6e615bd, 0xa020ce07, 0x411937c0, +0xbcd318ab, 0x2a1d1cf5, 0xc400ff28, 0xe32014f7, 0x2bf30b1b, 0x0e03fc25, 0xe1e41bf3, 0x2f1b0943, +0x06060721, 0xc601de26, 0xd3faee2e, 0xe51c05bc, 0x0411f4c8, 0xcfe7dcbf, 0x2d12083a, 0x0bedfb3a, +0x0efc1d2c, 0xca1f1909, 0x13f80e30, 0x3a21eab7, 0x3c2b1252, 0xf2e4140b, 0x2527e5b1, 0x2ffe03d6, +0x44e4f90b, 0xcf21e449, 0xd8e9f6c2, 0xeef8001f, 0xfb151513, 0x48f9de22, 0x2ad6e0ad, 0xe63602a3, +0xdae50e0d, 0xe3e5ff0c, 0xc721f449, 0xa0d5efac, 0xe70039d8, 0xc7f71031, 0x2b22eefa, 0xe5fb04d3, +0x0021f2f9, 0x2bce28a5, 0xdffefa00, 0x1af642e2, 0x1d190bf1, 0xecf515cd, 0x0af4cf34, 0xf406cbd1, +0x08ebd013, 0x14173b40, 0xe40e0bca, 0x29ff51d9, 0xd303abda, 0xe303f5d5, 0xf6facddf, 0xecef3cc8, +0x1324ecb4, 0xd213563c, 0xe00ff91a, 0xef121639, 0xf5d2cdaa, 0x1914f013, 0xe80f3fe8, 0xe31c0bbc, +0xe3d5bcad, 0xe610be38, 0xe72df155, 0x1cef44c7, 0x13ccc5a3, 0xe1e20909, 0xecf2c337, 0xf2fd35da, +0xe5cd435b, 0xdc1304c5, 0x050722d2, 0x121fea46, 0x36f8a1e1, 0x1223c6fb, 0x18040fd4, 0xef1517ee, +0xf8e32ff5, 0xf22cc953, 0xfe04d7dc, 0x10e7390e, 0x1ef4091b, 0xdbf2fde6, 0x1afa432e, 0x2211b5e8, +0x211eb8ba, 0xff1529c4, 0xdaea004e, 0x11e539ed, 0x13b4eaf2, 0xe10af7db, 0x11e9c81e, 0x1eeef6f0, +0x17b73fea, 0x09c61fde, 0xeefdebef, 0xd92fffdc, 0xd7e05007, 0xfe42d9f8, 0xd2feaae6, 0xf3001cd9, +0xee12c727, 0xe5d90cea, 0x2c21ac01, 0xe3de0b08, 0x265703fa, 0x1a410d30, 0x2f1f5719, 0xe742bff6, +0xfc2e23e5, 0x2746b105, 0x262efe1d, 0xe43ff4fb, 0xf01018e9, 0xd9014e18, 0xef043927, 0x23fb0524, +0x27094e23, 0x19ce41e1, 0xf70a200b, 0x1daff5e3, 0x274eb22a, 0xf612cd26, 0xf903df17, 0xe3f1ba25, +0xe45bbc18, 0x062a2132, 0x11bfea01, 0xf4fe34e7, 0x0637ddda, 0xede0c510, 0x02022509, 0xfbbf2c26, +0xeda8c619, 0xecb0edd0, 0xeb00d0d8, 0xf054edf7, 0x0356182c, 0x5fd3252e, 0xd74e3805, 0xc2100026, +0xe4161517, 0xf106f412, 0x26f219df, 0x440bfde5, 0x1b031d1d, 0xe6c4f325, 0xd349f213, 0x2b0605df, +0x4bdbfcdf, 0xe32e23fd, 0x49460a07, 0xcff022e3, 0xf9c2f617, 0xdfbc2116, 0xed2f071b, 0xd6a91406, +0xc33bfe2f, 0xc623eb12, 0xc2d0ee06, 0x243aeb07, 0x1fe8fb13, 0xffc409f0, 0x1ed6d814, 0xde4bf602, +0x31dcfb22, 0xf5da0afd, 0x0afd1e03, 0x1cc01fdc, 0xf82bf4e8, 0xcccce003, 0xfb0f0df4, 0xd2ff23e8, +0x0845fbd9, 0x16e821e2, 0xe5c5eef0, 0x14bbf4ec, 0x1cefeb1d, 0xffcd0de9, 0x5bbad90b, 0x0630331d, +0xf826dd09, 0xc62c2001, 0x00ed12fc, 0x16e8eb40, 0xf3e4120c, 0xc105e52d, 0xca181610, 0xb9e0f149, +0xfc011f28, 0xfc17233f, 0xc5f12318, 0xea1dedbb, 0x37e91240, 0x47d9f0b2, 0x19271e01, 0x09240ffb, +0x1edd1f4b, 0x02240ab3, 0xf3e6d90e, 0xfd03e425, 0xb12325b5, 0xb92cd9ac, 0xcee71e41, 0x47fa0bdf, +0x0fcb1fa3, 0xd406e7de, 0x31defcfa, 0x300af71e, 0xfa0608dd, 0x10e922c1, 0xb2f3e936, 0xf336265e, +0x0610e517, 0x16e72242, 0xab23134b, 0xbaddd34b, 0x1afee225, 0xfc100d38, 0x44e6dcbe, 0x02ede4c4, +0xbe19da41, 0x2ff1e6c9, 0xea14f8c3, 0xc21bed0d, 0xac1017e8, 0x35202df8, 0xd9f10dca, 0x03fc02dc, +0xf2e425bb, 0x29efe5e9, 0x3ce10046, 0x09153000, 0x0b061d2e, 0x1f1a0abe, 0xfb18d2c0, 0x0008d730, +0xf3071b30, 0x17dfeff9, 0xc5d09da8, 0x23d7fbaf, 0x07e921ee, 0x16e4ee45, 0x1624c304, 0x01f828d0, +0x20ddb8fb, 0x030325d5, 0x0424d5fc, 0x0216dac3, 0xe5040c23, 0xd71552ec, 0x28ef00e9, 0x16de13fb, +0x1520edb8, 0x0416db12, 0xdeee07c7, 0x07fc20d4, 0x08fadf2e, 0xf0f639ce, 0x30f0a7e8, 0xef0d39e6, +0x210d071c, 0xf2f235cb, 0x05192242, 0xd6095130, 0x2605ff2d, 0x19f8bf30, 0x1320eb48, 0xfbd72e51, +0x01d92701, 0xe127f601, 0xdc0a051f, 0xfdf1dc37, 0x0b0bce1e, 0x09ff1f29, 0x1507c3df, 0xf7e3e1bb, +0xdb204df8, 0x11e517bd, 0x3004a7dc, 0x0d1300cc, 0xf42134ea, 0xe9b9c1f8, 0x1fbef720, 0x07bcdfe5, +0xeccf3ce5, 0x17301109, 0xf2b4e6f7, 0x1258c723, 0xdaf84dd0, 0xd507ae1f, 0x011d2a22, 0xfd3f25f5, +0xfd4edb17, 0xf9cadf26, 0x144dc3f1, 0x09003124, 0xec503b27, 0xfed1d628, 0x0ffbc9f8, 0xf2e619de, +0xf832dff1, 0xf3d034f5, 0x1dea0b09, 0xe80e0fef, 0xf1dd38e6, 0xf8fce104, 0xde1f07db, 0x06ccdef7, +0x24554b0c, 0xcde0a6d2, 0xe2e0baf8, 0xeae6ef09, 0xe31b440f, 0xe5c6f3f2, 0xf447ccee, 0x1944f2e1, +0xe33fbb1b, 0xebad3eea, 0x0b1de32b, 0x0d24e50c, 0x18e711fc, 0x2830b110, 0x14413cf8, 0xe7e7f219, +0x1c49bc0e, 0x0734e0df, 0xe255f50c, 0x540045d2, 0xe3c0d3e3, 0xef0ff518, 0x34c017e6, 0x052af517, +0xff4e2402, 0xc4eadada, 0x00d3ed12, 0x4bc727fa, 0xaaf62211, 0x28e1d21e, 0x530a00f7, 0xfd40d5e1, +0x0dafdbe8, 0x2b0de5d6, 0xc722fce5, 0xca241007, 0xe1eb0efc, 0xfa63f8ee, 0x28dfdec5, 0x07420107, +0xccf6dfe7, 0xcb1df4e1, 0xd9def40b, 0x47ef0105, 0x1041e0e9, 0xe849e919, 0x27f10fdf, 0xef2c02e7, +0xe74fe805, 0x4cb7f2d9, 0xe7ee2521, 0xee430f17, 0x5b09161b, 0xdce2cd1f, 0x2343050a, 0xc22c05e5, +0x5ae7eafc, 0x9749cef1, 0x4dea4221, 0x0e2726ee, 0x24a8e6ff, 0xcc5afcd0, 0xbcd2f432, 0xf514e306, +0xbd0ee414, 0xc4dbe41b, 0x310a1302, 0x090cf8e2, 0xd5dce11d, 0x1c13fdfb, 0xeacbf515, 0xc5eeeef3, +0x3027eee9, 0x29420701, 0xc01700e6, 0xc4b7e910, 0xaaebec21, 0x129fd2ee, 0x2ec5eac7, 0x0eecf9ec, +0xe10b1aec, 0x57b3f7e3, 0x3939d1db, 0xcf42f012, 0x163ef719, 0xd5111217, 0xe92203e9, 0xf13eef05, +0xece5e6ea, 0x3301ecf4, 0xfff6f527, 0x39ded9e2, 0x110deffa, 0x262fe91a, 0xb7bd0207, 0x39d8dfe4, +0x0a2e1101, 0x2adf1ef9, 0xe52ffef9, 0xb70ff307, 0x1cdd22e7, 0x2b3bf4fb, 0xe4eb02ec, 0xd80bf4ed, +0xfebaff1d, 0xcd12271e, 0xfeee0aea, 0xc70726e9, 0x0f4fefdf, 0xd9b6e6da, 0xf5d300df, 0xccbc1dfa, +0x34270de4, 0xf0d3f400, 0xc649e7fb, 0x18deeede, 0x4d0f0600, 0x2bdf2549, 0x1f07fe21, 0xf0e6f60f, +0x0c1b1743, 0x0f0a1b1d, 0x46efe7e9, 0x0ae6e242, 0x0ff61f33, 0xae21e84a, 0x500ad6e2, 0xfcddd74b, +0x4a17ddc2, 0xe9d9dd4f, 0xc7241205, 0xb9e8110f, 0xda111ec7, 0x37e1fe46, 0x39f3f1e5, 0x000512d2, +0x0e05d92d, 0xfeec193c, 0xf519daf0, 0x1e291d52, 0x362bf652, 0xbce9f311, 0x30fae5d2, 0xe6fcf7db, +0x36ec0fc4, 0xddfa0f23, 0xf1f605e3, 0x1ed1e857, 0xb9def6b6, 0xc0091ee0, 0xcff1e8ca, 0x3fe5090d, +0xf3edea15, 0x23111b16, 0x0207fbd2, 0x1d2cdbac, 0xbdf6f6cd, 0x1a311ba8, 0x420af2cd, 0x1bde1ab7, +0xc512f217, 0x0fe3edbc, 0x1afe19d6, 0x200bf1e3, 0x100900c8, 0xfe1a29f2, 0x12c51663, 0xdfe606f2, +0xf7ea1f12, 0xf841df96, 0xe2f50a33, 0xe9f5efcd, 0x13fdeb25, 0x2524044d, 0x0d1a35be, 0xfaee23c7, +0xf0043824, 0xda200108, 0x04f225c9, 0xe101f7d6, 0xf30dcb36, 0x07fb22dd, 0xd1fba92c, 0xdf03f824, +0xd0efa8e9, 0xf5dd1d05, 0xfe332a5c, 0xf4221bb6, 0xe5ddbdb5, 0x1401c3d9, 0xfc12d33a, 0x1b14bd3b, +0xddf2fb36, 0xf0e13709, 0x12d9c601, 0x0cf7cce1, 0x1b06bc22, 0x22d6b7ae, 0xf7291eae, 0x15f81321, +0x1a04f2db, 0xe425f404, 0x05f32435, 0x060f2218, 0x0bd93250, 0x26f9fd20, 0xfe2fd5aa, 0x121deb45, +0xf6e731bf, 0xf90c2e1d, 0xe9dbc2b3, 0xfb0edd36, 0x1500b93e, 0x281db01e, 0x061ddff5, 0x2712fe0a, +0x0f461ae9, 0xe32f0b1f, 0xed691507, 0xf61d32bf, 0x0d24e5f5, 0xfae422fd, 0x03d126f4, 0xee0ac6f9, +0xd8a4b11e, 0xebf51335, 0x15f7c31e, 0x07fd22e1, 0xe40a0cdb, 0xdd04b5e3, 0xd1f25825, 0xec033de6, +0x12fee925, 0x292051da, 0xdff54af7, 0x34d0a4e3, 0xf53de209, 0xedeceb14, 0x0f4cca13, 0xe3340b25, +0xf023e90c, 0xf6be32fa, 0x2254fae7, 0x07c1212b, 0x31015816, 0x1bd24227, 0x1438edf9, 0x151f3df0, +0x02d1270a, 0x0fffc9f9, 0x0d4235da, 0xf638e2e6, 0x241ffc0f, 0x1d380b08, 0x1a93bdf0, 0x1b4f44bb, +0xfe57da26, 0x24c8fbd1, 0x1112c711, 0x1f3ef6eb, 0x00173116, 0x100311f8, 0xf2dfe7db, 0x292ce5f8, +0x46e1fffb, 0x45c71ef7, 0x2bc31c11, 0x4abcfceb, 0x00e0dd1c, 0x202628f9, 0x42f2f801, 0x0bb41b1a, +0x0d16e225, 0xd2f11a12, 0xf6ec06e6, 0x13ef1deb, 0x48c51617, 0x09ff1f13, 0x314b1f28, 0x1ecf0823, +0xf9adf6f7, 0xd4da222a, 0xef0f0501, 0xe146e8e8, 0xfe4308e1, 0x4803dbe5, 0x21bc2025, 0x0fc5f9e4, +0xd341e813, 0xa9d9fae6, 0xaaf5d1ff, 0xb546d2e3, 0x12e2221f, 0xbee715f6, 0xd325e60f, 0x1adc05fd, +0xf533f3fb, 0xf3d0e3f6, 0xcb11e5f7, 0x33fb0ce9, 0xa3be0b23, 0x29b8351a, 0x312f0120, 0xebbb0afa, +0xc31a12e3, 0x0c47150d, 0x21c21be2, 0xe1570715, 0x19e0082e, 0xc8f2f800, 0x2a0b11cd, 0xcd3001a8, +0x2f1e0bf7, 0x1909071f, 0x1c15f2ed, 0x4612f53b, 0x40021d27, 0x3108e821, 0x0c1108c7, 0xcd0d1ce4, +0xe51c0b44, 0xb4ebf412, 0xf1dddc04, 0xd6f7e6e1, 0xc32efe56, 0xfd10ea18, 0xdb2eda55, 0x48ff0329, +0xe2fce12c, 0xd5f9f5df, 0x2cda04fe, 0xdf0dfdcb, 0xc5e70741, 0xdf16eec2, 0x3ce1faf7, 0xcc0eec35, +0xfa28f450, 0xf1e72210, 0x451de7f5, 0x4dfde324, 0xfdf3db35, 0x31eadc3d, 0x391c09bc, 0xcc09ee1f, +0xdaf90dd1, 0xf6e1ff09, 0x22f8e1d0, 0xe70efae5, 0x13faf02e, 0x271e15ba, 0x03df0007, 0x11ff2627, +0xb8e9173f, 0x38ecdfec, 0xc4ed0f3b, 0xe4d91402, 0xd5010c29, 0x050400d3, 0x1e01ba26, 0xe3390b60, +0xe11d0a0c, 0x2adaaefe, 0x1222e906, 0x0ff6e7ce, 0xebeb3cc4, 0xfd082b31, 0xf52a1e52, 0xf1dde7b4, +0x0ee8e6c1, 0x0c2ae352, 0x0df3cbca, 0x131bebf2, 0x23214bf9, 0xd6f5ad1c, 0x21fff8d6, 0xe3e00a48, +0xf00119d8, 0x03d02ca7, 0xf1fcc92c, 0x0c0b1b1c, 0xf4daccb1, 0x131ac50f, 0xd6295250, 0xe4f6f3e2, +0xe9f91221, 0xe6e0be49, 0xdcebb5c3, 0x1f1bf8f3, 0x27f6b2e3, 0x1a1c0e45, 0x0504d224, 0x0918e20f, +0xe4dabb4f, 0x0a001d27, 0xec193b42, 0x08e9d1c2, 0x0c09e51e, 0xff28d700, 0x0d0ae5cf, 0x0704202c, +0xfce0d408, 0x0c0b341e, 0x27ee00ea, 0x1206c7d1, 0x1000e9d8, 0xf8000830, 0xe334f5e1, 0x0bc81ef4, +0xeed33b10, 0x280050fb, 0x02d1d928, 0x0e3be608, 0xdc130412, 0xf6ffceea, 0x03142a28, 0xfeea2aec, +0xfdc4d613, 0xf1151aed, 0x1425c4ee, 0xfdd0db04, 0x10df1807, 0x0af7cd07, 0x20c108e2, 0xf61bce17, +0xe6e6f10e, 0x03d0da0e, 0x20edf808, 0x37dc60eb, 0x1bfd0e04, 0x18b73f24, 0xe8f91021, 0x0ed31ade, +0x1fe50afc, 0x07e020f3, 0x0c2ccb07, 0xedd91405, 0xf700e202, 0xdce903d9, 0xf4b31c12, 0xfcbbd425, +0x08d431e2, 0x1f00b905, 0xd04158d9, 0xd2acabe8, 0x18e911d3, 0xedca3c12, 0xe467bb0f, 0xd9c350c2, +0x00c12816, 0x082220e9, 0xfe692bfa, 0xfc322cc0, 0x0d1ae5f5, 0x1de446f2, 0xe136b80d, 0xf2001af2, +0x1be3bdd8, 0x013e29f4, 0x272301ea, 0x0a021f04, 0x22d1b5da, 0xedafecfa, 0xf958d0d8, 0xb1f48ad1, +0xf816d01b, 0xe7caf2ee, 0xf7e11ef1, 0x0ae31df7, 0x0feb180b, 0xece61513, 0x3f10670e, 0x0baa1c17, +0xf233e6d2, 0xce46a7f5, 0xe6c642e2, 0x033225ee, 0x0b3de20a, 0x00f7d814, 0xf8cccf20, 0xe2ee0bf4, +0xd93601ea, 0xefdf3a0e, 0x2202fb07, 0xe5c70c25, 0x01da29ee, 0xf8182101, 0xdef7b610, 0xeb0aed1e, +0x0017d8e2, 0x2fa4a9ef, 0x0c25cd35, 0x05f62304, 0x08e7e01f, 0xe11409f2, 0xf5e81ded, 0xeefeeb10, +0xfe122b25, 0xe9153f16, 0xe34f44ed, 0x1cca0c27, 0xf3dbcb0e, 0x08c231fc, 0xe2cfea00, 0xf031f5f7, +0xc7e91809, 0xdfe5ef10, 0x1234fa0e, 0xee28ea0c, 0xc6061700, 0xccb8eede, 0xed190b20, 0x21441510, +0xb1e007e5, 0xe1c7d9f9, 0xcacbf611, 0x3acb0f0c, 0xd7e4110d, 0x41c2010c, 0xef0418ea, 0xffc917dc, +0xc7e4270f, 0x40e3eff5, 0xaf4c19f6, 0x413f28db, 0x2e12e616, 0xd1e6fbeb, 0x1008f80f, 0x070719e0, +0xbdebdfde, 0x16f31b12, 0x2e1c12e6, 0x41eafaf4, 0xccc7e6ef, 0x26ff0bef, 0xc201fed9, 0xe733ead8, +0x373cf10a, 0xfdd5f213, 0xe1dcdcfc, 0xa9400904, 0xe432d018, 0x15340c0a, 0xcd3aedf4, 0x24210cef, +0x02d90407, 0x43c827ff, 0x27341c11, 0x1226fef4, 0x192cebfe, 0x280cf204, 0x96ff00e4, 0xb5dd4204, +0x1a2adc52, 0x1bf00ec8, 0x1d07f320, 0x09ed0a15, 0xb5e3200a, 0x12f7de1f, 0xe915eaee, 0x3bf11219, +0x3df1ec19, 0xe3ecec13, 0x1ce6f543, 0x0d18f4ef, 0x21ea1a13, 0xc8f507ce, 0x22e511be, 0x0b1cfaf3, +0xcfe8e3f0, 0xc104f6dc, 0x301817f0, 0xb825084d, 0x3fe920c2, 0xeff3171a, 0xc7e01809, 0x2efc10d3, +0xb602f92b, 0xdecb215c, 0x0d13053c, 0x3908e5d1, 0xb5efefc7, 0x44e0dcb9, 0xe8161def, 0x1de8f040, +0x2f140b3c, 0xc5ec07ec, 0xd403ed25, 0x0aeefce9, 0xd412e23b, 0xf4f804e0, 0xf218e5c1, 0xca17e612, +0xc5050ed3, 0x1afc142b, 0x2e2c0dab, 0x39ecfb13, 0x1de6100e, 0xf81ef4ba, 0xf600211d, 0x12c03968, +0xe8f90f2e, 0xfbe42c44, 0x260bfee3, 0x051b22be, 0xe50842df, 0xfef6d632, 0x130514d3, 0xe6def24a, +0x09213107, 0xdfddfa05, 0x0418d310, 0xf00739d1, 0x0afa1ed3, 0x1bfff4d7, 0xfddc2b4c, 0xf1e0c9b8, +0x04e1d4f6, 0xd423acb5, 0x320c5b1b, 0x160eefcb, 0xe11347ec, 0xedfc16d4, 0x10f8c7e0, 0x220e4ae6, +0xf0f5c834, 0x272002f8, 0xdf0df9cb, 0xe02ff8a9, 0xd7df5106, 0x152512b2, 0x02fe2bd6, 0x1707112f, +0x2519b2bf, 0x051e2eba, 0x0bdbcefd, 0xf1eae8c2, 0x08dee007, 0x02142b3c, 0x001628ef, 0xe21fb9f6, +0xe0e9f7c1, 0x31ed593c, 0xe4ff442a, 0x16e1c247, 0xe6e3be0b, 0xdbfc0324, 0x001110c7, 0xda2affe8, +0x05b6d302, 0xf94a2023, 0xe5350dde, 0x1ee0f60d, 0x282a00f8, 0x28ba50ff, 0x08e5211f, 0xf1f8190d, +0xd5abad20, 0x16fbc22c, 0xf8cc1fdd, 0xda1efe0d, 0xeb0ec3f5, 0x014c26e7, 0xf0e118dc, 0x1d11bb08, +0xe4bc44e8, 0xd4d0541d, 0xe43a44f8, 0x28445112, 0xdae0fde5, 0x13e3c6f8, 0xe11b09f4, 0x2ad2520d, +0xef59eafb, 0x1dcaf530, 0xde4d06f2, 0xf617e325, 0xf3c5e512, 0xf4b1cc13, 0xf91d2128, 0xd8affff4, +0xe54d43d8, 0x1a060fdc, 0xe7c3be22, 0x2441b415, 0xf3171a19, 0x180ff011, 0x30e4a919, 0xfaed2ff4, +0xd54253ec, 0xe7df0e1a, 0x232afa08, 0xe9f7effe, 0x1cb5f31f, 0xf4083422, 0xdfd4b620, 0x2223fc00, +0xddcf06fb, 0x6cc004f8, 0x041c4418, 0x4b04230d, 0xc6f92225, 0xf61aee22, 0xba001d0d, 0x06c3e2d9, +0xd5c2dd15, 0x240ffde9, 0x405a0319, 0xe3a01731, 0xde3c0b37, 0x2206faeb, 0x27fcfb22, 0x33af00dc, +0x3ba60a29, 0x2cc913cd, 0x4ef505f1, 0xccbbda1c, 0x203df4e2, 0xe407f815, 0x1a15f321, 0x18d30eed, +0x2d1f10fa, 0x3c47fb09, 0x3cf9eb1e, 0x4cf114e0, 0x0caadde6, 0x594ee5d2, 0x1ed6d026, 0x2229f602, +0x18f4faff, 0xddf5f0e4, 0xaf07fae3, 0xdc0fd8df, 0x14c3fce7, 0xf2d2edea, 0xb106e6fb, 0xdb0ada22, +0xc6bdfce1, 0xcdf111e4, 0x1dcf0ae7, 0x27c4f6f7, 0xe5e5ffeb, 0x20380d0c, 0x4b3e070f, 0x322300e9, +0xede80a0f, 0x65f7ebe0, 0xcbddc306, 0x572b0d53, 0xb0082f2f, 0x5d0729e0, 0xbf0f3537, 0x18c2199a, +0x57e3110c, 0x4512d23b, 0x18f01e17, 0xf9f3f11a, 0xc6f1df37, 0xeae1ef47, 0x0df111c9, 0x19fce6d4, +0xd3f90fd2, 0x08f70520, 0x15f4e0cc, 0xefff12d8, 0xaee6ea0f, 0xc1db2afd, 0x1cdaeab3, 0xbe0bf433, +0xe6e01b47, 0x5503f125, 0x30fbd422, 0x3723f7b4, 0xcb1510c3, 0x09fdf4d5, 0xeeee1e39, 0x0ae7ebc0, +0x02d81d00, 0xbc33db5a, 0xc501e429, 0x12e0edb8, 0x03f1eac8, 0xf0eb25c4, 0x09e4e7bd, 0xd71de0f6, +0xc8ee0017, 0xdf00f0d9, 0x4209f8e2, 0x3100e6d8, 0xe2190942, 0xd5070921, 0x0b0c02e3, 0xee00e4c7, +0xf5eaceef, 0x2011f7e9, 0xdf1106c7, 0xfede29fa, 0x1c1f0d0a, 0x1f0d08e6, 0x0d2ccc54, 0xf6e61df3, +0x121f1647, 0x1610c3c8, 0xded7b6af, 0x1ef8f61f, 0xf9172ef0, 0x391f60f7, 0xf3ebcced, 0x273050a9, +0x1305ebd3, 0x16221206, 0x12e5c6f4, 0x0006d822, 0xecfa152f, 0xf9f92f2e, 0x1fe2470b, 0x2425fc4e, +0x33285cb1, 0x070430d3, 0xe421bd48, 0xede1c5b9, 0xf4ea1c12, 0xe1c809a1, 0xfa05d223, 0xedf83a2f, +0xc4259d03, 0x1f180810, 0xdb10fee8, 0x11daeab2, 0xec2614fe, 0xb31e74ba, 0x14143b14, 0x07ff2fd7, +0xfb23d305, 0x25fffcda, 0xe7dbbefe, 0x13e43abc, 0x070c301c, 0xff0726d1, 0x16dc3eb3, 0x04ee2dc6, +0xdeff05d7, 0xf8dfd0fa, 0xde13fb15, 0x0b1acebe, 0xe2f1f6e6, 0xee1d3a0b, 0x3003572b, 0xf9d32fab, +0x0420ddb8, 0xcef05919, 0xd126a9b2, 0xdac9fda1, 0xe0ee083a, 0x30e259ba, 0xecf1ed19, 0xd0da58fe, +0x071f2e46, 0xe2da4602, 0xf3cecba5, 0xd8e20045, 0x18d6ef53, 0x11dbe9b3, 0x1819c00e, 0x09f930e0, +0xf2183640, 0x250bfd1e, 0xcae3a2bb, 0x17d13fa8, 0xefe6c8bf, 0x1ff8f620, 0xfaee22c7, 0x0afdce25, +0xd9010029, 0x1727c2ff, 0xf2221bf9, 0xd8e10147, 0x0726224d, 0xe0ea4811, 0x1a2e0ea9, 0x0d3635a3, +0xf0f1e7e8, 0x0c0234d9, 0x0ef8c920, 0xecf4151d, 0xe817c112, 0x0e0637dd, 0xddf305cb, 0x16e2ef0b, +0xf1c10000, 0xfd2ce717, 0x092ada04, 0xb929e202, 0x2902e1ff, 0xcecaff25, 0xd23c09f2, 0xee2107eb, +0xd9c0e908, 0xd3bc0018, 0xb3bc051d, 0x29ab261b, 0x0b0bff2c, 0xdf5ae41c, 0xe6ef07cf, 0x2ea10fea, +0x1db5fac8, 0x083a0cde, 0x4ff5e1ee, 0x4643281c, 0x08dee1e6, 0x1a0e1ffa, 0x2a30f11b, 0x0b4a02f8, +0xe1f5e321, 0xf5eaf71e, 0x21e3e312, 0xccfffaf6, 0x3932f5d8, 0xe3efeff5, 0x314df518, 0xbdc709db, +0x1336e5ef, 0xeb2bec0d, 0xc4a6ec02, 0xe3b2eccf, 0x33210b27, 0x1a2b0a08, 0xb536f304, 0x51f7def2, +0xa946d6e1, 0x0f1fd2e2, 0xd5cee6f8, 0xe60303f5, 0xdc25f225, 0xab360303, 0xd9aa2c0f, 0x30f9ffd2, +0x611ff8e0, 0xc6fa3909, 0x33241323, 0x023ff505, 0xc3f827e8, 0x1601ebdf, 0x35d91228, 0x0cf70d01, +0xbc271ce1, 0xbae8e400, 0xed09e310, 0x13ec16e2, 0xbffb1513, 0x3452e724, 0xff010cd7, 0x53ddd9d8, +0x18da2cfc, 0x3b28f102, 0x1ae1ed00, 0x051c0ff7, 0xe16b23f4, 0xfb4f0844, 0x091fdcd8, 0xbea1e109, +0xbcdc1a36, 0xd908e5fc, 0xd7f301e1, 0xbd1d02e5, 0x2319e5f4, 0xed0cfbf0, 0xed46eae5, 0x3442ebe2, +0x0323f4e6, 0xd62a25fb, 0xbec7fd03, 0x3841e6ee, 0xd3cc1018, 0x4d06faf4, 0x31422521, 0x15b1f8e7, +0x354b12d9, 0xe3ca0dde, 0x1fd30cf1, 0x25ddf705, 0xf5e8fdfb, 0x0a121df0, 0x5bc81e17, 0xd8e8cdef, +0x000000f1, 0xdfec4a13, 0x0d1e1cba, 0x0ff918d1, 0x00de274a, 0x07242105, 0xe5030cd4, 0xe0f40734, +0xee02ead7, 0x15dc3d4d, 0x0de3cc46, 0xe817f1ee, 0xfbe3d40b, 0x0d09cbe2, 0xd22256b6, 0x1904bf2d, +0xea17ee10, 0xe019f842, 0x15ebc33d, 0x39ef6217, 0xe6f8f1df, 0xf1001ad8, 0x1cf90dde, 0x19e40fbc, +0xe9e6ef0e, 0x0103d72a, 0xeffe3929, 0xf7d83151, 0x11273901, 0x28fdb0dc, 0x07f22f36, 0xddddb505, +0xe0e3f8ba, 0xf3dc354d, 0xe613f1eb, 0x09f4e2e3, 0xe70df1cc, 0x28d950b1, 0x001128e9, 0xfe312658, +0xf624cdb5, 0xee1217eb, 0x1ac04268, 0xd414ab15, 0x0d191b0f, 0x0e021b26, 0xe10cba1b, 0xfc09d4cf, +0x1b2b0c53, 0x0be8343f, 0x04d92c01, 0x01f4d733, 0x0d01cad8, 0x16f2121b, 0xddebfbc4, 0xdff2fa37, +0x1dea0bee, 0x1b0cf4cd, 0x2ff7a9cf, 0x13da15fd, 0x35fb5c2d, 0x0b0fcd36, 0x0e19ca40, 0xef1e380b, +0x0303d5db, 0x02fdda24, 0xe30845e0, 0x14fc1424, 0xeef115ca, 0x04e5d40e, 0xde2bfaad, 0x2ddaaaff, +0xf2e91a3e, 0xd51e530b, 0xdde405f3, 0xd908b11f, 0x0fe0c807, 0xecf0c318, 0x20064723, 0xf814d03c, +0xe9081120, 0xe0f808cf, 0xdbf4b235, 0xfdf225e6, 0x10da19fd, 0x170fef19, 0xedc53a64, 0xea133e16, +0x1725effd, 0xe4fbbcdc, 0x0208dadf, 0x1ff50933, 0xf706d0d2, 0x0901cfd6, 0x130515d2, 0x1e1e0909, +0xe01247e9, 0xbe3f0000, 0x35dbe517, 0xdfcd0d03, 0xff29f9f5, 0xb0c0d900, 0x30eed8e7, 0xdabff816, +0xaa4203e7, 0xd9f62f19, 0x5453ff1d, 0xdbd32bd6, 0xec07fd06, 0xeedd14df, 0x462debfc, 0x173e1d04, +0xe423efea, 0x4f6cf505, 0xd701da45, 0xb13602d9, 0xa2fd28f3, 0xeec7c9dc, 0xfae21610, 0xd4c9de0a, +0xd5db04f0, 0x1503fd03, 0x0360eddb, 0x2abf2639, 0x34fdfde7, 0x462bf5dc, 0x3c1f1e04, 0x1d04ec09, +0xbdd20cdd, 0x1c561a06, 0xfb1d0c2e, 0x19b824f6, 0xd1f4f120, 0xbe23061b, 0x1dcb1bfb, 0x22f10cf4, +0xc5f8fa18, 0x5227ec20, 0x33d1d501, 0x03caf6f8, 0xd3fc260d, 0xb9150523, 0x5811e114, 0x211bd0e8, +0xcd1d08f2, 0x44c2f50b, 0xf3d6e417, 0x0535e5ff, 0xfbccddf2, 0xc102def4, 0xf5d01825, 0xfb27e207, +0xeff62300, 0x3a1417e1, 0x17b6efec, 0xddd9ef23, 0x2ddefb02, 0x0a2bfbfb, 0xfb561e03, 0x2ceedd2e, +0x17f50416, 0xc4f3efe3, 0xb9e2ece5, 0xe3e71f0b, 0x47e5f4f1, 0xebdbe1f3, 0xe4e514fc, 0xfc280c0d, +0xfdbb2501, 0x0508251c, 0x0d0add1f, 0xcb201b1e, 0x2e480df9, 0xf3f2061f, 0xc7181b19, 0xe42310f0, +0x32b3f4fc, 0xe5c70b26, 0x1ceef3ee, 0x3def0d16, 0x2309ec18, 0x3544fb1e, 0x5050f3e5, 0x000d29d9, +0xef5b271a, 0x57b0ea33, 0xecc6d0d7, 0xd619ecee, 0xc60101f2, 0x3954efd9, 0xf2db11d4, 0xf7b31afc, +0x433d20da, 0x0000e515, 0x06f5d234, 0xdef0b6c8, 0xed1dc4f4, 0xe8fef126, 0xe215bb3d, 0x14fcedd5, +0x0deb1aed, 0xe528f2b0, 0xe0e84840, 0xdd1ab542, 0xe91ec10b, 0xe11ff648, 0xbf0968e0, 0x0ddccab5, +0x0b1ae30e, 0xe6020ed7, 0x07f42f33, 0xf4e91cef, 0xfbe9d3c0, 0x24e4fb44, 0x12fa162f, 0xe9063f21, +0x01f6d931, 0x22f806df, 0x1d04f423, 0xdffc4923, 0xe4eff3c8, 0x190af11e, 0x18f4111c, 0x170311da, +0x16f73e20, 0xdbcb04a2, 0x0dfc1bdc, 0xedf43a1d, 0x10cfc8a6, 0xfb402d68, 0xf2273500, 0x21fc4a24, +0xfae2d345, 0xe7001028, 0xeaf1ed36, 0x00f329e6, 0x0d1234c6, 0xdf27f8ff, 0xecf8eb20, 0x09f432e5, +0x0bfacdde, 0xeffbe9de, 0x14e73c40, 0x2007f821, 0xe005b9dc, 0xeafe3eda, 0x0701d0d7, 0x29015027, +0xe9fdee24, 0xe31ff408, 0x24244db4, 0x03f5dcce, 0xf3151c3c, 0xf5da1e01, 0xe01b0843, 0x02db2b4c, +0xdcf34c35, 0x07f620e1, 0xf40e1b19, 0x0cffcc27, 0xf9112ec8, 0x3c0e9bca, 0x1615123e, 0xf7ddcf04, +0xfe0326da, 0x2a26524d, 0x09031fd4, 0xe6dc0dfd, 0xfbf32ecb, 0x18121115, 0x231d050a, 0x011a2abe, +0xfc0bdc34, 0x0a0632d1, 0xe3dabbb2, 0xeee2c60a, 0x11eae9ee, 0x031ad6bd, 0xf6131e3a, 0x0d22cbb6, +0x13103c18, 0xf5161c3e, 0xf01ae8f2, 0xf306e42d, 0xed04c623, 0xf1211a08, 0xdbf34d34, 0xe9e6eff2, +0xd90db1cb, 0x0aedcf3b, 0x3f110000, 0x2dffe818, 0x20fbfbda, 0x08fc07dd, 0xd6182025, 0x9d27fff0, +0xb5e63c01, 0x295323f2, 0x06d401d5, 0xa0ee22fd, 0x15ec3816, 0xba24ed14, 0xe13be1fb, 0x39360aed, +0xb60f10f3, 0x15e2dee7, 0x174aedf6, 0x1146ee22, 0xee16171e, 0xed1d1512, 0xd7fdec0b, 0x110200db, +0xfb5116db, 0xf50923d8, 0xd7bae3e1, 0xc960021e, 0x14400e38, 0x33d01419, 0xc737f407, 0xdc6011f0, +0xebfd05c8, 0x120ced25, 0x0845e9e4, 0xfaf620e3, 0x394923e1, 0x5217efde, 0xc1c8d6ef, 0x030fe810, +0xf418db19, 0x0ff3e510, 0x0ec7e71c, 0xfbaae5ef, 0x27c823d2, 0xc3c6ff0f, 0x0ed0ecef, 0xbbd11b07, +0xb8411cf8, 0x42e6e1e7, 0xafcee50e, 0x29bdd8f7, 0x4cf6ff1b, 0xefe2dc1e, 0xc518e9f5, 0xc313ec10, +0x0ccfea14, 0xba27e309, 0x0c9ae201, 0x19d31bc3, 0x4aec0e06, 0xfa0e23ec, 0xeac5211b, 0xe9281313, +0xdd711101, 0xed41fab7, 0xbb2f14e7, 0xd6a1e4fa, 0xb6b3fec9, 0xd0b32324, 0xcbdbf8db, 0x4fb60cfc, +0x3503d9df, 0x06c3f225, 0xd2c7de15, 0xbefff9ef, 0x0aec1ada, 0xbe401eec, 0xc92fe519, 0xb6bf0ff9, +0xddbcde1a, 0x2047fb1b, 0xcbbaf91f, 0xa7fdf3e2, 0x30eed025, 0x435b07e9, 0x2efc1b34, 0x0ac70723, +0xbefce311, 0x4ca719dd, 0x1d04db31, 0xead20a24, 0x2a5b12fa, 0xc613ffce, 0xb52c12ea, 0x03bd23fc, +0xc9b9dc1b, 0xd0e40e1f, 0x000007f4, 0x0cde354a, 0xd2dcaafc, 0x21d908fe, 0x1bf5f3ce, 0xe40d0d1a, +0xcd06a42f, 0x0fe7180f, 0xda0c4e35, 0xfe28db00, 0x13efc5e9, 0x08032024, 0xff35275d, 0x22eeb639, +0xeb2bc3ad, 0xe9ec3e14, 0x2523fd06, 0x06f3d1cc, 0xdb2903b0, 0x0fdfc8f9, 0x18e50ff4, 0x0efbcb2e, +0x11fae8d3, 0x33e4a545, 0xe22af6ae, 0x1f10ba38, 0xe30bf633, 0x0c06e322, 0x0eeb1b3e, 0x09f1cf18, +0x10ef1939, 0xe9f21036, 0xee17e9c2, 0x1aec0f13, 0xeff4ea1c, 0x16dbedb4, 0x03eb253d, 0x1400c5d9, +0xfd00db28, 0xdcda03b2, 0xcdff5a28, 0x0bd6ceae, 0x24f8fbd0, 0xfdecd5c4, 0xdfe4faf3, 0x0cedccec, +0x0807e030, 0xf6213208, 0xebdfc208, 0xf312cbea, 0x1d21ba49, 0x03f02517, 0xda0e4ee6, 0x14ed3dea, +0x090ae132, 0xfdefd5c7, 0xe019f9f2, 0xf4121dc7, 0xe7febfda, 0x27ee0116, 0x18fbbf23, 0xff04d625, +0x252d4eab, 0x271d000c, 0xf8ed3116, 0x2009f831, 0x1b17bd3f, 0x201209c6, 0x2af95220, 0xee0816d0, +0xd6e352bc, 0x240f04e6, 0xf21d19f6, 0xf41b1cf3, 0x171b3ff4, 0x0c26cd4e, 0xfe05d6d4, 0x021dd7f5, +0x22cf4a59, 0x14efec39, 0x1c0abbce, 0x12d3c655, 0xe8fb4023, 0x07e82fc1, 0xdae6b30e, 0x1d0af5e1, +0x27f0ffc8, 0x01ed29eb, 0xe625be02, 0xdfe2490b, 0xf11c1945, 0xca21a1f9, 0xe91810c0, 0x20190842, +0xf1fec929, 0xe2e9b9ef, 0xe814c0c4, 0x6d0c0000, 0x14d0451b, 0x11aeebf8, 0x393d16d6, 0xa9dbeeec, +0xdbf7d103, 0xa45e0320, 0x1ab1cbc9, 0x0b25f2d9, 0xbf111e02, 0x2b2e1818, 0xb2000305, 0x1608dbd8, +0x210e1220, 0xfdf1fa1b, 0x2d9e2519, 0x01d0fcc7, 0xd5b8d9f8, 0x08d603df, 0xce25e0fe, 0xea000a03, +0x234b12d9, 0xe72ffa23, 0xedf00efa, 0x361614e9, 0x2c3a0e13, 0xe114fd11, 0x2a3a0915, 0xbfe3fdef, +0xd95a180b, 0xbd20fece, 0xbd391c07, 0x610de4ee, 0xbb0d38e4, 0xee08e4e5, 0x10fe1720, 0x2f0fe8d9, +0xc92af919, 0xdf24f001, 0x64f0f9fc, 0x1551c317, 0xccf61229, 0xbd39f41e, 0xe9ece511, 0xf9f21214, +0x1dc5e01a, 0x3ac00aec, 0xbf1012e8, 0x2f00e8e8, 0xdffc07d9, 0x1477f9dc, 0x0c301450, 0xfb0d1cf9, +0xd92ddd1b, 0x43df0006, 0xcc541cf9, 0x0de40d2d, 0x043c1b0b, 0x48ec24eb, 0xcc27e0eb, 0x2ea1f401, +0x02e00536, 0xec2c25f7, 0x380dec04, 0xe9faf01c, 0xb01b10de, 0x41f527f3, 0x31a6191c, 0xdfeef6ce, +0x4632f916, 0x2a2c1ef6, 0xff17fefb, 0x3fc2d9ef, 0xc3e717eb, 0xe70915f2, 0xc7110fe1, 0xe9caf0e9, +0x3a26eff2, 0x44361301, 0x40f11b0e, 0xedc81818, 0x09d8ebf1, 0x2013e2ff, 0x35ccf8eb, 0xaa2c0ef4, +0xb033d2fd, 0x03dd280b, 0xd0d926fa, 0x4203f7ff, 0xcff31a25, 0xe8cef7e5, 0x403ef00a, 0x0bd8e817, +0x30251d00, 0xaa10f8fd, 0x3ea7d218, 0xa708e9ce, 0xdde231e0, 0xc3e5040a, 0xeda7140d, 0xb51715cf, +0xd2dc22f0, 0xedb70603, 0xfc19ecdf, 0xb2e2ddf1, 0x1e06d90b, 0x28ee0a21, 0xb3c90117, 0x21212510, +0x31b8f8f8, 0xeae0f8df, 0x48bbeef7, 0xd20e20e2, 0xd029f9e6, 0xdd01f9ff, 0x0726fa27, 0x3a0edf01, +0x160ced1a, 0x1cb412e3, 0xd24c0b23, 0x01b50624, 0xe1e526dd, 0x321ef7f3, 0x200e0af5, 0x362bf819, +0x04130efd, 0x1af72315, 0xc7f3f31f, 0x2ad8f0e4, 0x47f1fe00, 0xa9411fe6, 0x0be92ee7, 0x1f2be4ef, +0x14c30804, 0x5c1214ea, 0xf1cd34ea, 0x5535e7f5, 0x3410d30e, 0xfa2e0c18, 0x4218defa, 0xbcbd19f0, +0xe6fd1c1c, 0xf5fe0eda, 0xf5e0e426, 0xb1041d08, 0xbf06da24, 0xc0d41823, 0xcfaae8fc, 0x25b0f82e, +0xd9d5fc29, 0xc8ccff02, 0xf04cf00b, 0x32d0e725, 0xda090af9, 0x5604ff1f, 0x2fef2edd, 0xdce2f818, +0x1fd30409, 0x4c300afb, 0xe018dcf8, 0xb54308f1, 0x281fdde5, 0xc202fff8, 0x3ca11525, 0xc241ecc9, +0xc06d15e7, 0xc11ce8bb, 0xf51a180c, 0xb43b1cf2, 0xcd2b2413, 0xefd3f4fc, 0x3e30e8fb, 0x43df1708, +0xd44ae4fa, 0xe912fd22, 0xeb16eee9, 0xe1aa12ee, 0xe2be0a2e, 0x24f90a19, 0x02b1fbdf, 0x4f33d9d9, +0x1fe4280c, 0x0336f60c, 0x09bc250e, 0x2b36e11d, 0xf2fffd0e, 0xfd33e727, 0xd2ecdb0c, 0xebdc0614, +0x16ec13fb, 0x2427ee15, 0x0a5d03ff, 0x2f0a1e36, 0x08e20000, 0xe0eaf813, 0xfe1c2bf5, 0x300c57cc, +0x13e8c510, 0xe2e50a43, 0x0cdd35fa, 0x040f2c37, 0x0001d828, 0x4404932d, 0xdaddfe4c, 0x012d29ab, +0xee1ec6ba, 0x1b0ef236, 0x16f6121e, 0xd4faab2d, 0xe9e5eef3, 0xe80d4035, 0xe5f642ce, 0xd8010028, +0x141d3c0c, 0xe7110fe9, 0x14eac5ee, 0xed1c3af3, 0xd616523f, 0xdef7facf, 0xf2e01b47, 0x2211fa16, +0x290cb033, 0x29e05007, 0x300d58e5, 0xf82b30ad, 0xf01c370d, 0xd4ebadc4, 0x080c1fe5, 0x1324eb4c, +0xec13c5eb, 0x08fbcfdc, 0xe8ff4029, 0x08e920ee, 0x02fddb2b, 0xf20ce6cd, 0xe6184140, 0xfd1624c2, +0x27f0b0e8, 0x1b15423d, 0x151c3d0b, 0x21e0f9f7, 0x16e1c148, 0x11161812, 0x0b0e3435, 0x07fb20dd, +0xe30045d7, 0xebecc33c, 0xe10b471d, 0x1ceb0cc3, 0xf020e809, 0xf412e53a, 0xe905c124, 0x2eef56c7, +0x10dbc804, 0x13181541, 0x0913cfc5, 0x0be3cef4, 0x34ed5dc5, 0xff10da37, 0xd827ff4f, 0x11d81701, +0x13e81540, 0xcddd5a04, 0x090ce233, 0x22db49fd, 0x3d09651f, 0xf5edcc3b, 0x25224c06, 0xf7c8e15f, +0x190dc0e4, 0xf81fe009, 0x08272fb1, 0xec11c417, 0xefe7e9f2, 0x13e1eab9, 0xe6fdf325, 0xf1f01918, +0x0c27cb01, 0x0e0d36e6, 0x1a0bbf1c, 0x1122c649, 0x0020d9b8, 0x2120f9f9, 0xd50f531a, 0xdbeeb416, +0xf0fdc9d5, 0x1ee20bf6, 0x24ebfcec, 0xebf1c31a, 0x00003030, 0xd1570709, 0x5a3707d1, 0x482033f1, +0xfb262008, 0xb8fddcfd, 0x33301fdb, 0x39fff4f8, 0xf9ac11d8, 0x0115e0d4, 0x0401d814, 0xda3323d8, +0xc72803f5, 0x0bba1100, 0x48e11de2, 0xe81f1f09, 0xfe3af00a, 0xd105da12, 0xacc1fa23, 0x1bc92d16, +0x05f90cf1, 0x49f52421, 0x35e6211d, 0xe52f0df3, 0xdf50f3f9, 0xf6c3f9d8, 0xf3df1e15, 0xc45e1bf8, +0xf9c6ebcb, 0x07ec2111, 0xace8dfed, 0xc606d311, 0x3f161223, 0x3e071712, 0x5e3ae9de, 0xdef5c912, +0x34fcfae2, 0xd0e9f525, 0x0431f9ee, 0xc2bdddf6, 0x0aee161b, 0x0827e217, 0xdf322001, 0xcfbd06f5, +0xe0daf71c, 0xde3bf802, 0xe11c0613, 0x013ff7f3, 0x1df1d917, 0x46ecf419, 0x40d41e15, 0xd22618fb, +0xa1360702, 0x1d25c8f3, 0xe0310a04, 0x0939f8f7, 0xe8a51e11, 0x1fbe10cd, 0x4126091b, 0x3c2ee6fe, +0xdae21406, 0x2624fff6, 0x37380205, 0xb4db0ff1, 0xfddd2504, 0xe51eda05, 0x09c1f30a, 0x15f420e9, +0xb138ed1b, 0x26142811, 0xc41e0315, 0xba1eedf7, 0xe3ce1e0a, 0x9cef0b0a, 0xfbe2c4e9, 0x370f23f6, +0xc4f9f019, 0xdb5fece0, 0x140f03c8, 0x1ec115e6, 0xbecd0be8, 0xd1cce60b, 0xc3b007f4, 0xe9ca1628, +0x1adbf00e, 0xbdbc0efc, 0xc930e51d, 0xb42cf2f9, 0xc2a723fc, 0x35fa17cf, 0x539b0edd, 0xfaffd63d, +0xe4b62228, 0xe3fc0d22, 0x4ab40a24, 0x3a0cdd24, 0x13bc121c, 0x15e30000, 0xfcee2c3a, 0xf7f2e1e7, +0x1528c2b0, 0xe51c0df4, 0xdc03fbdc, 0xffe12af8, 0x1c294451, 0xfc18db10, 0x1f0048d8, 0xdc06032f, +0x0bdee2fa, 0x04162cef, 0x09e7d0c0, 0x0c24e34b, 0xbddc6bb4, 0x09f72032, 0xe11d4746, 0x0bf2e4e6, +0x2122fab5, 0xd60653d3, 0x30dda84a, 0xfded2b3a, 0x1de80b10, 0x08ead13f, 0xd3055522, 0x2610fe37, +0x34fc5cd4, 0x1fecf614, 0x0cf41ce4, 0x16f7ed1f, 0xe6150e12, 0x0c00e427, 0x11ee3917, 0x0eee363a, +0xe3de0cfa, 0x24054c23, 0x2efb5623, 0x27d00058, 0xe71441eb, 0x231b4bbc, 0xf3141c3d, 0xb9f66f32, +0xd3ecab3c, 0xeb04122d, 0x1a1ebef7, 0x0c0fcce6, 0xeae81240, 0x14ec3c3c, 0x41ea9711, 0xfeded64a, +0x00ccd95c, 0x22fef926, 0xec063b2d, 0xeae93f3f, 0x1ffff827, 0x25264cb3, 0x180a101e, 0xdcfc0523, +0xf1dbe7b2, 0xfbe424f4, 0xe9053ed3, 0xdafc4ddc, 0xea23c2b4, 0xf2fe1a26, 0x252403fd, 0xfd0e2537, +0xd01b59f3, 0x101be80d, 0xd9dd4fb5, 0x1edf0a08, 0x27e702bf, 0xd9f400cc, 0xf914d214, 0x1debf53e, +0x1524ee05, 0x121f3a0a, 0x01dcd604, 0x3901a029, 0x1dfef629, 0x11dc1804, 0x23f00538, 0xfd1124c8, +0x32215a07, 0xf7173112, 0xdc0e4dc9, 0xe00008d8, 0x1c1dbc0b, 0x200c09cc, 0x0805e0dc, 0x14e03bb8, +0x2bf3ac36, 0x2f0aaacf, 0xeadc124c, 0xff16d8ee, 0x01f3da1a, 0x0000163b, 0xfaf213ed, 0x0a08221a, +0xcebf1f20, 0x46dd0a1a, 0x3fcd1e05, 0xe7cee8f5, 0xf23410f6, 0xedf0e50b, 0xef42ec17, 0x08fdeae6, +0x975ce0db, 0xca19bf34, 0x241b0df0, 0x27bd04f2, 0xf8f8001b, 0xccbfdfe0, 0xf607f4e6, 0xec4ce3e0, +0x17d614db, 0xdfd0effd, 0xf751fa07, 0xc7081fd8, 0x225011e0, 0x16e7fbd8, 0x2af71210, 0xef32fd1e, +0xf5e9ea0a, 0x11f2e411, 0x26cceae5, 0x4e00fff3, 0x724f2728, 0x29ea4a26, 0x1aca00ee, 0x460df30e, +0x201c1ee4, 0x46dff7f3, 0xd3d41ffa, 0x3beb04fc, 0x4401ed14, 0xd3b3e4da, 0xc03cfb25, 0x1c1de7eb, +0xcccc0bf4, 0x2ae10df5, 0x402802f7, 0x0117e901, 0xc201d9f0, 0xf7fc15d9, 0xfedc2024, 0x24c82503, +0xc1fafdef, 0xfef2e9de, 0xd4c1da1a, 0xaff104e8, 0xc12a29e6, 0xb625e8fd, 0xd7c72204, 0xb5c4ffef, +0xcd45dd15, 0x25330a1d, 0x3ff0030b, 0xa9b2e917, 0x25c12fd9, 0xc23a0218, 0xd2bfea11, 0xe3170518, +0x282a0aee, 0xda0300ff, 0xfeb6fe26, 0xebd5db22, 0x1ce6edfd, 0xe4d4f5f2, 0x16f2f404, 0x26c8ee1a, +0xe2e9feef, 0xa8fbf5ee, 0x01073123, 0xe843d920, 0x1416f0e6, 0x0b24ebed, 0x5123e204, 0x0000d7fa, +0xbd290000, 0xbef21bfe, 0x19eee7e7, 0x0000f1e9, 0x00000000, 0xfa490000, 0x17e0df22, 0x000012f7, +0x00000000, 0xbadc0000, 0x0f3fe1fd, 0x00001a17, 0x00000000, 0x31c60000, 0x09ee0000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 + +hard_output0 = +0xcddb5769, 0x627f0a91, 0xbcf446fb, 0xc930d133, 0xdd7b82f0, 0xbd591856, 0xb73094b3, 0x32436132, +0xac268fdc, 0xeeb03845, 0x0087cc79, 0xd372eb62, 0xb595a1bd, 0x511605f7, 0xa428b83f, 0xfb9380a8, +0xe43c2efb, 0x0a04017a, 0x674608a3, 0xc2464812, 0x208bc693, 0xb03d0580, 0xdc62b803, 0xc2808805, +0xccaf7f16, 0x848dacef, 0xedf69b49, 0x222d2bd9, 0x78ffa7a5, 0x749cb755, 0x9f4abc69, 0xa27f3a72, +0x9652d070, 0x8a83be54, 0xa9820b7f, 0x8b00e52a, 0xb02159e4, 0xeddabc6a, 0x47b69064, 0xf161d47f, +0xc243b0df, 0xebb2ac39, 0x4fd6cddb, 0xc0d38ba3, 0xd9ddef17, 0x8c638875, 0xa132cfef, 0x5893a4f6, +0x1a29ce5f, 0x9072f3eb, 0x486bda03, 0xf5250cda, 0xdc10edbc, 0x392e6b7f, 0x4664de06, 0xfde2f3f2, +0x37c7d442, 0x32b60d29, 0xbcb7c8ff, 0xa1542cd0, 0x91b36e18, 0x231f9e19, 0x97cbc4a6, 0x51ceb1df, +0xc02dedf8, 0x4bfd9728, 0x569deda7, 0xc0790501, 0x132e47b0, 0xb8229185, 0x81cbe103, 0xa30dbf5f, +0x72c54db6, 0x4abdbdf7, 0x557d21d3, 0x3b75be17, 0xf540476b, 0xc48d78e1, 0x767d44ec, 0x87dc6193, +0x7a4c52fc, 0x306c3548, 0xa66e51b3, 0xfc627273, 0x5f349268, 0xf5e74974, 0x232388b6, 0x01d8e49e, +0xbf29f75f, 0x378a9d02, 0x59a45656, 0x46fb25b1, 0x75219b62, 0x4f0d6f71, 0x220c6763, 0xdde47b70, +0x8c0dce8f, 0x3e3db100, 0xabad4262, 0x800eddcb, 0x32416450, 0x61d260ce, 0x6e2a0fac, 0x73cdb847, +0xeee01947, 0xe5a3a37f, 0xa710e211, 0x98ecaa36, 0x69197474, 0x6e73e13f, 0xb2100f2e, 0x7714b2fd, +0xec3d81cb, 0xc50e5194, 0x5c69a3d8, 0xce0bdc13, 0x08e7df1c, 0x5fb014a5, 0x75b5801e, 0xf57d3b75, +0xac5dd149, 0xa5096b77, 0x4b5da0af, 0x656506d3, 0x3823d675, 0x3840530c, 0x1c0980ba, 0x3bd4150f, +0x3df8c0f8, 0xa2834aa4, 0x479ea855, 0x4f4557b5, 0x15d730db, 0xe6809738, 0x0aec8f7d, 0xf5e2188e, +0xa9a0fac7, 0x785c3166, 0x049dd244, 0x72cada2a, 0x5655bab3, 0xff73ca65, 0xb3dc51f1, 0x259f555b, +0xdaa1933d, 0x0f716b8f, 0x1e29bde6, 0xbe8bf3a9, 0x908b5a60, 0x667636c0, 0xfce80b0b, 0xcf43d154, +0xfffc60f5, 0x324b1632, 0x1609e45b, 0xf8e09aed, 0x1a01f6ee, 0x013cea78, 0xdd89ba42, 0xfe5bca8f, +0x894e3aef, 0xda292f76, 0x457a93bd, 0x73a9c254, 0x7b5ca9d1, 0x947355ba, 0xb8a797c8, 0x3b5e97f4, +0x946889ec, 0xd49316a3, 0xa5383830, 0x55f63bbf, 0x47b3a4ca, 0x415bd007, 0x1227c60a, 0x3610fab0, +0xa68c75fc, 0xa1441c62, 0x68a02596, 0x6d083d07, 0xceeb9057, 0x915054bc, 0x6f2c7fa8, 0x62be6166, +0x963703a3, 0x5af6e614, 0x5bc3e4f0, 0xf517a95e, 0x408a2bfd, 0xab7fe7a8, 0x37aac030, 0x9956f2f5 + +k = +6144 + +e = +10376 + +iter_max = +8 + +iter_min = +4 + +expected_iter_count = +8 + +ext_scale = +15 + +num_maps = +0 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE, RTE_BBDEV_TURBO_CRC_TYPE_24B, +RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN + +expected_status = +OK \ No newline at end of file diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_negllr.data b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_negllr.data new file mode 100644 index 00000000..e946963a --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_negllr.data @@ -0,0 +1,645 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_DEC + +input0 = +0x7f7fb2e1, 0x467f05ef, 0x3cabba6a, 0x1f826cac, 0xbe817fe7, 0x584dab0f, 0xdcb55f81, 0xe47f915f, +0x37b1a534, 0x4b478145, 0x9db5727f, 0x7fa6943f, 0x819550ea, 0xd18eb72a, 0xa0ba9c10, 0xcf7f817f, +0x915a957f, 0x2c699d0a, 0x89815c0f, 0x7f569358, 0x1002587f, 0x54314130, 0x817f1b7e, 0x81a85e2a, +0x5981bdf2, 0x0c81b00f, 0x6781be0e, 0x297fb17f, 0xfd811fb3, 0x7f077fdb, 0xd37fc2ab, 0x817f1024, +0x81817f7f, 0x3d848ec6, 0x4486cd9c, 0x7f6d7f81, 0x1d7ffa93, 0x813648ca, 0xdc7f125e, 0xaa81a798, +0x60838ef4, 0x815fcdbd, 0x81cb9d93, 0x817f8188, 0xc8c3d697, 0x817af2d6, 0x36888781, 0x818117ee, +0x37a67fbf, 0x89f464b2, 0x7f1e7f81, 0x7fe5c17f, 0x8ca1effc, 0xed7f82cf, 0x4a7f5ac6, 0x7fffb8d6, +0x427f7f73, 0x747f7f06, 0xdfcd22a9, 0x7fb99981, 0x76f77381, 0x2cd70f12, 0xd8b138e1, 0x81a4ea29, +0x222db281, 0xc3a07fd2, 0x217f8177, 0xd1cd87c4, 0xd181679f, 0x7f7fdee2, 0x7c81a72f, 0x6e7f5a7f, +0x77b0c96d, 0xff7f8110, 0xe8c6ad46, 0x376ade0a, 0x447f687f, 0x5916d238, 0x88816b81, 0xda7f6d88, +0xe2b2b55f, 0x750f7fc8, 0xc624d47f, 0xc6837f79, 0x81de5272, 0xd0847f29, 0x81ad587f, 0x3e7f7f61, +0xc803e070, 0x237ff600, 0x8c01815e, 0x814e0e81, 0x81657eef, 0xab2a62bc, 0x73d42cb4, 0xaaf03a8a, +0x817fa57f, 0x5e7fe2c3, 0x6deda37f, 0x6e87818d, 0x767f6e4d, 0x5c76819a, 0x8182b6f0, 0xdcd3037f, +0x81184c7f, 0x7ff1501c, 0x22c7f159, 0x45a6817f, 0x9b53c5e7, 0x57966a5e, 0x0a7f7f81, 0x4a5c155e, +0xf37fff81, 0xad97f8f2, 0x052f397f, 0x79638181, 0x7f7f7f7f, 0x81e3fae4, 0x7659cf03, 0xbd7a5a81, +0x46bcab7f, 0xa57f69a1, 0xba818186, 0x38a347a2, 0x91a88ec0, 0xd76087fc, 0x81818b8c, 0x7fa3424e, +0x7f89ef7f, 0x81ac183d, 0x3aa8912a, 0xc4937da6, 0x7a4c136b, 0x81568e05, 0x8182812b, 0xebb17f1d, +0x84b87f4c, 0xf4eab6c9, 0x81034197, 0x8183bd0a, 0xa1b92676, 0xceca2ef1, 0x9b7f7f7f, 0x5d813181, +0x817fea35, 0x0a6c81b0, 0x9a559b4f, 0x93987691, 0xad819510, 0xa47ab324, 0xfa577f7f, 0xbba8ac3e, +0x6694b753, 0xee63c4d7, 0x17b4ce81, 0x83c0dc81, 0x6a7f4481, 0x81d87fe0, 0x7f5cc5c0, 0xf828ea77, +0xbb817fa3, 0x3e816619, 0x817f6aab, 0x26417f7f, 0x0c40817f, 0x9cf01834, 0x32b37f7f, 0xe74fd0a7, +0x7f7f4167, 0x0e7f6ae0, 0x813d81b2, 0x8172027f, 0x4ea6c275, 0x814525de, 0x46817f7f, 0x7fcb2cd1, +0x7f8198ca, 0x3d7f60aa, 0x8d7fac81, 0xe2c081ed, 0x656f8181, 0xb2ae9365, 0x6ddd8d81, 0x86e0be65, +0x5495ec58, 0xcb981829, 0x429ee669, 0x81445fc5, 0xd3435724, 0xce51b323, 0x92b16972, 0xf0137f8e, +0x7f199a7f, 0x81be813c, 0x8155817f, 0xd481037f, 0x167f8181, 0x5ad131d4, 0x7fbc4149, 0x81046d60, +0x87174b9c, 0x94bc63ba, 0x68817c7f, 0x12ef7f8d, 0x7a81f4f6, 0x7f7f6db9, 0x4823417e, 0xa4989a48, +0x7e4c787f, 0xa47f8123, 0x819afb20, 0xbf7f447f, 0x8d95af5c, 0x33a6812e, 0x07c75cd5, 0x337f8151, +0xa2e5e15a, 0x847f7209, 0xf9f4b934, 0x72a77f2f, 0x1a9f2f7e, 0xac224bb2, 0x81467f0c, 0x3f8fd28d, +0x4a78b07a, 0x5269a71e, 0x4fd57fb2, 0x0b5262bf, 0xea39ca0a, 0x577f3d1e, 0x509481d7, 0xb7816b48, +0x35c31d7f, 0x818bdc1d, 0x24087fb9, 0x7fe27f07, 0x55866481, 0x7f9edd28, 0x07811ed1, 0x1b317fb2, +0x8cb19dee, 0x396920a6, 0xd965dd85, 0x9ed5ff47, 0xc97f7f7f, 0x7f8c1d55, 0x06a69701, 0x9e348881, +0x7f6cb281, 0x388181d1, 0xd07d61ac, 0x7fdef7f1, 0x08177f7f, 0xd87fa1c0, 0xfe9ebeb5, 0x813d3430, +0x2181811e, 0xf4dab306, 0xa36bbb7f, 0x05e87f7f, 0x957ff3cd, 0xe4814219, 0x7fbf3f81, 0x818a7fab, +0x43e081d9, 0x87c08729, 0x0db52785, 0x8181b0da, 0x4c94697e, 0x817fa9b2, 0x7f7f81d9, 0x8bda8193, +0xb6811781, 0x81796220, 0xdb082ac1, 0x2c7f5d3b, 0x8c8103f3, 0x4dcbd01e, 0x7fc1a87f, 0x457f7f23, +0x5e26bad6, 0x7f8174f3, 0x30109f08, 0xef8526cc, 0xe8690849, 0xd2b981b0, 0x7f7f7fb5, 0x8156ef7f, +0x921a7f98, 0x7fe6681e, 0x244e497c, 0x5ef68149, 0xbfbcaa7f, 0xf5011204, 0xfe7f8163, 0x8cd08f31, +0xc88a6b43, 0x7c8105b6, 0xf6d881aa, 0x074c2c2d, 0xd6815c7f, 0x1d15d265, 0xce816f0c, 0x14c77f2e, +0x9a7f08c4, 0x1a8b8ed2, 0xbe9637ed, 0xb59fac5a, 0xa4d07f91, 0x7f847fab, 0xd981597f, 0x7328575d, +0xb83af760, 0xb536b941, 0x81c1937f, 0x16b7814d, 0x3f7fb081, 0x7f41e102, 0x727f77ea, 0x9558c381, +0x48ac819e, 0xa87f99f2, 0x819e7f05, 0x92816eb0, 0x59447f81, 0x4f6cb056, 0x818163a8, 0xe00a64e2, +0x81bb7fa3, 0x7f816de7, 0x86ac6a50, 0x48a277b0, 0x8e37cd81, 0x7f88e646, 0x023351d8, 0x1ee7a64b, +0x8145f239, 0x81810859, 0x557f7f7f, 0x6971d292, 0x813d7fdf, 0xe86cade7, 0xdab67f7f, 0x6d4f247f, +0x9df2297f, 0x1e8172ec, 0x3f81a56d, 0x907f02d3, 0x45663970, 0x817f46a4, 0x817fd1ce, 0x9843b181, +0x7fdf817f, 0x784881da, 0xd8a281c2, 0xd77f88d8, 0x8c81817f, 0xeb625633, 0x73fe4eae, 0x57787f19, +0x7f81ea7f, 0xca8143c8, 0x767cbf1d, 0x7fbc81e1, 0x7f9bbeb5, 0x7fea8a12, 0x7f5c7fde, 0xb5fe1685, +0xa4da81c8, 0x7f6b542e, 0x81814b7f, 0x6252f7d3, 0x55eebb81, 0x6a0824cf, 0x586128f0, 0xded58181, +0xe32a819b, 0x79c67f20, 0xf2812781, 0x7f58c369, 0x7ffa2d7f, 0x817f81f4, 0x7ffc2c07, 0x457f9981, +0xff4281c7, 0xd78e6955, 0x3e10e57f, 0x632b817f, 0x7f0197d1, 0x813e7feb, 0xf981ac81, 0x8f81317f, +0xc37f137d, 0x01918101, 0x81b007a4, 0xaab059ad, 0x7f7f81f7, 0x6ac590a9, 0x817f81a0, 0x68816067, +0x0ae08182, 0xb47fb268, 0x33d94a7f, 0xbf43d243, 0xccff8e4e, 0x44ebe23f, 0x0d4cfd2e, 0x7f81947f, +0x81d1cee8, 0xda0e16a3, 0x34a74376, 0x117b81ae, 0x1d853f6b, 0x81d6d2ed, 0x8d7f5be6, 0xcf1fb4a0, +0x3e1ec13a, 0x7fed0cf4, 0x65d3817f, 0x0a1fec83, 0x7fc4811f, 0x666381c1, 0x8147c754, 0xa9cf7f53, +0xa076cf81, 0x4c7a4712, 0x1a814ff5, 0x9e7f8175, 0x8181698b, 0x7f1a7f63, 0x8191f19f, 0xb1037fa8, +0x7f52aa2e, 0x42b65c09, 0x94afab7f, 0xbda37dda, 0x81f2d66b, 0x3e737f1f, 0xcb816031, 0xd56181ef, +0x2c6ffb81, 0x44852329, 0x1b4b81cc, 0x055a7bb7, 0x4c6f2e1d, 0xb14ae1b7, 0x7f43c2c4, 0x2368b80d, +0x7f7ffb9f, 0x990b0523, 0x74b33ea5, 0xd64c8199, 0x98ed81a2, 0x311bc8e9, 0x599963cf, 0xb0eeed4c, +0x8181ed70, 0x42023428, 0xd00270bc, 0x812a7fd3, 0x7fc4101a, 0x3448d8db, 0x068814a8, 0xfeb5701c, +0x46db1481, 0x3cc5b001, 0x8181c2b3, 0x9c3d7f7f, 0x7f13e346, 0x2b75819c, 0xf57fcfe1, 0xfdaac581, +0x7f814c6f, 0x7f186b39, 0x7f818150, 0xea7f5b7f, 0x816776b0, 0xb8657f57, 0x7f447f5e, 0xc3628122, +0x84849d7f, 0x7f8cb914, 0x92813dd5, 0x81557a3a, 0x6b4cbf7f, 0xc6818b94, 0x7fbb7f98, 0x3c88812d, +0x8de4267f, 0x8dee81fd, 0x37f98131, 0x3d8164d2, 0xd97f8181, 0x3ccaebf9, 0x8165cb7f, 0x63cb81d3, +0x817fea94, 0x48817fa1, 0x4ea0b16b, 0x3cb5ab6f, 0x3148f975, 0x8c7f1ab5, 0x7fae5a87, 0xad06c881, +0x0681c187, 0x81a5a11b, 0xe29ea383, 0xf07f536d, 0x81b0e702, 0xc37fbb59, 0x9e6167e8, 0x3f55c9be, +0x819a40a9, 0x6450b823, 0x15927b65, 0x72a149d4, 0x6c280b5c, 0x5d7fd8f3, 0x998d831a, 0xbb7f817f, +0x10ab3f81, 0x96f65101, 0x257f743a, 0x7f04a024, 0xe0c981b4, 0x81163a22, 0x8e197481, 0x21addfdc, +0x7f2dcdaa, 0x787f4407, 0x6b7f81ce, 0x9fd252c5, 0x85e8979b, 0xb049819d, 0x8f7f7f4d, 0xbe284c7f, +0x78135181, 0x5d33db0a, 0x8123817f, 0x81496a7f, 0x81889281, 0x8145bf37, 0x2e9f7fdf, 0xe4543a2d, +0x811e6024, 0x7f62e6ba, 0xaad07f1f, 0x4f73894f, 0x4c7f8181, 0x57817f06, 0xd27feb81, 0xcf1a815c, +0xa57fbe7f, 0x684f7fdc, 0x81f6cbc0, 0xb72bf82f, 0xda6d448f, 0xc6157f36, 0x7f81407f, 0x9a8c4719, +0x5fdf81dd, 0xaf817f0f, 0x813c11da, 0xd7ec1181, 0x570186e7, 0x8181f41e, 0xe5b8a2bb, 0x819051f3, +0x2a7fd68f, 0x37a13ac3, 0x5b81fd38, 0x6342817f, 0x84697f6e, 0x7f5eddfa, 0x6abc81d1, 0x8151d9d8, +0x3e5c6c53, 0x2b397fc6, 0x60d58548, 0xc540b213, 0x67815781, 0x89983030, 0x767c7f81, 0x71effe3a, +0x817feb42, 0x05818897, 0x810ea318, 0x99bb0e8a, 0xf1dd4b7b, 0xae6dcbae, 0x8155adc0, 0x15b4c0c1, +0x81ac9b14, 0x895057be, 0x9599d781, 0x38968fcd, 0x8171e981, 0x7a62d80c, 0x8681509a, 0x7f5d489b, +0xa381467f, 0x7fa12952, 0xa07f8109, 0x5acd5d7f, 0xbb7fc77f, 0x531f3429, 0x7fb77f5a, 0x566d89f6, +0x817d4c23, 0x406a81e6, 0x1381c99c, 0xad89d258, 0xa67f0cad, 0x510b916e, 0x7ffa81a1, 0x0b6a11e1, +0x2be02284, 0x7f4e4338, 0x817fd023, 0x058174bf, 0x677f950a, 0x97a6a01d, 0x81817f81, 0x136881da, +0x717f7f9a, 0xbf811404, 0x6714c965, 0x51828181, 0xcaf329a1, 0x8881bc0a, 0x7f7d8305, 0x2c7f5a7f, +0x7f7f63e9, 0x797fced4, 0x818154f5, 0xf13681c4, 0xc04b1d81, 0x0e81d244, 0xbf3f66bf, 0x5a3f2d7f, +0xfa931714, 0xce485844, 0x263ae55b, 0x7fd37f81, 0x79477ee4, 0x7f8187e0, 0x81ab8144, 0x7f8d7f7f, +0x6f715068, 0x7f5577f6, 0x7f487f41, 0x81813b84, 0x33431981, 0x81a181c6, 0xca1b667f, 0xe75a7f71, +0x168128b0, 0xe0a092d5, 0x6e8164de, 0x2e817f61, 0xc4b1818d, 0xea7f8a32, 0x1098817f, 0x14a18156, +0xdd568862, 0xd3176a4b, 0x81308181, 0x697b3faa, 0xe256cecb, 0x697f1691, 0x4e228c05, 0x7fa6682a, +0x7449816e, 0x4558a15f, 0x9f513638, 0x7f7f819c, 0x7fb77aa1, 0x5eaf4912, 0x14817f7f, 0x0853817c, +0xa47fbe41, 0xdfd77f0e, 0x81371d7f, 0xc9cd7299, 0x81d0673a, 0xb133492f, 0xfd7f4003, 0x897f7f1c, +0x1990d405, 0x9f3ee2bd, 0xc168817f, 0xc6f57fa4, 0x83ee483e, 0x90f781f6, 0x81dcb39d, 0xd37fe881, +0x28819181, 0x83da81d4, 0x7251dc7a, 0x7928ffc3, 0x7bca8181, 0x7f7f817b, 0xf6fa907f, 0x37916bd4, +0x0d3fbc7b, 0xde81bdd0, 0xa0147fdc, 0xbf0f7f0b, 0x25958132, 0x7f6c9931, 0x7f44812c, 0xd4868181, +0x6d9622ad, 0x817f30b6, 0x3c7f817f, 0x81238823, 0x7f2d5c81, 0x6f81531b, 0x947faff2, 0xa5ea038c, +0x4268817f, 0x814f8cfc, 0x81481809, 0x814d7f1a, 0x3c87507f, 0xa47f1219, 0x4f65f27f, 0x5d7f2c21, +0x7f4631a5, 0x5a81475a, 0x7f81ef60, 0x73aaa890, 0x327f7f7f, 0x81b614c1, 0x7fc6477f, 0xe5b7a6be, +0x55816e73, 0x7ffaffb1, 0xb93d1b81, 0x3ce80fe2, 0x37ec6194, 0xe681e2db, 0x81c1d37f, 0x33a2bad3, +0x582b8181, 0x7fdc7f25, 0x81978169, 0x9733486f, 0x3de13ef3, 0x247fd5cf, 0xa91d817f, 0x606a7f56, +0x817f8178, 0xe61d7b08, 0x7fa08181, 0x058bd29b, 0x28237fbf, 0x4f81b057, 0x488d8103, 0xe1817fa5, +0x81666297, 0xfabc75c8, 0x7ff75edb, 0x4a62fab5, 0x687fad84, 0xa7221c28, 0x773a5a6e, 0xf27f8581, +0x7f7f9569, 0xa31d812f, 0x81d48457, 0x7f88bcfe, 0xd743b981, 0x7f608102, 0x18de2185, 0x81d77fc4, +0x937f337f, 0xe67fe52e, 0x95d481da, 0x887f6071, 0x81301872, 0xd9088132, 0x7f7f5a81, 0x8181aba8, +0x7f9a7fcd, 0x7f7f4136, 0x3d87d3a8, 0xd581d381, 0x5d306ed0, 0x615b98dc, 0x812b3281, 0xdbe5c443, +0x817f7fe9, 0x815462da, 0x8122da0a, 0xb5817d81, 0xe12bd56c, 0xfa819a68, 0x907f8183, 0x236aa83f, +0x1b576c9b, 0x5b6424ab, 0x3f3e3953, 0xec81507f, 0xddb43aa9, 0x818134f1, 0x81c37fb0, 0xbf4a7fcf, +0x810b7f7c, 0x705b81d5, 0x23636581, 0x3caf7f9d, 0xbe765d81, 0x189a11ae, 0x8194cabc, 0x6063c5b0, +0x0e3c5353, 0x388235bb, 0x81904632, 0x2b9781d3, 0xd78db758, 0x953fe437, 0x7423f22d, 0x81ad9c95, +0x7fd74748, 0x329b582d, 0xe0e07f7f, 0x7f81ae8e, 0x962fe50c, 0x3b7faf1d, 0xcaeb8e37, 0x7f54276a, +0x927f6c9a, 0x828143c1, 0x90456105, 0x7f757ff6, 0x796681eb, 0x7478b225, 0x70d852bb, 0x7f280bdb, +0x56597f47, 0x51e6fff8, 0x1e7f818d, 0xdc4a8175, 0x10712985, 0x2ee6daa1, 0x81c7d046, 0xa1a36e79, +0x9adf7f7f, 0x812ee70d, 0x91bc1062, 0x443b7fa9, 0x818f47b3, 0x7f817f3b, 0x7981a09c, 0x7f9c817f, +0x1b46d2cc, 0x786179de, 0xd97f61f2, 0x81b4f213, 0xd381874a, 0x7f4f7fc4, 0x9c81c3d5, 0x17d59842, +0x7fc9a8fc, 0x996840b5, 0xd5a79281, 0x88378181, 0x7fd4ca81, 0x81ba7fee, 0x5dae297b, 0xc8fa0e5c, +0x987a5757, 0x5debc016, 0xa64eb973, 0x61a51b81, 0x66509c7f, 0xeb7ad965, 0x7ada817f, 0x4d5bf7bc, +0x8876b8a4, 0x7d73b4d0, 0x762cb5ad, 0x6ec14d66, 0x6bfdad7f, 0x7f7f810e, 0x81bc2b15, 0x0f817f9c, +0xc72fe940, 0xc9168b0c, 0x7f6e8d79, 0x7f81fadd, 0x706181f6, 0x678181e7, 0x3f68c5e8, 0xa06d8195, +0x78d73d7f, 0x7295109a, 0x05897f55, 0xbdc07281, 0x81817fe3, 0xcd157f30, 0x489ca757, 0x53468119, +0x3f6a81b8, 0x03fe7c43, 0xb511cda3, 0xe581a17f, 0x5e3e3ec9, 0x7f507f96, 0xbe12a481, 0x81006ddd, +0x49322cb4, 0x27f7a81e, 0xc6085618, 0x8148d270, 0x68668181, 0xec5ed498, 0xa011833a, 0x81475300, +0x071d3a63, 0xa2c55142, 0x7fe9767f, 0x6e3c21fe, 0xeb63814a, 0x7fd57ffd, 0x9f91630b, 0x475f219f, +0xbbb5c7bc, 0x686de142, 0xb652909f, 0x6b8e5a7f, 0x7f885858, 0x7f027ee1, 0x7f818191, 0x818881d4, +0x1b7c017e, 0x8581aac9, 0x627fa790, 0xaf837f7f, 0x69064681, 0x311d32e9, 0x6f7f0ba1, 0x7b81a1c5, +0x7f4f8144, 0x81e38144, 0x528daa3d, 0x1c098175, 0xba6b0ac7, 0x732614ab, 0x5b2c5d7f, 0x81815ce8, +0xb37f7fbb, 0xc87f8ec5, 0x81b08181, 0x5bcb8136, 0x7f2524c5, 0xf5e881d8, 0x50ceaf7f, 0xf72387a1, +0xd4c74e74, 0x663f7fe8, 0x26ae4a7f, 0x5d7f6358, 0x8178bbdb, 0x46f38102, 0xb0327f53, 0xa019f297, +0xeff57f12, 0x7f7f79e2, 0x8120817f, 0x73e4b322, 0x999c6349, 0x92ba973e, 0x7f7fbc5c, 0x81946449, +0xaf4b817f, 0xca523d00, 0x81817fbf, 0x3b145a81, 0xec54886e, 0x81cb173e, 0x5f02667f, 0x967a8181, +0x8a817f81, 0x3cb93c41, 0xfa811336, 0x74feb5a9, 0x821b9081, 0xaea865af, 0xf17f4e71, 0x52810059, +0xaebc3b36, 0x6a8f7fd2, 0x109de3eb, 0x4c555f48, 0x8166cc78, 0x7f4c7b48, 0xc7d1d381, 0x1381537f, +0xa0ab5c1f, 0xa00d0039, 0x81157f81, 0x2f206f81, 0xacb45b81, 0x2ec382f3, 0x7ff0383e, 0x44dc5aeb, +0xc37f81cd, 0x43553306, 0xa63d7f81, 0x7f7f8140, 0x7f7f818a, 0x7d82bd27, 0x817f8b7f, 0xa3579535, +0xc2e47871, 0x81815a0b, 0x49977081, 0x724cc372, 0xc681c27f, 0x2b7cabf0, 0x3081a6ac, 0xec7f964e, +0x7fda3ad3, 0x422671a4, 0x60008158, 0x7f738e7f, 0x538e6f81, 0xb73e0ab0, 0x796a6551, 0x81ad178b, +0x19731c7f, 0x1bf8c26a, 0x4acb7fd4, 0x5121b547, 0xda547fff, 0x7e505ff7, 0x817fff48, 0xa0813934, +0x3d4f77d1, 0x83f327ef, 0xc7937f6b, 0x357c4a84, 0xc183ca7f, 0xbd6081db, 0x30e0d783, 0x492aa57f, +0xbba6671e, 0xb062baef, 0x7f817f20, 0x7f267f58, 0x7fb94b3f, 0x447f5b21, 0xb864997f, 0x7fb057f8, +0xb1401a7f, 0x7fb57f9d, 0xefbb8175, 0x66892581, 0x7f06fa7f, 0x817a90f0, 0x7bd5793b, 0xc37ead24, +0x6feaa4be, 0x3a548cde, 0x8e81f98c, 0x81b78199, 0xe6c11940, 0x4d647f2f, 0xbc5c8181, 0x8175fa91, +0x8188864b, 0x85591890, 0x47ce5d57, 0x38a4efb7, 0xce5e739f, 0x81737fd4, 0x6d7f635e, 0x9c861774, +0x89818262, 0x45bc8114, 0xd1552a89, 0xc68174f8, 0xaf9081d8, 0xcb08e35b, 0xf5c404b2, 0x2ce96576, +0x11895a36, 0xcf20e0ea, 0x2548e7dc, 0x8f774c59, 0xf85a8f56, 0x7f81ddae, 0x7f7f627f, 0x1481491c, +0x7f97661b, 0x787f81e0, 0xa2192f7f, 0x6cb37fb0, 0x8181ae27, 0x3f7fafec, 0xce7f5a86, 0xae067f7f, +0x3c1c8181, 0x817eb443, 0x8e741041, 0x5a4ef2ba, 0x61029de4, 0x9d637e34, 0x819d7fca, 0x8181cd7f, +0x8175b96c, 0xbc4f6ecf, 0xf826aead, 0x567fa36d, 0x81968197, 0x9f9dcf3e, 0x265c8131, 0x81d9deab, +0xc21cf772, 0x7f6bd907, 0x313f795b, 0x817f197f, 0x09ecfe43, 0x6114402f, 0xe0995a81, 0x5f263281, +0xbe7f8158, 0x7f7f26dd, 0xfd7f7f7f, 0x3da2c609, 0x6ce3810d, 0x7f81cd1b, 0x81d68809, 0x42be9c6c, +0xf3d37f5c, 0x7f81e031, 0x774e0c30, 0x7f8f468f, 0x9ebb42b2, 0x7f817fc0, 0x1376817f, 0x25f78181, +0x7f8eb3be, 0x20228505, 0x229fe181, 0x81ac277f, 0x4f308129, 0xb8818111, 0x81d47fc7, 0x818181f0, +0xc27fd34a, 0x9dbc810a, 0x8481d981, 0x61817581, 0x8a2c817f, 0xe7815625, 0x96a7946f, 0x3a7f5d5d, +0x835f2fc1, 0xb6082dd7, 0x567f687f, 0x838181b6, 0xc663e9ad, 0x517f7fbb, 0xa570adb0, 0x12cfc2b4, +0xc1587f39, 0xbd8140ba, 0x7f814366, 0xd00981ec, 0xc27f217f, 0x5cd1b2ad, 0x81992c7f, 0xb1b57f62, +0x3bb9b1de, 0x9213cd8a, 0x814f136e, 0xa5c27fc3, 0x947f7f59, 0x8ef30ddd, 0xd3f290e9, 0x39bc7f8c, +0x7f814e81, 0x5381ff22, 0x7f81f04f, 0x0adde113, 0x5b93867f, 0x687f8111, 0x7f7f1c60, 0xb856814f, +0x7f8e8196, 0x5cfb66fc, 0x20310ab4, 0xa2599964, 0x973f0065, 0x817f95d6, 0xa43b7f7f, 0x816bb1c3, +0x7c7581e9, 0x521770cf, 0x81456be1, 0x5e26818d, 0x3b54057f, 0xd4e231d6, 0x977f0346, 0xe777814e, +0x81b6095a, 0x7b337fd7, 0x18bb202b, 0x219b7cf9, 0x2bdeaa81, 0x55dcd1d2, 0x8123905c, 0xe463fd74, +0xd16ec17b, 0xd90f4e3b, 0x817fd97f, 0x817f7f7f, 0x816afd35, 0x3dd87f51, 0xa62f0e5f, 0x9f4766ce, +0xa933c505, 0x81049129, 0x627f3e7f, 0x025ea181, 0x7b5e4119, 0x0d11aff9, 0x817faaa2, 0xe3bc817f, +0x7fb27f81, 0xc07bc500, 0x9451c181, 0x1fc6811e, 0x7f9f9e3b, 0x7f4c1a91, 0x7f1f7148, 0x8358819a, +0x81119a81, 0x16d55f0c, 0xc07f7f2e, 0x467f9991, 0x237f8c7f, 0x81638101, 0x2aacb9ae, 0x4f439f32, +0xc65e96aa, 0x7f826508, 0x7fa86ab6, 0x602ddbc4, 0xd07fd00a, 0xb0e381dd, 0xca81cc2e, 0x7f7b7f15, +0x7f81b9bd, 0x21627f38, 0xa481ed5c, 0xeee8d79b, 0x64369481, 0xb97f81b1, 0xf443d7e8, 0x50a410b8, +0xd87ffbbe, 0x18589760, 0x814998d5, 0xc1814b2f, 0x7fc33b7f, 0xd25b7335, 0xb6b2987f, 0x988a7f48, +0x9aa4fb81, 0x597fd920, 0x7f4e7f9a, 0x7f819892, 0x81daeac9, 0x7f538fcf, 0xac7f8198, 0xd2529c90, +0x0ff00ba8, 0x765cb53e, 0x7f0fd3ab, 0x0d81cb33, 0x81989961, 0xc13c7f25, 0x20d6a6bf, 0x7f817fd0, +0x6f6594db, 0xec8c68eb, 0x638194a0, 0x3b89815c, 0x81734c65, 0xaeb69ec3, 0xd20b6440, 0x2f71b6dc, +0xc990816e, 0xb6077fda, 0x7f4cc48a, 0x816aa8e8, 0x817f397a, 0xfb8181b7, 0xe64d11e2, 0xe97f1576, +0x10706f7f, 0x9b935d4c, 0xaf7f3547, 0x7f175ec9, 0x7632a981, 0x2f61d4f4, 0x7a597f81, 0x499e7f00, +0x7f81315e, 0x81c4230c, 0x8cb66a2e, 0xe76a8105, 0x81817fc6, 0x7f0e50bd, 0x83ba337f, 0x81009281, +0xcac68465, 0x617fd326, 0x893981e4, 0x7f8ed9b3, 0x59467f7f, 0x0181e258, 0x7f81b281, 0xc9816629, +0x7fbc8181, 0x08e1cadd, 0xa8328181, 0x76b6e071, 0xb36d7feb, 0x505490d7, 0x09d5dd31, 0xba81632d, +0x7f7f6fd0, 0x9e45813c, 0x7c4c4f70, 0xd97f5090, 0x247f335c, 0x357fbb4d, 0x267fd5c8, 0x26d08181, +0x51447fa8, 0x81358efb, 0xb9197f52, 0x40aadca1, 0x7f6781aa, 0xaf272a09, 0x8177a4b2, 0x497fb515, +0x819b7f7f, 0x8b817f41, 0x257f7f92, 0xa8d2244e, 0x477af9ef, 0xa8817f36, 0x5ca681f5, 0x7f1a76cb, +0x8181c5f1, 0xc4ddc7aa, 0x81a1c2bc, 0x7f466fa2, 0x625c237f, 0x3ddd1f44, 0x4f54fdde, 0xa281f2d5, +0x95d14e95, 0xf26e7fd4, 0x9a813a90, 0x8154361c, 0x327f7f42, 0x7fa2983b, 0xd9c28197, 0xf00d1898, +0xb95848d4, 0x387f81f0, 0x9f8e7f4a, 0x1c5cb9ec, 0x1f63b5fc, 0x187f7dcd, 0x7f1c7f05, 0xa855e2cc, +0x5a747f98, 0x7fa1b4ee, 0xe8e51aaa, 0x7f81ec81, 0x38818175, 0x81815f07, 0x84888181, 0x81810852, +0x316e0060, 0x819b1c6e, 0x3d814577, 0x5ecc5d81, 0x5e81814c, 0x7f76914d, 0xfc7f814b, 0xd2819d96, +0x70a34ad0, 0x7fd544f3, 0x71e51581, 0x817d45a4, 0x7f479a68, 0x42447fae, 0x9a974f7f, 0x817fbbd2, +0x989f367f, 0x78fa30c0, 0xbb902e75, 0x7fda7f02, 0x7f82c5b8, 0x33c325c4, 0xdf89815c, 0x81c37f33, +0x52f37ffc, 0x7fcc4702, 0x81488124, 0x7f548196, 0x4214a4a3, 0xa034b302, 0x6a817e04, 0xc67f795e, +0x17a98157, 0x7f7f7f5b, 0x7f7f4c56, 0x7fe0b181, 0x7f907fba, 0x42f82fd9, 0x223ea236, 0x3d2025d1, +0x097f81b1, 0x81a35969, 0x373cb170, 0x44a57c3b, 0x26bc5a7f, 0x34657ff6, 0xc83b3d3f, 0x99be6c2c, +0x1c811f7f, 0x5c7f81ac, 0x33fa49f6, 0xa5c38181, 0x81ed8144, 0xc88a652a, 0x7f817fe3, 0x953cfc91, +0x466681d3, 0x366f16d6, 0x187f2ff5, 0x077ff351, 0xbe1b817f, 0x3884ec81, 0xb17f8181, 0x9a39437f, +0x476fac61, 0x7f911514, 0x24b0038f, 0x5647567d, 0x8143c7ae, 0x81b03a31, 0xec7f085e, 0xc10d7f55, +0x0a50b9f6, 0x7f1fd244, 0x7f81a1c7, 0xc48148ca, 0x877fc97f, 0x81115e37, 0x827ffdf3, 0x48b741c2, +0x7f9fc5c0, 0xf87f81ce, 0x43a1029c, 0x2a78427f, 0xff9a164d, 0x7ff981f6, 0x1b7f81aa, 0x6f48ca6a, +0x547f7f2a, 0xa9ce5cf0, 0xc07f7f7f, 0x7f6fa54d, 0x58817fbb, 0x7f96fa32, 0xfdc18381, 0x7abd7e7f, +0xc604b981, 0x64217ffe, 0x8893bd7d, 0x7f140281, 0x7759b9e3, 0xcc648129, 0x81c981c6, 0xaf2ba351, +0x9ecd819f, 0xac816ff2, 0xb516672e, 0x4157ab81, 0x81d21796, 0x35643433, 0xf13d5a7f, 0x9881d003, +0x5081bbd0, 0x7d7fbdcc, 0x6ba50d3a, 0x8170677f, 0x85b29f81, 0x8aa72210, 0xfbaaa381, 0x7f0e8f34, +0xcf367b7f, 0xe60399ab, 0x7f7f7f01, 0x817f7f11, 0x81818932, 0xe39a6e31, 0x8d3b8181, 0x107d2bc1, +0x908164f7, 0xe41a7fc5, 0xe944503a, 0xeb4b6559, 0x22817f81, 0x816ea8ff, 0x1557e37f, 0xaa7f701c, +0x817f828e, 0x738106c4, 0x7e0e7f05, 0xac768181, 0x17898158, 0xc081ea33, 0x827f9958, 0x3b7fd4fc, +0x16181f7f, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x0c7f3a86, 0x7750813d, 0xd2b385bc, 0x6f7f7f72, 0x7d487fa8, 0x7f7f8d65, 0xb6578196, 0x4d93815a, +0xf9a54748, 0x317d6469, 0xe0466f8a, 0xef869693, 0x3da5a9e2, 0x7f7f05a1, 0x14c68157, 0xc2148135, +0xad9ed181, 0x7a7f6081, 0xf10d659b, 0x4a38dce0, 0x114d7f2a, 0x2bc7f79f, 0x0f7881f1, 0x423e5553, +0x9c2b817f, 0xde7ffb85, 0xe18117ca, 0x67fd7f7f, 0x3fbbf57f, 0x9b34ad7a, 0x487a543f, 0xf06322f8, +0x9ce93023, 0x3581818a, 0x6a7f4d69, 0xad3d627f, 0xbd812c92, 0x8be4cc7f, 0x23b47fc4, 0xc87f819b, +0x104d6e7f, 0xcfcb8134, 0x0d8152cf, 0x467f7f19, 0x113f1ec5, 0x589bff9e, 0x457fa8c8, 0x2a1246d0, +0x731e257f, 0x40e460e8, 0x13d3057f, 0x817c8de1, 0xc09dc798, 0xab81d4f1, 0xa1a49d8c, 0xbf50df7f, +0xbc8192f7, 0x73aa4d5a, 0x1c817f43, 0x428fd6fc, 0xc282884e, 0xb9f67fd8, 0xd7777f4d, 0x1aa2d89a, +0xa5537f81, 0x8878b631, 0x4bdfd881, 0x7f21b57f, 0xcd7f7f7c, 0x4d2aa1d0, 0x3e138144, 0x3c7fe433, +0x146e5fe3, 0xa53b7f85, 0xa9537ffd, 0x819eda1c, 0xf93e2881, 0xc3814f7f, 0x018d69b4, 0xf87fa19f, +0xb09f9144, 0x6ec23f94, 0xdf7f39c6, 0x48087f7f, 0xfc818184, 0x18815bc3, 0xc764598b, 0x11ed68c7, +0x81df81bd, 0x7feceb38, 0xc90a473a, 0x81817f81, 0xef3a537f, 0x7f4d3873, 0xf45c846f, 0xfeb68afc, +0xd326d181, 0x7fe4ae7f, 0x3e817fb5, 0x81c95de9, 0xbebdcd7f, 0x23053726, 0xdd8d8181, 0x1d545233, +0xe37c933d, 0x81e52c1d, 0xc0ab2d81, 0x81e2667f, 0xcf2e70db, 0x9370319c, 0x21add116, 0xe6c34081, +0x076912e7, 0xa781aca0, 0xd6f5a5d0, 0xa9af5d7f, 0x2e817c77, 0x81e1661c, 0xd5308181, 0xae7f6ec4, +0x45815295, 0x2dbc2581, 0x50bde53d, 0xcd8cb1bc, 0x2e67977f, 0x32fc789f, 0x2b81ca3f, 0xef7f7f61, +0x19a7d611, 0x81427f81, 0xe98181fc, 0xd4397d7f, 0xd9ab7fb7, 0x819c8c92, 0xd5478339, 0xf0555aae, +0xfe09f481, 0x8128527f, 0x337f1081, 0x6dae3aa2, 0xccb67f81, 0xb1d8d87f, 0xc2d3dd02, 0x97af814d, +0x812f1c5a, 0xc50c8181, 0xcd70610a, 0x7f555df4, 0x567f7f23, 0x0a0d917f, 0x4b81815c, 0x184a5b8e, +0x5054c67f, 0x7f7f8ced, 0xf0b31f7f, 0x11698131, 0xe0ac5442, 0x57ffc57f, 0xb681cc02, 0xb78c8193, +0xa629b48c, 0xf2a8d43b, 0x59167d37, 0x81818bf2, 0xc87c817f, 0xbf7f8e7f, 0xcdbc2243, 0x2ed78e74, +0xfe817d61, 0xa3613c01, 0x25e281af, 0xbd6a7f36, 0xe57f4b7f, 0x81898161, 0x2a415081, 0x50b07fb6, +0xe77f9c72, 0xd43681aa, 0xd07fb648, 0x3c7fbd5b, 0xa555c475, 0x8169dc64, 0x09106247, 0xf9b660ab, +0x389542ac, 0x7fa22c4e, 0x81304e71, 0x29208c90, 0x317fa43a, 0x8281819d, 0x7f212065, 0xc0682681, +0x3f2b5dc7, 0x87d86d81, 0x5c7f4192, 0x7f54332b, 0xc564a449, 0x7f817f58, 0x3c6576aa, 0x147f816d, +0x4957d091, 0xbdaba3a7, 0xa17fa381, 0x49628138, 0x372f815b, 0x81676748, 0x8dba817f, 0xc89e8129, +0x866fc855, 0x43ba4a81, 0x8f4b7ec2, 0xa47f054d, 0xc3b07185, 0x7f8d3ad0, 0x2c7f9f8e, 0xc17f27d3, +0x33956381, 0x8e12a081, 0x7be52e32, 0x44b28122, 0xe4947f6d, 0x6eae4f7f, 0x34df816a, 0xd7938168, +0xa7a9528b, 0x7f81ec45, 0x41857f6f, 0xbc28817f, 0xcbd67c06, 0xaeb678d2, 0x529a547f, 0x44813b6e, +0xd44f247f, 0xbe1b9879, 0x877f4996, 0x5b9a8dcd, 0x2e558281, 0x1d817f84, 0xda7f8186, 0xdda9f4a6, +0xe698ae70, 0x7fb54c8a, 0x0aae8550, 0x857ff8fb, 0x1ac42e8e, 0x3f27814b, 0xe27fdc2d, 0x2db1eb81, +0xca817e5a, 0xec7f62e2, 0xc7dea079, 0x7f7a7f26, 0x7fdfbf81, 0x7fe89da8, 0xa9187264, 0xf98192c9, +0x6af88181, 0x667e3981, 0xf97f3c48, 0x7fdf817f, 0x3b9bbc81, 0x55bfbc81, 0x096fa6e1, 0x1b75a67f, +0x357f83ed, 0x7f7f6f7f, 0xf3b43812, 0x9d4c0d92, 0xa8817f4d, 0xb96c877f, 0x34818178, 0x5d7f997f, +0xdb7f5c7f, 0x39b056a4, 0x33bcffaf, 0xb398b57f, 0x285a4bf1, 0xb181574b, 0xc1e56d31, 0x4b717fd2, +0x56337f81, 0xed938157, 0x7f4b7f47, 0x49a3812b, 0x2e667fcd, 0xcd817240, 0x8b5470e6, 0xf693814e, +0xd3e07f7a, 0x7f7a4781, 0xf44cd60c, 0xfd546281, 0xef995981, 0x81e70628, 0xd3b97f47, 0x07819aa8, +0x707f447f, 0x9181c079, 0x72ae8173, 0x988d2a78, 0x7f814478, 0xb57fa076, 0xde7fb260, 0x17817f3b, +0xf045c31b, 0x7f7feb13, 0xbee5d07f, 0x5f8192ae, 0x0c9381f7, 0x28817f7b, 0x985964d5, 0x81817d81, +0xb381b799, 0x38b0817f, 0x3a887f0f, 0xf353fe81, 0x53817fdb, 0x5c814d56, 0x39b415ff, 0xc9c78c59, +0xb2817fad, 0x5884bec5, 0xd629b07f, 0xb67f81cb, 0xcb81abe9, 0xab7f9b7f, 0x35815781, 0x85878181, +0xb0b2177f, 0x86223495, 0x32cd7f69, 0x3c664743, 0x1d7981c2, 0x817f817f, 0x16cadc06, 0x03786a7f, +0x03818107, 0x5a8dd1d1, 0x58e57f20, 0x7f818152, 0x444c50f0, 0x3d81810c, 0x66ae7f62, 0x39d01614, +0xbd817f81, 0x789b7fa6, 0xadcd819c, 0xd37f8171, 0x291ebf81, 0xd5923ac9, 0xc37f63bf, 0x1c7981ae, +0xda5e969d, 0x248c5899, 0x8fbdc4b3, 0x6c50d2cb, 0x2a5fab0b, 0xc42d818d, 0xa4776781, 0xaa6281fd, +0x20534bc6, 0xb729727f, 0x6481de5c, 0xffc8a5e9, 0x707f25f2, 0xc674ba57, 0xe2ff9cb8, 0x02433e79, +0x108e223b, 0x617f9cd7, 0xb5507f81, 0xa6f63ee6, 0x09b45856, 0x7e7fe581, 0xe4f8417f, 0xe11d98be, +0x44817f7f, 0xc9c581df, 0xf7817fb0, 0xac6da566, 0xc681ed7f, 0x3249810f, 0xab55b281, 0x35e64c81, +0x61e9b982, 0x8d8c4781, 0x4eafd663, 0x7fb8302e, 0x40a9bcb1, 0x17b17f9e, 0x817f92c0, 0xe35d2fb3, +0x4ca815dd, 0x817086b9, 0x389b817f, 0x3c0d955f, 0x447f7f1a, 0x7f11307f, 0xada0815a, 0xafca7f7f, +0xce48b124, 0x7fd94e81, 0x2edc81a4, 0xaa098a97, 0x14030263, 0x97813a7f, 0x8181d97f, 0x2581c9e6, +0xe57f7f05, 0x16880f81, 0xa6e88184, 0x818490a2, 0xd5a87f94, 0x8b81a081, 0x38813781, 0x14a59a5a, +0x486b8c40, 0x893449c7, 0x1f03d659, 0x81bb2831, 0xe0d8c66b, 0xa1bb7066, 0x1b7f8c71, 0x07815581, +0x3820cb84, 0x8d3926b5, 0x81f27fd2, 0x46ff3a84, 0xfddc057f, 0x16d21ad1, 0xdbc27852, 0xf9817f34, +0x7f81397f, 0x6c0f8101, 0xf22a100c, 0x7c998171, 0x4b68bad6, 0x7f588114, 0xc3465c81, 0x439e037f, +0xd9817f52, 0xf1bd5594, 0xf93f9f81, 0x39447f7f, 0x46dca006, 0x93f30f7f, 0x6e215e81, 0x092ffdc3, +0xad49e8a0, 0x7f4e7fba, 0x9b9c4fc6, 0x8131f77f, 0x0700b27f, 0xf0de61ab, 0x258181fb, 0xd8e8bb56, +0x36932155, 0x9c7f2981, 0xb2bfe055, 0x7f0095b9, 0xaf73327f, 0x70817fbb, 0xb3a281d6, 0xfac08a53, +0x24e4727f, 0xcbb5dbbb, 0x38be99fa, 0xbcb94e7f, 0xdf2653d0, 0x93df5efe, 0xfcdc697f, 0xe754909d, +0xc6810060, 0x7f0ce4d1, 0xaec4a5ca, 0x4a7f81bd, 0xc17f817a, 0x7feecc3f, 0x2981285d, 0x3e0e6e81, +0xab81814f, 0xbf27187f, 0x12409322, 0x1c9dc451, 0xc48e03e7, 0x64013b81, 0xb5847aa7, 0x350a1b84, +0x27458181, 0xa1237f61, 0x9714287f, 0x817f8170, 0x1a7f7f09, 0xf07fecc7, 0xd686d4c9, 0x9abd7fa9, +0x449dc0bc, 0xc1177f9d, 0x7f547f9a, 0x7fbd7fe3, 0xcab19d2b, 0xc981b27f, 0xdc8ba548, 0xe95304a6, +0x22225820, 0xd29262ec, 0xf27f9f78, 0x1b94427f, 0x4738a436, 0x7f814fe0, 0x52817f7f, 0x1d818181, +0xf8d92d89, 0x24817fa3, 0x66d6aadb, 0x4e55b9ca, 0x7f81de8c, 0x7f814e64, 0x24141e31, 0xa5232778, +0x34e27f81, 0xdc87c78f, 0xc016dfb1, 0xce91829c, 0xe662c230, 0xda74e681, 0xcfb57bb8, 0xf18179ff, +0xcc818481, 0x4736ed45, 0xcc6ee881, 0x3a815a81, 0x81b77fb2, 0x56816f2f, 0xe82bcb89, 0x3875ad5f, +0xe05197e1, 0x1a7bc17f, 0x31a3f841, 0x7f90a77f, 0x1df3b910, 0x583f815b, 0x257f814c, 0xee7f7f8e, +0x7dbdc37f, 0xb6810a1c, 0xce816d85, 0x459bf72d, 0xd4fe0481, 0x0d49024c, 0xbfa3a055, 0xed9baf30, +0xf27f7f7f, 0x7dd24f9b, 0xc481817e, 0x7f899052, 0x074d5bdb, 0x2335c3fe, 0x00759f7f, 0x251bacbb, +0x147fc935, 0x0b180994, 0x9f7fa637, 0x387ff150, 0xcbac8967, 0xf37f60cb, 0xe4c216be, 0x7829816a, +0xd7b89981, 0xf8ae2592, 0xbeca818c, 0xf3817a81, 0xc681017f, 0x2f2f2e8f, 0xfe577f1b, 0xd503d778, +0x2d7f7f67, 0xb281c0f0, 0xf7d7812c, 0x817f9598, 0x1af6fb36, 0x157f3781, 0x5e31a27f, 0x038181ae, +0xf9fb81ad, 0x7a8eec81, 0xc8652e7f, 0xc67ba487, 0xbfce7f7f, 0xdf02a27f, 0xf9c1813d, 0xcaac0f90, +0xdc448181, 0xb17f921f, 0x99be8181, 0x7fc09993, 0x11ad1bc3, 0x5c7532bf, 0x25a8ce28, 0x0ab6a823, +0xc3c29055, 0xad55b6ea, 0xed81e0dd, 0x8caa6553, 0xff817fc8, 0x7f70d513, 0x433671dc, 0x5db37f6d, +0x7f318170, 0x55bd4dd5, 0x00c181b7, 0x41c3bb90, 0x441a818e, 0x84e78140, 0xad8a2c81, 0x2bc38158, +0x82654c81, 0x29d6600e, 0x227f7fc8, 0xa43d577f, 0x2b8364a3, 0x144e2a57, 0x07817f1c, 0x4a8171cd, +0x47d5a981, 0x3c7d0781, 0xba636c7f, 0x2c7a7f7f, 0x90cf187a, 0x6b4c3671, 0x3a2c436d, 0xef81de81, +0x4a828117, 0x814dabc0, 0xf930e17f, 0x4c3265c0, 0xe5ef7a7f, 0x819b8504, 0xd5427f81, 0xd881e992, +0x946a45fe, 0x310162a7, 0x14819c81, 0x7f817f4a, 0x552fcf42, 0x7f707f18, 0xb9dc4881, 0xa17f81c2, +0x9e476d98, 0x7f7f7ff0, 0x37662d4e, 0xdb7f99cc, 0xf9fe8cc1, 0x1f4c7fd1, 0x3588817f, 0x547abea9, +0x356e7f7f, 0xab1e857f, 0xda0e7f5c, 0x9d777f7f, 0x451f1bb6, 0xa82e087f, 0xb71d347f, 0x0916d881, +0x177e817e, 0x7f5a66a4, 0x237f2e6e, 0x816dafde, 0x2f8124c1, 0xcee67158, 0x067f667f, 0xe1258187, +0x3a3165f7, 0x1f7fac81, 0xf636364d, 0xf2818e36, 0x0b9e207f, 0xed2d7e81, 0xaa6e2281, 0xcb50b6a3, +0x237f72c6, 0x95819fa0, 0xd7d0c888, 0x39988141, 0xc8b07f8d, 0x5a43c942, 0xf3cd687f, 0xa2e77aec, +0x9909bb1b, 0x7ff8e681, 0x5d39837c, 0x5625817f, 0xa9738110, 0x8b604bb2, 0xf3ffa881, 0x5f359eb5, +0x07bf7f49, 0x7f63c67f, 0xc9db7fcb, 0xa65881c5, 0xf2818130, 0xa5817f81, 0x26b4174f, 0xc06f817f, +0x3b8181c2, 0xbfeb4d6f, 0x9e857f91, 0x58a9a1b7, 0x67147fdc, 0xcd257081, 0xb78d4081, 0x3749ebb8, +0x22ac7f89, 0x4af84134, 0xe3f97943, 0x2d8187c6, 0x027c8181, 0x81c1bfef, 0xc32b2267, 0x42257f81, +0xb49b3eb3, 0xa3d47fc2, 0x27ff4d81, 0x69b4a46e, 0x81818181, 0x7fd5817f, 0x00a97d91, 0x10897f7f, +0xd87f0d3b, 0xcbec8b7f, 0x7f814de6, 0x8286209d, 0x06e2bf7c, 0x81e28881, 0xae64afdd, 0xabc5667f, +0xee22bee0, 0xe157d171, 0xb39a5bd6, 0xabb6d1c0, 0x53c19d81, 0x447fdb81, 0xfc086996, 0xd65f0481, +0xc87f4d35, 0x313a12db, 0x5d81a1a0, 0x01231072, 0x327f81d6, 0x7eb1ba7b, 0xe9485dad, 0x5781767f, +0x047f2481, 0x817f5b50, 0xda064bd7, 0x7ff07b3c, 0xfde4a68a, 0x7f83816b, 0xe67f7f34, 0x1a56f181, +0xe2bbb681, 0xe4687f74, 0xd5a20bd8, 0x81d581a6, 0xb2395342, 0x977f0fa3, 0x81e3b5dd, 0xf1cd8147, +0xe34f48a1, 0x258142c3, 0xe24d510b, 0x46d6816a, 0x2a608ae6, 0x5781ef81, 0xcce97f0d, 0x677f7f81, +0xaa2381bc, 0xbde77f4d, 0xefa38185, 0x81a67f75, 0x6cceb856, 0x4976bb25, 0x3efc977f, 0x86cdf691, +0x13497f81, 0x7fe7730a, 0xd71d8163, 0x8c7f7fbd, 0x1bec817a, 0x8154b897, 0xc71e45b6, 0xbed568c3, +0x256c6514, 0xd481af7f, 0x19ad697f, 0x817f745f, 0xd27fa187, 0xd5e769d9, 0xd816cf7f, 0x1aa4815b, +0x035e7ff6, 0x85de810b, 0x3ea73657, 0x216281b2, 0xe48144b7, 0x3277dfe8, 0xf8c57f20, 0x907f5091, +0xf481b081, 0x9d8152cf, 0x0e6a81c0, 0x7fff7f06, 0xf8884d81, 0x687f81b3, 0x2657ba35, 0x527fc24c, +0x9945694c, 0x1faa4e5b, 0x37819334, 0x57f52a7f, 0xec7f817f, 0x817fa0a8, 0xe3a8818a, 0x16beacdb, +0x2ad2b97f, 0xb981245b, 0x359a8181, 0x7f97265e, 0x9de4387f, 0xed7f058a, 0x0e817f81, 0x4e69bdb5, +0xc5e38552, 0x77482e81, 0x6c817542, 0x447f2990, 0x07b1f9dc, 0xc4247f82, 0x077f7f81, 0xe58bdca6, +0x55e07f74, 0x0e7fe47f, 0x04883e81, 0x4981f87f, 0x7d7fce29, 0xe07f7d7f, 0xee817fa6, 0x7262da81, +0xe158813b, 0xe35e41b9, 0xf17f6d65, 0xef7560ea, 0xae408493, 0x97812300, 0xf29f816e, 0x557fd5aa, +0x328bb778, 0xa27f7f25, 0xde7fb5cb, 0x03577f0b, 0x577f18a5, 0xb9816d20, 0xe28d2a7f, 0x248c7fb9, +0xff5fb965, 0x93b27f9c, 0x2d817581, 0x41958195, 0x3a7ff044, 0x7f7f6faa, 0xdb81b784, 0x5081603d, +0x363ff38d, 0xc747d8b3, 0x537f812b, 0x8176a97f, 0x1d8181d3, 0x81811929, 0xe893d4c7, 0x3c7fcf7f, +0xf6385c81, 0xaa63b42e, 0x81c3a781, 0x488142a7, 0xcbef89be, 0x3f748159, 0x50c52932, 0x298198d7, +0x14c68e7f, 0x7fd8a798, 0x085a632f, 0x7f1d7f7f, 0xb2e34289, 0x88c3b262, 0xb044817b, 0xbf7f238a, +0x5ae6b554, 0x9e833081, 0x607f985d, 0x0203226c, 0x333f63fb, 0xc77f8181, 0xe8297f7f, 0x9dea7f33, +0x6f2e1f54, 0x2e41327f, 0xff0f7f81, 0x7ffa7f30, 0x343f74a6, 0x81f59f81, 0xaac67dbb, 0x6f81da1f, +0x79ec8153, 0x78eae674, 0xec7f6e71, 0xa46a48bf, 0xe5810efa, 0x0f41b9a7, 0x3a1124a5, 0x5ff97f7f, +0x0bd5be75, 0x7f8181f0, 0x242f409e, 0xf47ac606, 0x1e96d94f, 0x43a66a81, 0x18fc815c, 0xa51a7f7f, +0x1185d27f, 0xa00c810f, 0x078ae481, 0x71d7866c, 0xd47ff796, 0x813cf240, 0x45814d48, 0xbc6c3618, +0xa8b937ea, 0x838e7f7d, 0x19b3817f, 0x7f73be95, 0x00fa46b3, 0x4c43655a, 0x20f67fc2, 0x98fdda7f, +0xe879817f, 0x4c9e7f2d, 0xcd7fefc2, 0x9681a909, 0x1e59e7b3, 0x35818136, 0x9e3a977f, 0xaf7f207e, +0x9633227f, 0x44450e81, 0xc2c25f7f, 0x7f1a3f81, 0x08acf783, 0xd083a838, 0xd2627f59, 0x6d327f32, +0x362c2b5e, 0xee8e507f, 0x48b431c5, 0x00816381, 0x1e6c1150, 0x7df48149, 0x36f94f81, 0xc5667844, +0xf7cbf52c, 0x520081b8, 0xc6967f7f, 0xc88981b8, 0x328181ee, 0x697fa9f2, 0xc99a7f81, 0xb5a08181, +0x1d74dc89, 0x643ab5cb, 0xeb81f9be, 0x987f277f, 0xad38747f, 0x61778c52, 0x59817de7, 0x05ae8144, +0x217aa1de, 0xc0817fe9, 0xa1a6b4dc, 0xece08189, 0x4118207f, 0x9a8181a8, 0xcf0f81c9, 0xba81d081, +0x138181b8, 0xd537957f, 0xfc7f9b8e, 0x7fc9607f, 0x16c085fe, 0x81fa7fb6, 0xf7847f66, 0xc20ee0fd, +0xf983ec97, 0xce209f7f, 0xe2815922, 0x39b07aa8, 0xd3a82353, 0x6ee38287, 0xc60d7f29, 0xcf478f57, +0x594d731b, 0x0e8c42fc, 0x6a702ce3, 0x7b81b301, 0x3181a450, 0x7fe94481, 0xe581b66f, 0xd87f7f2a, +0xc1b385ee, 0x81816964, 0xbb7f5093, 0x8151817f, 0x51e3547f, 0xb3fa7549, 0xca81b044, 0x258143bf, +0xb18181c8, 0x7f54eb08, 0xd4c9db57, 0x68a981b2, 0x4ef17f69, 0x917f977f, 0xda38865e, 0xe97f3e5f, +0x0bbc63dc, 0xec34daaf, 0x1dc38186, 0xb381810e, 0xcbacafd2, 0xdb3ec56a, 0x22817381, 0xbc44cc4d, +0xd681d0b0, 0xe614c8a9, 0x53577f7f, 0xdabc3012, 0x353ecdc2, 0x81728179, 0xb660cf7f, 0xef49ca49, +0xc645dd7f, 0xd16a7f81, 0x1fb99d7f, 0x7d36d57b, 0x04ce7f46, 0x602adede, 0x1142cbb1, 0x60de427f, +0x29814ccb, 0xe9ad7d99, 0x127b2881, 0x5c5aa08f, 0x38c0db9f, 0x7f76d981, 0x63b32381, 0xdd41ce81, +0xce14e57f, 0x74ed7398, 0xdd81e6eb, 0xb6f2d581, 0xa581b77f, 0x468c81d7, 0x5f2d88b8, 0xf481fd21, +0x6ec1e381, 0x71568485, 0xcbfa7f6c, 0xa02e0c9b, 0xd9a09a7f, 0x813d8176, 0xcf43c52b, 0x31a8bca1, +0x2432dce5, 0x78277f94, 0x474e592e, 0xdefb7f81, 0xff147f51, 0xbb7f3858, 0x25f1e591, 0x36fad833, +0x0ac87f39, 0x815d9781, 0xaebaaa81, 0x4981815f, 0xd7817781, 0xb44838b5, 0x32626981, 0x417fe2f6, +0xdf638c92, 0x8624067f, 0x4432e69f, 0xd8688a7f, 0x06815215, 0x25bbbb7f, 0x5b5b6f63, 0x417b5b81, +0xcc4a7b7f, 0x6ea47fd3, 0xe866b481, 0xc630c9aa, 0x083781ed, 0x69de817f, 0x0381f9f2, 0x84812ce2, +0x027f4dd9, 0x7f5a9e81, 0x00ea319c, 0x222b9a9b, 0x34a143f0, 0x7fefd1d0, 0xb4947981, 0x11474bc1, +0xcd44df7f, 0xa2878439, 0xd6f040b9, 0x6f86817f, 0xc67f7f81, 0x7f693443, 0x8d21b1ce, 0xdab1aed5, +0x3f17ae71, 0x45b7ba7f, 0x05229255, 0xb181d8f5, 0x688681e3, 0xea81e181, 0xf57f817f, 0x7fdc5881, +0x6e817f7f, 0x817f537a, 0x3987247f, 0x300a7f05, 0x3128057f, 0xe731b77f, 0x4881b50b, 0xb86dae65, +0x00c015a0, 0x81397552, 0xb9bcab24, 0xf5818181, 0x57d18190, 0x81c3648d, 0xed813a67, 0x4c5d3a06, +0x3581a4e3, 0x64db6467, 0x21af7d27, 0x5adf7f57, 0x66d9c99f, 0x5b183375, 0xbf4c41e5, 0x577f7ff0, +0x4e818181, 0xf295493c, 0xd2c28170, 0x5c3a7f8e, 0x09c18181, 0x7f7f7f9c, 0xae675fa9, 0xf6d781a4, +0x66ba10dc, 0x7f882981, 0xfe7f7f7f, 0x7f7ffea4, 0x2798d845, 0x899cb6fb, 0x4982bcd9, 0xc0815ab0, +0x558181aa, 0x52110702, 0xd179e3c5, 0x482923bd, 0x187f8181, 0x81b3f6d9, 0xfd96a581, 0x2c5d1a81, +0x6fde23dd, 0x9e815063, 0xfba49bcc, 0xfa775f4d, 0xb1b8b681, 0x81894f1b, 0x3c8163fa, 0x2d7f3d81, +0xc57f6681, 0xbd7f7e81, 0x137f7f81, 0x6b3d6f81, 0xe5e2610d, 0xb4683186, 0x7f6e817f, 0x90c12abc, +0x13a3502a, 0xdddeaa8c, 0x07a67f79, 0x1a5481ce, 0xb87fb01b, 0x0a4b1e7f, 0xd77ffcb7, 0x03535781, +0x5a5f697f, 0x02ca7966, 0x379638bb, 0xc507c941, 0x5e7fdf8e, 0x7ce0c99b, 0xa162ccfd, 0xff8196cc, +0xd43aa081, 0x67798185, 0xd6277f2e, 0x7f6dcb7f, 0x217fcd76, 0x2c88fb91, 0x06815b8f, 0xe37f7dae, +0xe937257f, 0x648181ae, 0x21652c4f, 0xbf81685b, 0x36abe921, 0x3cb5de30, 0xa88e8162, 0xd87f812b, +0xb981d14d, 0x75c181a7, 0xf7c08aa1, 0x57904e7f, 0xf6f688d8, 0x7fcf6281, 0xd33dbf7f, 0xc07f9a73, +0x197f8135, 0xcabf8bc6, 0x3f81c37f, 0x487fac4f, 0x8ddc924b, 0x50257f7a, 0xf75e1496, 0x28d2d48f, +0xcd91817f, 0x2a7f81b2, 0xbd7fb67f, 0x3b877fe1, 0x407fd8fd, 0xbc815f20, 0x7f50e581, 0x8bdfae4d, +0xb1eb7f81, 0x07819f40, 0x3765274c, 0xde7fd081, 0xa0817f7f, 0x8dbe5081, 0xfa436f7f, 0x4848463b, +0xc5c45cdb, 0x7f8b1865, 0x5bf381b6, 0x45aa497f, 0x7f813694, 0x54f15918, 0x72886c49, 0xe3fb31b0, +0x0db1a582, 0x4aadce1b, 0x39926d8b, 0x9a30a85f, 0x1b606181, 0x1c6d165b, 0xfb057f4b, 0xcd7f81ec, +0x12bf7ffb, 0x7f5f91d4, 0xca8158b7, 0x86134234, 0xf0f17f3c, 0x7f4e05c4, 0x34814f7f, 0xe5c1106a, +0x2d50c67f, 0xb57fa18b, 0xb7816481, 0x7fffaadb, 0x607f817f, 0x5048ea81, 0x82495d02, 0x17819b7b, +0xaba67981, 0xb5817f7f, 0xee57a2b6, 0x3591a602, 0x2885a153, 0x429ba160, 0x757fd0f5, 0x344e9d7f, +0xc77a7f8d, 0x2a7f8160, 0x3198b27f, 0x81c552bd, 0x0374e058, 0x31605de1, 0x40d0817f, 0xb281a37c, +0x156ce27f, 0x91893090, 0x06f58181, 0xc4487f6d, 0x0a7f7f9d, 0xc3a17141, 0xc03c39c5, 0xb581b2b5, +0x526a7f7f, 0x7f7f7f4e, 0x7fec7f7f, 0x95cead7f, 0x161ba8d2, 0x8ad0d081, 0xe4f19557, 0x5bb6fca9, +0x2d813a6d, 0xddbb7f81, 0x1cb1e281, 0xc2b43734, 0xd07f4641, 0x7f81bb39, 0xae9b1b81, 0xbe817f36, +0x1caf7f81, 0x7f704922, 0x108f777f, 0x81662d7f, 0x479ca35b, 0x818513c2, 0xa41d8607, 0xd081b235, +0xf6817fcb, 0x7f015960, 0xadb15f5b, 0x81be2abf, 0xf2b0e29f, 0x81819224, 0x38a7407f, 0xc9817b09, +0xf307556c, 0x7f818181, 0x0281817f, 0x3fce810c, 0x18b47cd4, 0x81e2a53c, 0xe359840f, 0xe27f81ad, +0xa1665081, 0x7ae062b6, 0xe0b27f35, 0xcd376e73, 0xd9a6d373, 0x817ffe7f, 0x02810946, 0x06498116, +0x9b7fbb1b, 0x8d2b812a, 0xff14f781, 0x937de081, 0xb77ff44b, 0x4b2732c7, 0x0720bd96, 0x9e647f13, +0xb831d474, 0x811d8183, 0xa5db837f, 0x8aa1b04a, 0xd4377cfd, 0x7f814241, 0xee8fec39, 0xcc81d881, +0x2e29d770, 0x7ff76366, 0x7fbd363b, 0x74bbd13e, 0x2e817192, 0x9f6b81db, 0x44913bb3, 0x327f0de1, +0x2d7f8881, 0x54f2afaa, 0x107f513c, 0x4c8191d1, 0x117fe67f, 0xffa21764, 0x2c931727, 0xd09ba693, +0xf74e4882, 0x51ed8bc6, 0x3a23d683, 0x47437f70, 0x7fad8181, 0x9a610813, 0xa57ff47f, 0xbe1b5b9a, +0xf4bf8c33, 0x43074944, 0xe4ef729b, 0x1e7165d6, 0x74e081c6, 0xe281810c, 0x2a4b81e4, 0x10ede221, +0xc8375d7f, 0x81c07f81, 0x49cb7f83, 0x5f589c22, 0x276441a9, 0x7fe67f0b, 0x4d7f1d7f, 0x1872814a, +0x05b6a577, 0x8140fa5d, 0xe47f9bc3, 0x44166064, 0x73896729, 0xb5c0867f, 0x247f445f, 0xccc92994, +0x29611b37, 0x27531258, 0xeac27f9c, 0xd43f4e7f, 0xc27f170f, 0x6d9bfc37, 0xfc819181, 0x077f598a, +0xe0c5d398, 0x7f23182f, 0xed171743, 0x20bff85d, 0xb0d0be7f, 0xb8938149, 0x45017f32, 0xcf2cc65d, +0xdf7fc481, 0x7f380777, 0x81b8f0dc, 0xf6696596, 0xff007f29, 0x896e1abf, 0xfaac2d81, 0x5bc357ba, +0x6a397f2e, 0xbfaeb45a, 0xd64e2b7f, 0xe49b4587, 0x565c6c51, 0x7f206a7f, 0xc381540b, 0xfc3d7f73, +0xd4ec9f39, 0x887b7f27, 0xe3c1e07f, 0xb721523e, 0x0310819f, 0xc0c793ce, 0x387da141, 0x814f61ac, +0xd1347fbe, 0x19244c1f, 0x8d3d58f9, 0x813a7fc9, 0xfae27f54, 0x7f39f481, 0xe37f818d, 0x034632d9, +0x5b06c07f, 0x65ab8b36, 0x127f34b5, 0x18a681ad, 0x73510e20, 0x2f2ffa85, 0x6814ca9e, 0x5b7f7ead, +0x08da93da, 0xc826da54, 0xc1686fd2, 0xb6f00fbc, 0x28bb5dbb, 0x7fe5297a, 0xb5ed8181, 0xccd9067f, +0x177f1561, 0x71b41a19, 0xd9817f7f, 0x505b50c5, 0x309dc978, 0xb97f4f81, 0x4e81c219, 0x2d81d04a, +0xdf3d5fa1, 0xfc6e5c30, 0xf7869481, 0xfc81e47f, 0x67b5817f, 0x7f72577f, 0x2c938136, 0xd576ad7f, +0x63beb36f, 0x7fb3817f, 0xce56ca81, 0xd801e58d, 0x425f2522, 0x8a25817a, 0xdc7673c9, 0xd7b57fca, +0x927fbe2a, 0x149b9881, 0x9ad781a9, 0x214b9d4d, 0x2dae1287, 0xfa772ca4, 0x3b7fdd35, 0xf59e4174, +0xad3a28f5, 0x8144698d, 0x1ca792df, 0xdb7c7fd1, 0xe96d497f, 0x567fa7d9, 0x489b70cb, 0x3442de7f, +0xa2554c37, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x4dc76c95, 0x29b78ec8, 0x3d927f7f, 0x307f93c2, 0x8181da7f, 0x99f741df, 0x6e1d417f, 0x1bbda834, +0x954b5a9f, 0xc6817f07, 0x7fc01005, 0x3b5a81e4, 0x73cd7f5b, 0x7f817f3c, 0x8e7f812b, 0x59d48a03, +0x982475ef, 0x65a1c068, 0x400cc79b, 0x81f231fe, 0x840329e0, 0x1f6054eb, 0xf8813d3a, 0x7f4a9296, +0xbff677c5, 0x7f3c6f0c, 0x7f2890cd, 0xe0aaaad6, 0x81ae9428, 0x7f8117f1, 0x7f097f88, 0x7f27cfcc, +0xca67205e, 0x815f7f37, 0x7f0ac07f, 0x717f95d2, 0x7f9232a7, 0x2bb803d4, 0x3c12c7e7, 0x937c81ea, +0x7f1cdd3b, 0x0622b6c0, 0x62d71247, 0x811cb00b, 0xaef9e981, 0x0f818ff5, 0x740e6f53, 0x30616410, +0x81fed3e0, 0x49167f3a, 0x4577315b, 0xe49130fa, 0x817c7f23, 0x7ffef7ec, 0x4fb47f17, 0xb365b1f8, +0x81a09c38, 0x7f818134, 0x9568e717, 0x4ead1921, 0x8ebd9806, 0x7f911cf8, 0x8c7f5efc, 0xf29114a8, +0x90bc6ae9, 0x0b5f26c1, 0x81bc9281, 0xf0fb5532, 0xc89ab77f, 0x7ec0b9d4, 0xf00bdcf7, 0x7f7f7a24, +0x097ff51e, 0x64d62dfc, 0x7f7f819e, 0x34c05cc3, 0xa2957f19, 0x81101e05, 0x7fd68181, 0x1dff81af, +0x7f81dc24, 0x7b3c6769, 0x7ac490dc, 0x814032c5, 0x9d81747f, 0x7fb46342, 0x72607faa, 0xd40c8161, +0xa5ab574f, 0xf0838bba, 0x33fd7f47, 0x289f81be, 0x817f480f, 0x4bfa552d, 0x8ca1d5d5, 0x817f492b, +0x418125e0, 0xe45d5af0, 0x811b1751, 0xe04d81f8, 0x7ff2917f, 0x65811a53, 0xe65da37f, 0xbafdd439, +0x53818189, 0x752c7ebb, 0xc301567f, 0xb0816f27, 0x708166db, 0xc0b57f49, 0x3a818169, 0x4f16189c, +0x13d6e83b, 0xd258a20d, 0x7fadbdb2, 0x7f93560d, 0x81fd5399, 0x6014dab2, 0x9cf45f1b, 0x107f71d5, +0x352e8109, 0x5a5d6efa, 0xb458ef47, 0xb4a96098, 0x7ab91c37, 0x81bd762e, 0x816cb0d7, 0x3f9819d4, +0x8113b97f, 0x7f09fa33, 0x9e7ffa99, 0xe98181ce, 0x4ffb3064, 0x7f0681ea, 0x81ef06ad, 0x385abdf5, +0x1414457f, 0x18dabc5a, 0xc27e0d7f, 0xd87f8115, 0x703f7fcb, 0x300b7ff1, 0xb081f55d, 0x024d78f8, +0x72949a16, 0x90603c38, 0x7fba6fd4, 0x4820e44f, 0xd57f81af, 0x9c7f7ef9, 0x61c6b27f, 0x81b6141f, +0x7c52514a, 0x5f817f78, 0x668119a7, 0xe441aa7f, 0x53819553, 0x7f721521, 0xbd7f7f17, 0x593081de, +0x8b252de8, 0x7f7f81c1, 0xaca9436e, 0xb5e0b104, 0x81e49d81, 0x75b48102, 0x5c278118, 0x9f6c36c9, +0x509481fa, 0x53d1e640, 0x52108181, 0xeb7f7fff, 0xde417381, 0xa781cb33, 0x4fab55f4, 0x3e7d75f1, +0xf17f0c1a, 0xe8e1817f, 0xcf1f7f81, 0x96d8ba40, 0x1318fc72, 0xd7ae3d1d, 0xf7e88126, 0x32bfe432, +0x453f694b, 0x7f7fbab6, 0xeae7de30, 0x3d81d7d1, 0x438d9cc7, 0x387f4a05, 0x7f46ab81, 0x635bdf10, +0xc281ae24, 0xc5c67fc4, 0x81acb53d, 0xba4f4d04, 0x424f72cc, 0xbf7faa1f, 0x55de1de8, 0x81429a1d, +0xdec94147, 0xff007f42, 0x3fa74821, 0x0cb74e9c, 0xff81627e, 0x6343810f, 0xf7393529, 0x7f44ad81, +0x7f811f39, 0x81e981a0, 0x8c8c4421, 0xc4819190, 0x81816881, 0x7f7fd234, 0x3688c16c, 0x9c34b357, +0x6046e3f6, 0x86123bdc, 0xbd45957f, 0xe6cd777f, 0xe0951ce7, 0x9b7f63a6, 0x817f29c1, 0x7f5a81ca, +0xde7ffcbe, 0x812c3207, 0xa099537b, 0x2a2f81d0, 0x16817f9a, 0xbb81817c, 0x7f81e538, 0x957a4200, +0x81753bab, 0xaf63a506, 0x3081691c, 0xb0a47f2c, 0x0f157feb, 0x8e4330fe, 0x635b8189, 0x7f81ee34, +0x7f40caae, 0xbce88e05, 0xabb52199, 0xd2ba102f, 0x7f7f8173, 0x810b32ab, 0x2e222781, 0x7f819951, +0xfbc73dca, 0x5d81d181, 0x7fb09d81, 0x7faa3098, 0x7e81435b, 0x61e995d6, 0x7f7974f3, 0x7fef81f9, +0x402faba7, 0x7a647f3c, 0x815c6826, 0xba2b2ffe, 0x7f095b7f, 0x7f9236b9, 0x9953636b, 0x344c64d2, +0xd784acc0, 0x7f818426, 0x61ab43f8, 0xd5813c70, 0x7f817f81, 0xc9867fa8, 0x6f4b4181, 0x3881b1df, +0x69813f27, 0x81d0780c, 0x17c84a36, 0xbf908ccc, 0x7f5e812b, 0x317f7fea, 0x7f817881, 0xf731c537, +0x7fb221e7, 0xbae09bf7, 0xcc5d947f, 0xcfc13926, 0x6e0db67f, 0xdf17be28, 0xcaa4ea2f, 0x9870810f, +0x357f701d, 0xa281815d, 0x91497468, 0x5fbcbeb0, 0x7fd54d81, 0x7d5e91f3, 0x8d6555f9, 0x35d416e3, +0x9a7f8c11, 0xd875779b, 0x6b7c6681, 0x24f77f50, 0x288bd781, 0x7fad2b50, 0x637f1a23, 0x527f2ea7, +0x277f7fc3, 0x819c13cb, 0xe1bb65f1, 0x48151c1b, 0x8e7fd27f, 0x7f7065ac, 0xfe0bb14d, 0x588e7f0c, +0x819d7f34, 0x81c455a5, 0x4f6b7fae, 0x7fb56506, 0x7fcfd5a2, 0x3e6981dd, 0xcc814569, 0xc5fb811a, +0xc0e07fbc, 0x817f8153, 0xac356315, 0xc94db6d8, 0x7f88e57f, 0x81e57fd0, 0x7f1f6c0f, 0x78d581e1, +0x7f2137f6, 0x1c57815a, 0x627f55c0, 0x69e4dbcb, 0xa68181c3, 0xb9907fea, 0x55897fa0, 0x7f54a91d, +0x3e7bb314, 0x7f167f5f, 0x75cc8181, 0x177f2fd0, 0x6d39f7e9, 0x7f817fd3, 0x7f6eb0ea, 0x817fc7ad, +0xae5b5b04, 0x86cd281f, 0x3528fe7f, 0x87217fff, 0x87860479, 0x8b7f4510, 0x7faa816c, 0xb9d62c33, +0xdd1c6ccc, 0xb48104fd, 0xbf508f81, 0x34994944, 0x8181b843, 0x74e47f47, 0xc6baf38e, 0x624b06cc, +0xfdadfaeb, 0x948189ac, 0x7f7f992d, 0x813edf00, 0x81c3587f, 0x5eb2819f, 0xcee54cef, 0x277f38d9, +0x8a187fb6, 0x9736ed05, 0xed70487f, 0x4f307fa9, 0x81ba975e, 0x81dd81ed, 0xd0da7f88, 0xf24881cc, +0x12e656d0, 0x7f9b6a1f, 0xcf7fcf7f, 0x78787ebd, 0x7f8319f2, 0x7c7fe9ed, 0x29b381ae, 0xfa4f81cb, +0x13cc81e2, 0x9a2aa0a8, 0x817f8112, 0xfda22318, 0x7f65e85a, 0x727f6812, 0x7f814b0b, 0xcfe2bcd5, +0x6b816ce3, 0xb2965ab7, 0x819981bf, 0x73799635, 0x82854b71, 0xbc3e8132, 0x91c714a2, 0x81817f2e, +0x2cc55af1, 0xb87fb038, 0x7f7f1718, 0x02ba14fe, 0x5e348128, 0x817ff97f, 0x81d03daa, 0xde37813b, +0x443781c0, 0x0e4e32e2, 0x85a7ad81, 0xba276c16, 0xb61ed9c6, 0x81ca27d3, 0x7f762e7f, 0x4181d581, +0x7f7f5508, 0x6eb1b250, 0x5ad47f4a, 0x7fd56756, 0xbc817f81, 0xd481a9ba, 0xf8a74db3, 0x7d1aae25, +0x81397f36, 0x7f62a402, 0xab3b3081, 0x4c45c301, 0x3eb57f88, 0x6d2e7fdd, 0xeb1ac681, 0x813881c0, +0x7fd127b4, 0x67e57f90, 0x4fcb6d3a, 0xc566afe7, 0xc58bdb7f, 0xe0486a7a, 0x363f8181, 0x7fdd8191, +0x67149cc2, 0xfe7fcf00, 0x7feeb9dc, 0xccabbd95, 0x7fab347f, 0x98eda0d8, 0x7faf81a3, 0x3b8c7f0a, +0x1c387fa7, 0x7f055ed9, 0x567fbd7f, 0x4d7f7244, 0xbb8175af, 0xef7f7f5f, 0xa9a58119, 0xada9eec7, +0x592da3bd, 0xb5a98177, 0xe17f813e, 0x94816af7, 0x69ec7fc5, 0x7f29dd4a, 0x9481c981, 0x5d3198da, +0xf84e81bb, 0x41817f7c, 0x7fb31b60, 0x7f7f5e03, 0xea81c490, 0x7681571c, 0x9b54b581, 0x812d993b, +0x7fefd381, 0x7f907fe3, 0x7f81a862, 0x758181bc, 0x46d6507f, 0xae81a50a, 0xd1562a1d, 0x3dcb58d2, +0x489e40fd, 0x7f8fa98a, 0xb07f7f81, 0x238b7b40, 0x49686384, 0xca817f4f, 0x607f7fd7, 0x3a0d81ab, +0xb0fa42e3, 0x7f81563d, 0x829a817f, 0x51d48167, 0xcc817781, 0xd54f5638, 0x7fee1218, 0x81a49e22, +0x0081ec2f, 0x5f5eecf5, 0x6bbc9ce7, 0xa28138d6, 0x86811081, 0x7f0283cf, 0xfff67fb7, 0x81c2814a, +0xc7817f52, 0x85818146, 0x8781a77f, 0x8181e9f6, 0x4731f881, 0xf51a35d8, 0x236931b2, 0x6eaefadb, +0x7f81c61c, 0x947f3d7f, 0xbcf43b52, 0xd5814849, 0x17b6e07f, 0x7f6a81b6, 0x817f8163, 0x817f595f, +0x7fa5ae33, 0x2b63a90a, 0x77627e81, 0x81816660, 0xae7f6281, 0x26c6dedd, 0x757ff9c4, 0x7f7fce30, +0xab9d7800, 0x68fc444b, 0x3d7ff681, 0x93264a1d, 0x81361db5, 0x817fb6d3, 0x1681136f, 0xc5655027, +0x792781e2, 0xeaada9c7, 0x34d1817f, 0x81687fc6, 0xa88d451a, 0xdce97f0d, 0x443935cd, 0x810a68b9, +0x73ef4023, 0x66372979, 0x9a31076e, 0x81813dee, 0x56b11a7f, 0xc76a81da, 0xe0e48181, 0x6681b8b7, +0xa99858ec, 0x7fb38de5, 0x7f6f817f, 0x92168381, 0x53819b02, 0xc3c27f2d, 0x5299d77f, 0x81e47f60, +0x687ffbc6, 0x7f5f9df9, 0x0ab4c09d, 0x44856927, 0x5e63405f, 0x7f9fe864, 0x8c884d25, 0x794dd01f, +0x7f81fb68, 0x31bed5f5, 0x3e658638, 0x3e7f8151, 0x7f7ba8e4, 0x7f304a8b, 0x37c18157, 0x7fb2c6fe, +0x7fa39bef, 0xa37ff9a1, 0x29983fc2, 0x777facc9, 0x547ff37b, 0xe6815b17, 0xec5a457f, 0xacade303, +0x7f2603dc, 0x697fcd23, 0x7f7fd042, 0x4a81a54d, 0xd481bf87, 0x7f4e7f0c, 0x0d5c9d05, 0x26ca3c2c, +0x4f817fe3, 0xc9368b0f, 0xaddb8181, 0xc210e075, 0x248f7f81, 0x697f7f11, 0x817f2f81, 0xef697ff1, +0xd98181b9, 0x810a4f50, 0xbb9e045a, 0x8ea3771d, 0x787f6931, 0xfbb9812d, 0xb778f981, 0x7f354b27, +0xfc813658, 0x25818137, 0x7f3402a1, 0x27eabd01, 0x81c18181, 0xfd7fa22c, 0x9784817f, 0x7fa57fb3, +0xd881f3ce, 0x887f60bf, 0xdc816b0c, 0x3890b818, 0xc37fa281, 0x8f4681f5, 0xb07fb08b, 0x82ad24d0, +0xc189ab02, 0xe3fddb8f, 0x7f744610, 0x435781b0, 0x8104f653, 0x41bb81be, 0x9a4c717f, 0xf9d4c1e4, +0xbdc36404, 0x8ff9818a, 0x54d3ab9c, 0x24aec8ef, 0x81ac5bc8, 0x81908100, 0x74f05a0f, 0x0c5b7f2d, +0x5279a3e3, 0x2d8169b9, 0x7fdc4897, 0x7f5f89bb, 0x9f457581, 0xde817fd7, 0x59813d7f, 0x67817fe7, +0x7f817fd5, 0x7f5978ca, 0x23ab813a, 0x70819ce8, 0x41575433, 0xb17fa99e, 0x697f7f7f, 0x60aeb15a, +0x7fa5a9df, 0x39818181, 0x818ea205, 0xbd819fb2, 0x7f0fa491, 0x814981d8, 0xb9c186e4, 0x817f29e5, +0x81afcd17, 0x842f81ce, 0x058b8e1d, 0xea3da29e, 0x379f5ab7, 0x223c84c9, 0x93bcd8bc, 0xc895baea, +0x6b7d3cc9, 0x7f2b8152, 0x7feccb81, 0x7f71634c, 0x810f7f29, 0x89c14aff, 0xc0ce05d7, 0x7aed49e1, +0x817f4d0c, 0xeeaa0afb, 0xd7817f81, 0x6681d76a, 0xa37f4881, 0x38708103, 0x7f8181cb, 0x0a8f6639, +0x8581aec3, 0xe82836d2, 0xda7f4781, 0x40d67ff3, 0xcc335f7f, 0xce207f44, 0xff9181f8, 0xbae52614, +0xfccdf700, 0x9dc19f54, 0x5d7faefc, 0x5bcc2357, 0x817f7fb1, 0x6f5a9dd7, 0x81af5633, 0x810a7f59, +0x551c8a18, 0x4b816452, 0xe981d940, 0x81d175f6, 0xee7f5746, 0x276f51b7, 0x16ae817e, 0x652063ba, +0xd84e5e44, 0x7f385f2a, 0xa2e1b28f, 0x7f817fd2, 0xd8477f34, 0x508f31fc, 0x7e357fc7, 0x7fa07f7f, +0x7f6d7f52, 0x7f717038, 0x7f819c3a, 0x2f973937, 0x71ac7d7f, 0x790950c2, 0x9aa97f81, 0x7f427ffe, +0xf3f974a1, 0x8107812c, 0x817fb77f, 0x4d458151, 0xc93d2395, 0xb497d049, 0x4b955c87, 0x047eaef1, +0xcc6c07cc, 0x5eff94ad, 0x3bed1246, 0x74d8c531, 0x815afc81, 0xaccf6eca, 0xbe4e3f64, 0x079b37b9, +0xa0898136, 0xfe8a81b9, 0xb7df45fc, 0x7f7f7157, 0x725581a3, 0x7f0c56ec, 0xe6430781, 0x4356def3, +0x54ca7fc7, 0x234f3eee, 0x7f270367, 0x704981c1, 0xfe623836, 0x816d2725, 0x7b9d9e81, 0x3b49d3ae, +0xd2925440, 0x819a7fa5, 0xb025c9a5, 0xbbd77fc1, 0x81737f81, 0xbcb891ae, 0x442ab466, 0x812e41cd, +0xbdb081a8, 0x8e7f5c2d, 0x8458dfb7, 0xd77f8130, 0x9bcbbc47, 0x783a7f5f, 0x27c4d502, 0xadf8cb31, +0xa51e9baa, 0x814d812f, 0x7f1a7f8c, 0x1874f4be, 0x4ac57f81, 0xc7b67f37, 0xc0817ff9, 0x8c81e1ae, +0x6f7f9331, 0xa17f7f9f, 0x7bec5242, 0x81b79554, 0xa27f7f7f, 0x181da812, 0x52e681f5, 0x5c3781fb, +0x895981ae, 0x7f7fa003, 0x9555817f, 0x32391fdf, 0x0c7fff6a, 0xb681981d, 0x1b93a683, 0x7fc3d495, +0x83f5810b, 0x854399a0, 0x997fd57f, 0xe87f6929, 0x7fbafb81, 0x7fe201e3, 0x7f7234b3, 0xb6889ebb, +0x8ea7cdcf, 0xd27fab96, 0x9a562c7f, 0x6c7f7fd8, 0x2277467f, 0xb3108ed9, 0x818d0171, 0x81bc477f, +0x650719f9, 0xba7fe501, 0x7b5796ca, 0x3fc581eb, 0x9a90469b, 0x7f041112, 0x527f7947, 0x8d3d4657, +0x7f627fdd, 0x64053a56, 0x817739c6, 0x812268ee, 0x69ed9b7f, 0x3d7f7f1a, 0x7f2264f4, 0xb76c55f2, +0xd06981fa, 0xa72359d6, 0x3708812f, 0x7f7a4c29, 0x467a227f, 0x685eb4c5, 0xeeeb81c1, 0x2999b5e3, +0x7f7fa0be, 0x73586e92, 0xc7817f0a, 0xc32e7f4f, 0x819b9046, 0x8179acee, 0x29b1a517, 0x557f0ef9, +0x8116589d, 0x557f26f3, 0x861970dd, 0xcb817fef, 0x9bb6ba72, 0xaee4ddfa, 0xd17f81c9, 0x257f76bf, +0xb04b4df4, 0x8f7f8124, 0x818181fc, 0x4fcc8f06, 0x2f7cc181, 0x4900141a, 0xf0709c16, 0x6865b5fd, +0xd9ab9128, 0xa2321900, 0xa1a6de7f, 0x58e47f23, 0x54ac4a45, 0x1af1e618, 0xed633c24, 0xde7f7ff7, +0xd48139b1, 0x5a640dcf, 0x63d9de7f, 0x81669358, 0x4f4571bb, 0x7f7fa692, 0x7debaf85, 0x9d7f7f10, +0xd9c34e9a, 0x708981e8, 0xfc838ac6, 0x430bf6b2, 0x81a1a4ee, 0x7f8181ad, 0x35e98d7b, 0x4e3681de, +0x7f7256e9, 0x7fa3b47f, 0x5a813e7f, 0x8128aa15, 0xff42357c, 0x581e98a3, 0x81534281, 0xe58196fb, +0x11b90c6b, 0x7f81e5e5, 0x7473f57f, 0xa9475b2c, 0x24d77fa7, 0xa27f591e, 0x7fba81a5, 0xc9f19310, +0x3c64d05f, 0x86aca9f0, 0x69b57f01, 0xbb747fb6, 0xdd81e923, 0xbe814a26, 0xe6811b22, 0x7f817fdf, +0x7f81e46a, 0x647f8170, 0xf2812d9b, 0x7f765fa5, 0x812dfe0e, 0x2584ed72, 0x811a7f7f, 0x417f7fee, +0x81d28138, 0x7f8e7fff, 0x9c7e28d0, 0xa71eb5f5, 0x65993e7c, 0x927ff0e8, 0x7f5cd4ce, 0x2886b35e, +0x7f81d3bf, 0xc2d1441e, 0x2f2cbadd, 0x8c9c4cb4, 0x34dba16b, 0xaa37c5c8, 0x5881ec6f, 0xfc7f64a4, +0x81bb8128, 0x7395489a, 0x7f647f7f, 0x44ef7f9a, 0xc3b4d6a1, 0x8693af02, 0x87fab099, 0x7f7f2f05, +0x0191d5f8, 0x0b4281cb, 0x615b990b, 0x554b7f4c, 0x7f7f66a2, 0x817f793b, 0x5681aa97, 0x21aea0bc, +0x81925b0b, 0x81c69839, 0xb115814b, 0x9e81d8b0, 0xc70c6f7f, 0x8f14de16, 0x3f81b4bc, 0xaa0c25dd, +0x81f8346b, 0x6981226b, 0x81b1817f, 0x7f2081e4, 0xa2848117, 0xe4cbcc32, 0x817081d5, 0x817dbf75, +0x157f2614, 0x5a0477d6, 0x8b818105, 0x9e7f34d4, 0x48da3235, 0x397ff6d2, 0xa02f7092, 0x817f8b13, +0x307b35ca, 0x783c7fe6, 0xa2c29081, 0x555b8101, 0xf764ee86, 0x463a8149, 0x577f837e, 0xb1948151, +0xbd2f86f5, 0x6e5fc338, 0x7f67a373, 0x55817003, 0xa1e17f77, 0x8a817fff, 0x49365e44, 0x817f8933, +0x209ba8a9, 0x7f311515, 0x7f519140, 0xa9549352, 0xbd817f8a, 0xafc4a005, 0x620d407f, 0x88302fd2, +0x5cf481f2, 0xbd7fff40, 0x7f7f9cfb, 0x91ae81fc, 0x73f15c81, 0x93814581, 0xc7e8818c, 0xc481a1fa, +0x99818124, 0x7fecdbf3, 0x207d8d39, 0x64818328, 0x7f9dbf5d, 0x2181f9c2, 0x817f7f8c, 0x367c1801, +0x2a019011, 0x815f5d39, 0x81817fb8, 0x7f22815e, 0xb3ceca8f, 0xa6560501, 0xfb548181, 0x2b7ffe18, +0x5394bb1c, 0x7f9981d9, 0x7f7fce32, 0x815ba6e4, 0xabd0d732, 0x7f5c81fa, 0x7f369f77, 0xa42b7f51, +0x900db9cd, 0x9993bde1, 0x7fc8819a, 0x116f9310, 0x8ef767b0, 0x7a817f02, 0x56819b81, 0x29aee511, +0x7f2b8d3a, 0x818a442a, 0x555c3ff5, 0x1d6c4929, 0x7ea9697f, 0x780b8fa6, 0x097fa426, 0x9b7f89d9, +0x165081de, 0x45a184c0, 0x405c7fc7, 0x6bbcd6d3, 0x7b81837f, 0x33867f12, 0x878f7f62, 0xc6cf1002, +0xad819f7f, 0x3df9422a, 0x77ca9e81, 0x684f7fca, 0x30847f81, 0x81817fda, 0xedc6a081, 0x682c00eb, +0xca767fdd, 0x1f7fb0e3, 0x377f8a7f, 0x5e85d200, 0xb9237f1f, 0xf2817f2d, 0x0dd88132, 0x81cc0037, +0x739b39bb, 0xb47fb83c, 0xdf7f8181, 0x7f54f610, 0xbc9e3f06, 0x5762b272, 0x81ebc628, 0x7f9845b9, +0x7d901f63, 0x81ce7f36, 0x0ebd8e7d, 0x9a81d963, 0x644c3f81, 0x227fbdeb, 0x8e2d5f81, 0x54efc59a, +0x94ed5511, 0x817f944b, 0x38393d81, 0x592f7ad2, 0x75814881, 0x63275d7c, 0x4a65de10, 0xeb818ff6, +0x70f93fc1, 0xbbe83610, 0x7d94beba, 0xfd077f2f, 0x7f818181, 0xbb7f47d1, 0x4c8f7f6d, 0x587fa2fb, +0xbd817f02, 0xae7f8123, 0x4fb981ac, 0x3fd2c446, 0x817f7fa1, 0xb28d81ce, 0x7f813fbf, 0x5a8181a8, +0x2f6355c9, 0x75624fb5, 0x811842d0, 0x70a38146, 0xea52aef5, 0x7fffa3f0, 0x8f7fdcb0, 0xe31d81e5, +0x7fe15a51, 0x21894817, 0x9c810f41, 0xe9428145, 0x7f819edc, 0x7feb1761, 0x81368181, 0x2c71f15e, +0x26813c54, 0xf09c69aa, 0xd17751da, 0x25a1c23c, 0x7f812281, 0x4317b30f, 0x66908149, 0x81869b95, +0x062874c1, 0x843481a4, 0x81743c89, 0x813e9cae, 0xd8397fd1, 0x7b495f55, 0x7f7f7f53, 0xc0b419e7, +0xacf11d1b, 0x90a51003, 0x9efa5bb6, 0x77b8989f, 0x8b1ea17f, 0xc46537c3, 0xcca93d93, 0xa955d1db, +0xc42743ed, 0x7f8a6442, 0x79ab8152, 0xfea48ff7, 0xfbbb819c, 0xa1be81ed, 0xa481da42, 0x477f7fd0, +0xcc1acdfb, 0x767f1533, 0x99cdda46, 0xfb9c8109, 0x7f757f7f, 0x747f88d6, 0xb5404df1, 0x8b69b739, +0xc762670a, 0x8125bded, 0x6481a452, 0x943e5aef, 0x7fe3557f, 0x8181bfc5, 0xb14d907f, 0x7f747f9c, +0x54b4ff28, 0xe19d7fc2, 0x7f25887f, 0x7f6cc0b5, 0x34653ba1, 0xacd70015, 0x7901cc3b, 0xb081815e, +0x812781fd, 0x871cd81d, 0xd1818494, 0xa97fc1ed, 0x81817f86, 0x9f3cf7da, 0x9857bb81, 0x9d1661d1, +0x167f5814, 0x037fd0e4, 0x7f337f7f, 0x7fd40904, 0x376a4a69, 0x7f77a308, 0x73b61d36, 0xb4a318cc, +0x7f9cc3e7, 0xe0d55532, 0xab58707f, 0x9c467f07, 0x8c7f2581, 0x60155a41, 0x7f3fd21d, 0x7f7f59a3, +0x38176d64, 0xf77fd7a6, 0x0ed2ae81, 0x7f81cdec, 0x3c0f81a2, 0x747d9be6, 0x818c5f83, 0x432e7cfb, +0x4a44b96b, 0xc8748423, 0x6b818181, 0xe34914d7, 0x980b64bb, 0x7fbe165f, 0x7f104bbd, 0x5c7f9d02, +0x7f307fcc, 0x8399c1a3, 0x7233316c, 0x92aae7a6, 0x81deda79, 0x812805f0, 0x7f566985, 0xdeffbea8, +0x2bbf427f, 0x7f819e05, 0x90811687, 0x817f81a0, 0xb581c481, 0xa05cfdd4, 0xe4983e52, 0x813e81e7, +0x2b5ba42c, 0xf75aaf0c, 0x4bbc8181, 0x817fa502, 0x1a7f2d7f, 0xe97f0700, 0x724a5029, 0xe3700308, +0xae7fb93c, 0x7f9581ba, 0x819d7f3d, 0x2b2ef875, 0x341c52ae, 0x21b23dfc, 0x6a82a2e1, 0xaa81ecc7, +0xcf817ff6, 0x7fa17fb5, 0x1fc21a81, 0x817f895d, 0x54973cd6, 0x81a8c919, 0x9f4c7f38, 0x89817fd1, +0x325d7fea, 0x959581c9, 0x711638c8, 0x07bb81c4, 0x1bf0818c, 0x346b81aa, 0x5d55817f, 0x7f7f8192, +0x3b7f81d5, 0xe783f367, 0x863038b6, 0x7f53589b, 0x7f4de47f, 0x128165ac, 0x7fa6ccfa, 0xc19f8132, +0xc3b3483e, 0xa63d628e, 0x2d707fd1, 0xd93a1319, 0x9197277f, 0xd77f0fbf, 0x8156da7f, 0x88819a98, +0x3eb240f2, 0x81007fbe, 0x067fe653, 0xd3aa8150, 0xa625819b, 0x767ab13b, 0x937f817b, 0xd08181e1, +0xa47f81b9, 0x3481bd4a, 0x65507fa7, 0xb79181d1, 0xbf1d7d3e, 0xb2818178, 0x8f329d1c, 0x2081b481, +0x7b84a853, 0x7b679ce3, 0x9ba49881, 0x695ab122, 0x23be9081, 0xd7322a4a, 0x21db3fad, 0x81fc8145, +0x84a1bad2, 0x9f307bb8, 0x81815581, 0xb84f1450, 0x6d3acdb7, 0xfd64811d, 0x7f8132e6, 0xd7458fee, +0x81c54610, 0x7f7f73d1, 0x81a5c3bf, 0xe15f7fa0, 0x3c7f517f, 0x047f7fd0, 0x261e438d, 0xb3c47f34, +0x5c81a1f6, 0xa2c62f5a, 0xe67f9255, 0x4df92530, 0x58877578, 0x817fdbd6, 0x9381d867, 0xb3815eec, +0x3d8b8ad4, 0x814aab55, 0xa5477f7b, 0xed7f98d4, 0xae7f3681, 0x3b938138, 0x58e37f90, 0x7356b00c, +0x754181fd, 0x3181503e, 0xc58dc451, 0x818b9cbe, 0x747fcbc4, 0x0918d1f1, 0x5b5ae00e, 0x761d6116, +0x535ba62c, 0x8c0f22b4, 0xc98fb9ad, 0xad7f6d1d, 0x4f7fec7f, 0x1b437f35, 0x9004b258, 0x787c8d15, +0xd04b81e6, 0x703d10f0, 0x7f7f8b89, 0x1674b61c, 0x9789ac81, 0xac6c7f58, 0x0e81ac48, 0x0881812d, +0x117f7cc2, 0xa6bd8145, 0x59817f7f, 0x89cbb4fb, 0x697f9a81, 0x813e5281, 0xf57f4898, 0xee994b1d, +0x60228cc5, 0x4c341db8, 0x7f6e7f63, 0x107f8151, 0x14397f55, 0x81a951d6, 0xb762ed40, 0x81c39dd6, +0x546a0438, 0x7f8e88d7, 0x81810e7f, 0x7f7f654c, 0x3b412da8, 0x8aabcfe0, 0x4b83444b, 0x65bea3ee, +0x507fb43f, 0x16c3c7c5, 0xe322a2d9, 0xfd5a0fd9, 0xc757577f, 0x6cd0b9af, 0xc6078100, 0x3256b728, +0xc47fdc2f, 0x81dc4bb9, 0xd8d02073, 0xc38671f2, 0xbb9104b4, 0x7fb281b6, 0xcb817f81, 0x819f44af, +0x29f781bc, 0xdfc231b9, 0x7fae7f43, 0x575b7f79, 0xf4c4414e, 0xe807d15c, 0xd08181ae, 0x8e81cd9f, +0x942472c9, 0x816362dc, 0x9cba9281, 0xdc6fce74, 0x2b7f7f35, 0x37810c4f, 0xe8709e85, 0x5c7f984d, +0x8153bc24, 0xbfe581bd, 0x9a7f9c84, 0x3be9e533, 0xb6938127, 0x82ca63d5, 0x08eeb19f, 0x59ebeb0f, +0x4d7f14f7, 0x3c558f40, 0x62d26981, 0x8181f3a8, 0xe42585ea, 0x7f08817f, 0xaf8181a0, 0x0c5d16cd, +0xa4dfee3d, 0x3a8182fa, 0x6381883a, 0x7fe9d01c, 0x5e659f90, 0x7f7f812f, 0x4a9d8179, 0xb1e4acf3, +0x1b8189f9, 0xd983a4d8, 0x817f81df, 0xa2cf1f55, 0x817f337f, 0x7fa3564e, 0x14ae7c44, 0x4ca3859d, +0x81b37fa2, 0xf8a0b7b1, 0x7f79c0cc, 0x7f3010f6, 0xa5be8c16, 0x2420b129, 0x7f7898f1, 0x7e818139, +0x7fc919eb, 0x61819a67, 0x88928b15, 0x7eade0d3, 0x4e7fc323, 0x8deb5fe6, 0x7f3f7f9c, 0x81817fbf, +0x989026db, 0xef81c203, 0x81e4a1e1, 0x35939d41, 0xbbaae870, 0x3d217fca, 0xddd9a1ce, 0x7f7f81a3, +0x5bf18122, 0x81c55a52, 0xc95c450c, 0x7f5a03e6, 0x65654a14, 0x75d401f6, 0xe0cb297f, 0x5f3c8132, +0xf7a312d3, 0x81c6d3af, 0x978e8181, 0x1476539f, 0x502534be, 0xd2909112, 0x6ad663ad, 0x7f7fbbac, +0x8b70da23, 0x4d78af31, 0x7f8159f5, 0x81bad147, 0x348151dd, 0x5e68eace, 0xc2737fa0, 0xf97fa32e, +0x7e7f81de, 0xa68baddb, 0xbb39817f, 0x7f9bc454, 0x6d09c327, 0xc37f3c33, 0x0a865f7f, 0xba8281e1, +0x1a3ff406, 0x652c3fd1, 0x94af367b, 0xb381d8cb, 0x4045cce5, 0x7f7fd613, 0x795f7f42, 0xb1818193, +0x7f371d3c, 0xef816b41, 0x2ca91acc, 0x48b5aa83, 0xaae807de, 0x26813f13, 0x81817f2a, 0x6083604f, +0xd29e81f6, 0x7fb218ca, 0x4aa50170, 0xc2d74a22, 0x4f4971a3, 0xe9dc5f10, 0x7fb8bedf, 0x1f93814a, +0xa97f7f0e, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 + +hard_output0= +0xb5bc6ac8, 0xf5373664, 0x1310345c, 0xd5bae4e7, 0x1fc9e83e, 0xebfdfded, 0x84bd86ab, 0xb7aabe00, +0x60b44fea, 0xb9067464, 0x30325378, 0xa9195955, 0xf70c6e5c, 0x90922632, 0xc90b1cdb, 0xf2f5fb69, +0x73056b63, 0x1a33bf3f, 0x17755b5c, 0xc58bff6d, 0x2f4390b2, 0x2869d508, 0xe7c7dfe8, 0x38552963, +0x21da5367, 0x07282b9b, 0xa4767105, 0x1e294251, 0xe350a940, 0xb8a6aa27, 0xed12d778, 0xf10d9ece, +0xab93527f, 0xcf2da7e7, 0x68f6d0b1, 0x811f4bca, 0x577b06b2, 0x3234f13e, 0x30bab7df, 0x8dc47655, +0xbb843bed, 0x86da3aba, 0x30950c97, 0xdd096d7a, 0xa871fd6c, 0x8bee4e6b, 0x8fea30d0, 0x6c05b4d2, +0xf3e144d3, 0xd24ebb1f, 0x065635e5, 0x8d3f2cf9, 0x536c6c6a, 0xfbb0a5d0, 0x3d707b42, 0xc44d5982, +0xa5f4ad8f, 0xf32c0970, 0x1bccf1a6, 0x05916020, 0xa64fb176, 0x5ede6a35, 0xaf4966da, 0x9df5e0e7, +0x75042abc, 0x9ef10481, 0x11ddcbc8, 0xa0f5518c, 0xd5c23418, 0x2393d558, 0xfbe7dfeb, 0xed1c64c2, +0x86a36508, 0xde2dfb1e, 0xb8d0fef9, 0x24505232, 0xc894e71c, 0xbcc752a0, 0x40b74e83, 0x90d23c8c, +0x728e4a61, 0x108f0b08, 0x66f522ee, 0xc258d851, 0x35a31c44, 0x11311b5b, 0xfd3d5be9, 0x5ae448ff, +0x4f64994b, 0x5b8247a9, 0x4021114d, 0x2f0b6e82, 0x5eaa9828, 0x50ac71c0, 0xfb86ee52, 0x0dc1ac9b, +0xbbd47645, 0x8f357115, 0x978ceea0, 0xd557db99, 0x99b30388, 0xfc9a8a1c, 0x0f75be1a, 0x50143e22, +0x8840989b, 0x738ec50e, 0xe6b2783d, 0xf67899c8, 0x27ebed69, 0x6c415a16, 0x3a6cc2dc, 0xcd4e4e5d, +0x6cb12b2e, 0xdb88d7c0, 0x79cd1582, 0xbc422413, 0xe72ad2f4, 0x8eaac30f, 0x0bd86747, 0x6d87f69d, +0x15d62038, 0x4b375630, 0x0d51b859, 0x16db2cb2, 0xf210603a, 0x0abeb833, 0x55c694d0, 0xe57ca43b, +0x0ba94428, 0x1398a406, 0xe47d3889, 0x5a20203d, 0x250d7a1a, 0xd930ffec, 0x03992e79, 0xf2759376, +0x024ec121, 0x91fc3a2c, 0xb7e11cc5, 0x4ff7d459, 0xb8700134, 0xd6e61758, 0x4eba0a32, 0xb747e3ec, +0x7073fad7, 0xded80f99, 0x331e2f1b, 0xfa1f1bed, 0x056424a2, 0x1d1d95e0, 0x550b9ec8, 0x51ee2a38, +0x19525153, 0xd70c4cd5, 0x0d6cd7ad, 0xe44d1cf2, 0x30dfecda, 0xdacd7fe8, 0x7321d795, 0xddf48ef4, +0xe271e6a4, 0x9c1feecb, 0x951fcd7b, 0x8acc5a03, 0x3fb83527, 0xe306de74, 0x7b9cd6ee, 0x8e140885, +0xd4c91e8d, 0xe8c39733, 0x0f02f87f, 0xfb06b1b9, 0x0dc9349c, 0xf76bae8e, 0x4f642a07, 0x3d48a9aa, +0xe3ea323a, 0xa1cd5c8a, 0x40aa0e70, 0x132042d3, 0xa9732f6c, 0xd15a00c4, 0x43d3b046, 0x9a51ebd4, +0xc46ee0ed, 0xe2a2148b, 0xf5c478f0, 0x1fb01cf3, 0xf4f321ec, 0xd973811f, 0x11ad11b9, 0x5c67adda + +e = +34560 + +k = +6144 + +rv_index = +0 + +iter_max = +8 + +iter_min = +4 + +expected_iter_count = +8 + +ext_scale = +15 + +num_maps = +0 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_posllr.data b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_posllr.data new file mode 100644 index 00000000..fe4f5eef --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_posllr.data @@ -0,0 +1,645 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_DEC + +input0 = +0x81814e1f, 0xba81fb11, 0xc4554696, 0xe17e9454, 0x427f8119, 0xa8b355f1, 0x244ba17f, 0x1c816fa1, +0xc94f5bcc, 0xb5b97fbb, 0x634b8e81, 0x815a6cc1, 0x7f6bb016, 0x2f7249d6, 0x604664f0, 0x31817f81, +0x6fa66b81, 0xd49763f6, 0x777fa4f1, 0x81aa6da8, 0xf0fea881, 0xaccfbfd0, 0x7f81e582, 0x7f58a2d6, +0xa77f430e, 0xf47f50f1, 0x997f42f2, 0xd7814f81, 0x037fe14d, 0x81f98125, 0x2d813e55, 0x7f81f0dc, +0x7f7f8181, 0xc37c723a, 0xbc7a3364, 0x8193817f, 0xe381066d, 0x7fcab836, 0x2481eea2, 0x567f5968, +0xa07d720c, 0x7fa13343, 0x7f35636d, 0x7f817f78, 0x383d2a69, 0x7f860e2a, 0xca78797f, 0x7f7fe912, +0xc95a8141, 0x770c9c4e, 0x81e2817f, 0x811b3f81, 0x745f1104, 0x13817e31, 0xb681a63a, 0x8101482a, +0xbe81818d, 0x8c8181fa, 0x2133de57, 0x8147677f, 0x8a098d7f, 0xd429f1ee, 0x284fc81f, 0x7f5c16d7, +0xded34e7f, 0x3d60812e, 0xdf817f89, 0x2f33793c, 0x2f7f9961, 0x8181221e, 0x847f59d1, 0x9281a681, +0x89503793, 0x01817ff0, 0x183a53ba, 0xc99622f6, 0xbc819881, 0xa7ea2ec8, 0x787f957f, 0x26819378, +0x1e4e4ba1, 0x8bf18138, 0x3adc2c81, 0x3a7d8187, 0x7f22ae8e, 0x307c81d7, 0x7f53a881, 0xc281819f, +0x38fd2090, 0xdd810a00, 0x74ff7fa2, 0x7fb2f27f, 0x7f9b8211, 0x55d69e44, 0x8d2cd44c, 0x5610c676, +0x7f815b81, 0xa2811e3d, 0x93135d81, 0x92797f73, 0x8a8192b3, 0xa48a7f66, 0x7f7e4a10, 0x242dfd81, +0x7fe8b481, 0x810fb0e4, 0xde390fa7, 0xbb5a7f81, 0x65ad3b19, 0xa96a96a2, 0xf681817f, 0xb6a4eba2, +0x0d81017f, 0x5369080e, 0xfbd1c781, 0x879d7f7f, 0x81818181, 0x7f1d061c, 0x8aa731fd, 0x4386a67f, +0xba445581, 0x5b81975f, 0x467f7f7a, 0xc85db95e, 0x6f587240, 0x29a07904, 0x7f7f7574, 0x815dbeb2, +0x81771181, 0x7f54e8c3, 0xc6586fd6, 0x3c6d835a, 0x86b4ed95, 0x7faa72fb, 0x7f7e7fd5, 0x154f81e3, +0x7c4881b4, 0x0c164a37, 0x7ffdbf69, 0x7f7d43f6, 0x5f47da8a, 0x3236d20f, 0x65818181, 0xa37fcf7f, +0x7f8116cb, 0xf6947f50, 0x66ab65b1, 0x6d688a6f, 0x537f6bf0, 0x5c864ddc, 0x06a98181, 0x455854c2, +0x9a6c49ad, 0x129d3c29, 0xe94c327f, 0x7d40247f, 0x9681bc7f, 0x7f288120, 0x81a43b40, 0x08d81689, +0x457f815d, 0xc27f9ae7, 0x7f819655, 0xdabf8181, 0xf4c07f81, 0x6410e8cc, 0xce4d8181, 0x19b13059, +0x8181bf99, 0xf2819620, 0x7fc37f4e, 0x7f8efe81, 0xb25a3e8b, 0x7fbbdb22, 0xba7f8181, 0x8135d42f, +0x817f6836, 0xc381a056, 0x7381547f, 0x1e407f13, 0x9b917f7f, 0x4e526d9b, 0x9323737f, 0x7a20429b, +0xac6b14a8, 0x3568e8d7, 0xbe621a97, 0x7fbca13b, 0x2dbda9dc, 0x32af4ddd, 0x6e4f978e, 0x10ed8172, +0x81e76681, 0x7f427fc4, 0x7fab7f81, 0x2c7ffd81, 0xea817f7f, 0xa62fcf2c, 0x8144bfb7, 0x7ffc93a0, +0x79e9b564, 0x6c449d46, 0x987f8481, 0xee118173, 0x867f0c0a, 0x81819347, 0xb8ddbf82, 0x5c6866b8, +0x82b48881, 0x5c817fdd, 0x7f6605e0, 0x4181bc81, 0x736b51a4, 0xcd5a7fd2, 0xf939a42b, 0xcd817faf, +0x5e1b1fa6, 0x7c818ef7, 0x070c47cc, 0x8e5981d1, 0xe661d182, 0x54deb54e, 0x7fba81f4, 0xc1712e73, +0xb6885086, 0xae9759e2, 0xb12b814e, 0xf5ae9e41, 0x16c736f6, 0xa981c3e2, 0xb06c7f29, 0x497f95b8, +0xcb3de381, 0x7f7524e3, 0xdcf88147, 0x811e81f9, 0xab7a9c7f, 0x816223d8, 0xf97fe22f, 0xe5cf814e, +0x744f6312, 0xc797e05a, 0x279b237b, 0x622b01b9, 0x37818181, 0x8174e3ab, 0xfa5a69ff, 0x62cc787f, +0x81944e7f, 0xc87f7f2f, 0x30839f54, 0x8122090f, 0xf8e98181, 0x28815f40, 0x0262424b, 0x7fc3ccd0, +0xdf7f7fe2, 0x0c264dfa, 0x5d954581, 0xfb188181, 0x6b810d33, 0x1c7fbee7, 0x8141c17f, 0x7f768155, +0xbd207f27, 0x794079d7, 0xf34bd97b, 0x7f7f5026, 0xb46c9782, 0x7f81574e, 0x81817f27, 0x75267f6d, +0x4a7fe97f, 0x7f879ee0, 0x25f8d63f, 0xd481a3c5, 0x747ffd0d, 0xb33530e2, 0x813f5881, 0xbb8181dd, +0xa2da462a, 0x817f8c0d, 0xd0f061f8, 0x117bda34, 0x1897f8b7, 0x2e477f50, 0x8181814b, 0x7faa1181, +0x6ee68168, 0x811a98e2, 0xdcb2b784, 0xa20a7fb7, 0x41445681, 0x0bffeefc, 0x02817f9d, 0x743071cf, +0x387695bd, 0x847ffb4a, 0x0a287f56, 0xf9b4d4d3, 0x2a7fa481, 0xe3eb2e9b, 0x327f91f4, 0xec3981d2, +0x6681f83c, 0xe675722e, 0x426ac913, 0x4b6154a6, 0x5c30816f, 0x817c8155, 0x277fa781, 0x8dd8a9a3, +0x48c609a0, 0x4bca47bf, 0x7f3f6d81, 0xea497fb3, 0xc181507f, 0x81bf1ffe, 0x8e818916, 0x6ba83d7f, +0xb8547f62, 0x5881670e, 0x7f6281fb, 0x6e7f9250, 0xa7bc817f, 0xb19450aa, 0x7f7f9d58, 0x20f69c1e, +0x7f45815d, 0x817f9319, 0x7a5496b0, 0xb85e8950, 0x72c9337f, 0x81781aba, 0xfecdaf28, 0xe2195ab5, +0x7fbb0ec7, 0x7f7ff8a7, 0xab818181, 0x978f2e6e, 0x7fc38121, 0x18945319, 0x264a8181, 0x93b1dc81, +0x630ed781, 0xe27f8e14, 0xc17f5b93, 0x7081fe2d, 0xbb9ac790, 0x7f81ba5c, 0x7f812f32, 0x68bd4f7f, +0x81217f81, 0x88b87f26, 0x285e7f3e, 0x29817828, 0x747f7f81, 0x159eaacd, 0x8d02b252, 0xa98881e7, +0x817f1681, 0x367fbd38, 0x8a8441e3, 0x81447f1f, 0x8165424b, 0x811676ee, 0x81a48122, 0x4b02ea7b, +0x5c267f38, 0x8195acd2, 0x7f7fb581, 0x9eae092d, 0xab12457f, 0x96f8dc31, 0xa89fd810, 0x222b7f7f, +0x1dd67f65, 0x873a81e0, 0x0e7fd97f, 0x81a83d97, 0x8106d381, 0x7f817f0c, 0x8104d4f9, 0xbb81677f, +0x01be7f39, 0x297297ab, 0xc2f01b81, 0x9dd57f81, 0x81ff692f, 0x7fc28115, 0x077f547f, 0x717fcf81, +0x3d81ed83, 0xff6f7fff, 0x7f50f95c, 0x5650a753, 0x81817f09, 0x963b7057, 0x7f817f60, 0x987fa099, +0xf6207f7e, 0x4c814e98, 0xcd27b681, 0x41bd2ebd, 0x340172b2, 0xbc151ec1, 0xf3b403d2, 0x817f6c81, +0x7f2f3218, 0x26f2ea5d, 0xcc59bd8a, 0xef857f52, 0xe37bc195, 0x7f2a2e13, 0x7381a51a, 0x31e14c60, +0xc2e23fc6, 0x8113f40c, 0x9b2d7f81, 0xf6e1147d, 0x813c7fe1, 0x9a9d7f3f, 0x7fb939ac, 0x573181ad, +0x608a317f, 0xb486b9ee, 0xe67fb10b, 0x62817f8b, 0x7f7f9775, 0x81e6819d, 0x7f6f0f61, 0x4ffd8158, +0x81ae56d2, 0xbe4aa4f7, 0x6c515581, 0x435d8326, 0x7f0e2a95, 0xc28d81e1, 0x357fa0cf, 0x2b9f7f11, +0xd491057f, 0xbc7bddd7, 0xe5b57f34, 0xfba68549, 0xb491d2e3, 0x4fb61f49, 0x81bd3e3c, 0xdd9848f3, +0x81810561, 0x67f5fbdd, 0x8c4dc25b, 0x2ab47f67, 0x68137f5e, 0xcfe53817, 0xa7679d31, 0x501213b4, +0x7f7f1390, 0xbefeccd8, 0x30fe9044, 0x7fd6812d, 0x813cf0e6, 0xccb82825, 0xfa78ec58, 0x024b90e4, +0xba25ec7f, 0xc43b50ff, 0x7f7f3e4d, 0x64c38181, 0x81ed1dba, 0xd58b7f64, 0x0b81311f, 0x03563b7f, +0x817fb491, 0x81e895c7, 0x817f7fb0, 0x1681a581, 0x7f998a50, 0x489b81a9, 0x81bc81a2, 0x3d9e7fde, +0x7c7c6381, 0x817447ec, 0x6e7fc32b, 0x7fab86c6, 0x95b44181, 0x3a7f756c, 0x81458168, 0xc4787fd3, +0x731cda81, 0x73127f03, 0xc9077fcf, 0xc37f9c2e, 0x27817f7f, 0xc4361507, 0x7f9b3581, 0x9d357f2d, +0x7f81166c, 0xb87f815f, 0xb2604f95, 0xc44b5591, 0xcfb8078b, 0x7481e64b, 0x8152a679, 0x53fa387f, +0xfa7f3f79, 0x7f5b5fe5, 0x1e625d7d, 0x1081ad93, 0x7f5019fe, 0x3d8145a7, 0x629f9918, 0xc1ab3742, +0x7f66c057, 0x9cb048dd, 0xeb6e859b, 0x8e5fb72c, 0x94d8f5a4, 0xa381280d, 0x67737de6, 0x45817f81, +0xf055c17f, 0x6a0aafff, 0xdb818cc6, 0x81fc60dc, 0x20377f4c, 0x7feac6de, 0x72e78c7f, 0xdf532124, +0x81d33356, 0x8881bcf9, 0x95817f32, 0x612eae3b, 0x7b186965, 0x50b77f63, 0x718181b3, 0x42d8b481, +0x88edaf7f, 0xa3cd25f6, 0x7fdd7f81, 0x7fb79681, 0x7f786e7f, 0x7fbb41c9, 0xd2618121, 0x1cacc6d3, +0x7fe2a0dc, 0x819e1a46, 0x563081e1, 0xb18d77b1, 0xb4817f7f, 0xa97f81fa, 0x2e81157f, 0x31e67fa4, +0x5b814281, 0x98b18124, 0x7f0a3540, 0x49d508d1, 0x2693bc71, 0x3aeb81ca, 0x817fc081, 0x6674b9e7, +0xa1217f23, 0x517f81f1, 0x7fc4ef26, 0x2914ef7f, 0xa9ff7a19, 0x7f7f0ce2, 0x1b485e45, 0x7f70af0d, +0xd6812a71, 0xc95fc63d, 0xa57f03c8, 0x9dbe7f81, 0x7c978192, 0x81a22306, 0x96447f2f, 0x7faf2728, +0xc2a494ad, 0xd5c7813a, 0xa02b7bb8, 0x3bc04eed, 0x997fa97f, 0x7768d0d0, 0x8a84817f, 0x8f1102c6, +0x7f8115be, 0xfb7f7869, 0x7ff25de8, 0x6745f276, 0x0f23b585, 0x52933552, 0x7fab5340, 0xeb4c403f, +0x7f5465ec, 0x77b0a942, 0x6b67297f, 0xc86a7133, 0x7f8f177f, 0x869e28f4, 0x7a7fb066, 0x81a3b865, +0x5d7fba81, 0x815fd7ae, 0x60817ff7, 0xa633a381, 0x45813981, 0xade1ccd7, 0x814981a6, 0xaa93770a, +0x7f83b4dd, 0xc0967f1a, 0xed7f3764, 0x53772ea8, 0x5a81f453, 0xaff56f92, 0x81067f5f, 0xf596ef1f, +0xd520de7c, 0x81b2bdc8, 0x7f8130dd, 0xfb7f8c41, 0x99816bf6, 0x695a60e3, 0x7f7f817f, 0xed987f26, +0x8f818166, 0x417fecfc, 0x99ec379b, 0xaf7e7f7f, 0x360dd75f, 0x787f44f6, 0x81837dfb, 0xd481a681, +0x81819d17, 0x8781322c, 0x7f7fac0b, 0x0fca7f3c, 0x40b5e37f, 0xf27f2ebc, 0x41c19a41, 0xa6c1d381, +0x066de9ec, 0x32b8a8bc, 0xdac61ba5, 0x812d817f, 0x87b9821c, 0x817f7920, 0x7f557fbc, 0x81738181, +0x918fb098, 0x81ab890a, 0x81b881bf, 0x7f7fc57c, 0xcdbde77f, 0x7f5f7f3a, 0x36e59a81, 0x19a6818f, +0xea7fd850, 0x20606e2b, 0x927f9c22, 0xd27f819f, 0x3c4f7f73, 0x168176ce, 0xf0687f81, 0xec5f7faa, +0x23aa789e, 0x2de996b5, 0x7fd07f7f, 0x9785c156, 0x1eaa3235, 0x9781ea6f, 0xb2de74fb, 0x815a98d6, +0x8cb77f92, 0xbba85fa1, 0x61afcac8, 0x81817f64, 0x8149865f, 0xa251b7ee, 0xec7f8181, 0xf8ad7f84, +0x5c8142bf, 0x212981f2, 0x7fc9e381, 0x37338e67, 0x7f3099c6, 0x4fcdb7d1, 0x0381c0fd, 0x778181e4, +0xe7702cfb, 0x61c21e43, 0x3f987f81, 0x3a0b815c, 0x7d12b8c2, 0x70097f0a, 0x7f244d63, 0x2d81187f, +0xd87f6f7f, 0x7d267f2c, 0x8eaf2486, 0x87d8013d, 0x85367f7f, 0x81817f85, 0x0a067081, 0xc96f952c, +0xf3c14485, 0x227f4330, 0x60ec8124, 0x41f181f5, 0xdb6b7fce, 0x819467cf, 0x81bc7fd4, 0x2c7a7f7f, +0x936ade53, 0x7f81d04a, 0xc4817f81, 0x7fdd78dd, 0x81d3a47f, 0x917fade5, 0x6c81510e, 0x5b16fd74, +0xbe987f81, 0x7fb17404, 0x7fb8e8f7, 0x7fb381e6, 0xc479b081, 0x5c81eee7, 0xb19b0e81, 0xa381d4df, +0x81bacf5b, 0xa67fb9a6, 0x817f11a0, 0x8d565870, 0xce818181, 0x7f4aec3f, 0x813ab981, 0x1b495a42, +0xab7f928d, 0x8106014f, 0x47c3e57f, 0xc418f11e, 0xc9149f6c, 0x1a7f1e25, 0x7f3f2d81, 0xcd5e462d, +0xa8d57f7f, 0x812481db, 0x7f697f97, 0x69cdb891, 0xc31fc20d, 0xdc812b31, 0x57e37f81, 0xa09681aa, +0x7f817f88, 0x1ae385f8, 0x81607f7f, 0xfb752e65, 0xd8dd8141, 0xb17f50a9, 0xb8737ffd, 0x1f7f815b, +0x7f9a9e69, 0x06448b38, 0x8109a225, 0xb69e064b, 0x9881537c, 0x59dee4d8, 0x89c6a692, 0x0e817b7f, +0x81816b97, 0x5de37fd1, 0x7f2c7ca9, 0x81784402, 0x29bd477f, 0x81a07ffe, 0xe822df7b, 0x7f29813c, +0x6d81cd81, 0x1a811bd2, 0x6b2c7f26, 0x7881a08f, 0x7fd0e88e, 0x27f87fce, 0x8181a67f, 0x7f7f5558, +0x81668133, 0x8181bfca, 0xc3792d58, 0x2b7f2d7f, 0xa3d09230, 0x9fa56824, 0x7fd5ce7f, 0x251b3cbd, +0x7f818117, 0x7fac9e26, 0x7fde26f6, 0x4b7f837f, 0x1fd52b94, 0x067f6698, 0x70817f7d, 0xdd9658c1, +0xe5a99465, 0xa59cdc55, 0xc1c2c7ad, 0x147fb081, 0x234cc657, 0x7f7fcc0f, 0x7f3d8150, 0x41b68131, +0x7ff58184, 0x90a57f2b, 0xdd9d9b7f, 0xc4518163, 0x428aa37f, 0xe866ef52, 0x7f6c3644, 0xa09d3b50, +0xf2c4adad, 0xc87ecb45, 0x7f70bace, 0xd5697f2d, 0x297349a8, 0x6bc11cc9, 0x8cdd0ed3, 0x7f53646b, +0x8129b9b8, 0xce65a8d3, 0x20208181, 0x817f5272, 0x6ad11bf4, 0xc58151e3, 0x361572c9, 0x81acd996, +0x6e819466, 0x7e7fbd3f, 0x70bb9ffb, 0x818b810a, 0x879a7f15, 0x8c884edb, 0x9028ae45, 0x81d8f525, +0xaaa781b9, 0xaf1a0108, 0xe2817f73, 0x24b67f8b, 0xf08fd77b, 0xd21a265f, 0x7f3930ba, 0x5f5d9287, +0x66218181, 0x7fd219f3, 0x6f44f09e, 0xbcc58157, 0x7f71b94d, 0x817f81c5, 0x877f6064, 0x81647f81, +0xe5ba2e34, 0x889f8722, 0x27819f0e, 0x7f4c0eed, 0x2d7f79b6, 0x81b1813c, 0x647f3d2b, 0xe92b68be, +0x81375804, 0x6798c04b, 0x2b596e7f, 0x78c97f7f, 0x812c367f, 0x7f468112, 0xa352d785, 0x3806f2a4, +0x6886a9a9, 0xa31540ea, 0x5ab2478d, 0x9f5be57f, 0x9ab06481, 0x1586279b, 0x86267f81, 0xb3a50944, +0x788a485c, 0x838d4c30, 0x8ad44b53, 0x923fb39a, 0x95035381, 0x81817ff2, 0x7f44d5eb, 0xf17f8164, +0x39d117c0, 0x37ea75f4, 0x81927387, 0x817f0623, 0x909f7f0a, 0x997f7f19, 0xc1983b18, 0x60937f6b, +0x8829c381, 0x8e6bf066, 0xfb7781ab, 0x43408e7f, 0x7f7f811d, 0x33eb81d0, 0xb86459a9, 0xadba7fe7, +0xc1967f48, 0xfd0284bd, 0x4bef335d, 0x1b7f5f81, 0xa2c2c237, 0x81b0816a, 0x42ee5c7f, 0x7f009323, +0xb7ced44c, 0xd90958e2, 0x3af8aae8, 0x7fb82e90, 0x989a7f7f, 0x14a22c68, 0x60ef7dc6, 0x7fb9ad00, +0xf9e3c69d, 0x5e3bafbe, 0x81178a81, 0x92c4df02, 0x159d7fb6, 0x812b8103, 0x616f9df5, 0xb9a1df61, +0x454b3944, 0x98931fbe, 0x4aae7061, 0x9572a681, 0x8178a8a8, 0x81fe821f, 0x817f7f6f, 0x7f787f2c, +0xe584ff82, 0x7b7f5637, 0x9e815970, 0x517d8181, 0x97faba7f, 0xcfe3ce17, 0x9181f55f, 0x857f5f3b, +0x81b17fbc, 0x7f1d7fbc, 0xae7356c3, 0xe4f77f8b, 0x4695f639, 0x8ddaec55, 0xa5d4a381, 0x7f7fa418, +0x4d818145, 0x3881723b, 0x7f507f7f, 0xa5357fca, 0x81dbdc3b, 0x0b187f28, 0xb0325181, 0x09dd795f, +0x2c39b28c, 0x9ac18118, 0xda52b681, 0xa3819da8, 0x7f884525, 0xba0d7ffe, 0x50ce81ad, 0x60e70e69, +0x110b81ee, 0x8181871e, 0x7fe07f81, 0x8d1c4dde, 0x67649db7, 0x6e4669c2, 0x818144a4, 0x7f6c9cb7, +0x51b57f81, 0x36aec300, 0x7f7f8141, 0xc5eca67f, 0x14ac7892, 0x7f35e9c2, 0xa1fe9a81, 0x6a867f7f, +0x767f817f, 0xc447c4bf, 0x067fedca, 0x8c024b57, 0x7ee5707f, 0x52589b51, 0x0f81b28f, 0xae7f00a7, +0x5244c5ca, 0x9671812e, 0xf0631d15, 0xb4aba1b8, 0x7f9a3488, 0x81b485b8, 0x392f2d7f, 0xed7fad81, +0x6055a4e1, 0x60f300c7, 0x7feb817f, 0xd1e0917f, 0x544ca57f, 0xd23d7e0d, 0x8110c8c2, 0xbc24a615, +0x3d817f33, 0xbdabcdfa, 0x5ac3817f, 0x81817fc0, 0x81817f76, 0x837e43d9, 0x7f817581, 0x5da96bcb, +0x3e1c888f, 0x7f7fa6f5, 0xb769907f, 0x8eb43d8e, 0x3a7f3e81, 0xd5845510, 0xd07f5a54, 0x14816ab2, +0x8126c62d, 0xbeda8f5c, 0xa0007fa8, 0x818d7281, 0xad72917f, 0x49c2f650, 0x87969baf, 0x7f53e975, +0xe78de481, 0xe5083e96, 0xb635812c, 0xafdf4bb9, 0x26ac8101, 0x82b0a109, 0x7f8101b8, 0x607fc7cc, +0xc3b1892f, 0x7d0dd911, 0x396d8195, 0xcb84b67c, 0x3f7d3681, 0x43a07f25, 0xd020297d, 0xb7d65b81, +0x455a99e2, 0x509e4611, 0x817f81e0, 0x81da81a8, 0x8147b5c1, 0xbc81a5df, 0x489c6781, 0x8150a908, +0x4fc0e681, 0x814b8163, 0x11457f8b, 0x9a77db7f, 0x81fa0681, 0x7f867010, 0x852b87c5, 0x3d8253dc, +0x91165c42, 0xc6ac7422, 0x727f0774, 0x7f497f67, 0x1a3fe7c0, 0xb39c81d1, 0x44a47f7f, 0x7f8b066f, +0x7f787ab5, 0x7ba7e870, 0xb932a3a9, 0xc85c1149, 0x32a28d61, 0x7f8d812c, 0x93819da2, 0x647ae98c, +0x777f7e9e, 0xbb447fec, 0x2fabd677, 0x3a7f8c08, 0x51707f28, 0x35f81da5, 0x0b3cfc4e, 0xd4179b8a, +0xef77a6ca, 0x31e02016, 0xdbb81924, 0x7189b4a7, 0x08a671aa, 0x817f2352, 0x81819e81, 0xec7fb7e4, +0x81699ae5, 0x88817f20, 0x5ee7d181, 0x944d8150, 0x7f7f52d9, 0xc1815114, 0x3281a67a, 0x52fa8181, +0xc4e47f7f, 0x7f824cbd, 0x728cf0bf, 0xa6b20e46, 0x9ffe631c, 0x639d82cc, 0x7f638136, 0x7f7f3381, +0x7f8b4794, 0x44b19231, 0x08da5253, 0xaa815d93, 0x7f6a7f69, 0x616331c2, 0xdaa47fcf, 0x7f272255, +0x3ee4098e, 0x819527f9, 0xcfc187a5, 0x7f81e781, 0xf71402bd, 0x9fecc0d1, 0x2067a67f, 0xa1dace7f, +0x42817fa8, 0x8181da23, 0x03818181, 0xc35e3af7, 0x941d7ff3, 0x817f33e5, 0x7f2a78f7, 0xbe426494, +0x0d2d81a4, 0x817f20cf, 0x89b2f4d0, 0x8171ba71, 0x6245be4e, 0x817f8140, 0xed8a7f81, 0xdb097f7f, +0x81724d42, 0xe0de7bfb, 0xde611f7f, 0x7f54d981, 0xb1d07fd7, 0x487f7fef, 0x7f2c8139, 0x7f7f7f10, +0x3e812db6, 0x63447ff6, 0x7c7f277f, 0x9f7f8b7f, 0x76d47f81, 0x197faadb, 0x6a596c91, 0xc681a3a3, +0x7da1d13f, 0x4af8d329, 0xaa819881, 0x7d7f7f4a, 0x3a9d1753, 0xaf818145, 0x5b905350, 0xee313e4c, +0x3fa881c7, 0x437fc046, 0x817fbd9a, 0x30f77f14, 0x3e81df81, 0xa42f4e53, 0x7f67d481, 0x4f4b819e, +0xc5474f22, 0x6eed3376, 0x7fb1ed92, 0x5b3e813d, 0x6c8181a7, 0x720df323, 0x2d0e7017, 0xc7448174, +0x817fb27f, 0xad7f01de, 0x817f10b1, 0xf6231fed, 0xa56d7a81, 0x98817fef, 0x8181e4a0, 0x48aa7fb1, +0x81727f6a, 0xa4059a04, 0xe0cff64c, 0x5ea7679c, 0x69c1009b, 0x7f816b2a, 0x5cc58181, 0x7f954f3d, +0x848b7f17, 0xaee99031, 0x7fbb951f, 0xa2da7f73, 0xc5acfb81, 0x2c1ecf2a, 0x6981fdba, 0x19897fb2, +0x7f4af7a6, 0x85cd8129, 0xe845e0d5, 0xdf658407, 0xd522567f, 0xab242f2e, 0x7fdd70a4, 0x1c9d038c, +0x2f923f85, 0x27f1b2c5, 0x7f812781, 0x7f818181, 0x7f9603cb, 0xc32881af, 0x5ad1f2a1, 0x61b99a32, +0x57cd3bfb, 0x7ffc6fd7, 0x9e81c281, 0xfea25f7f, 0x85a2bfe7, 0xf3ef5107, 0x7f81565e, 0x1d447f81, +0x814e817f, 0x40853b00, 0x6caf3f7f, 0xe13a7fe2, 0x816162c5, 0x81b4e66f, 0x81e18fb8, 0x7da87f66, +0x7fef667f, 0xea2ba1f4, 0x408181d2, 0xba81676f, 0xdd817481, 0x7f9d7fff, 0xd6544752, 0xb1bd61ce, +0x3aa26a56, 0x817e9bf8, 0x8158964a, 0xa0d3253c, 0x308130f6, 0x501d7f23, 0x367f34d2, 0x818581eb, +0x817f4743, 0xdf9e81c8, 0x5c7f13a4, 0x12182965, 0x9cca6c7f, 0x47817f4f, 0x0cbd2918, 0xb05cf048, +0x28810542, 0xe8a869a0, 0x7fb7682b, 0x3f7fb5d1, 0x813dc581, 0x2ea58dcb, 0x4a4e6881, 0x687681b8, +0x665c057f, 0xa78127e0, 0x81b28166, 0x817f686e, 0x7f261637, 0x81ad7131, 0x54817f68, 0x2eae6470, +0xf110f558, 0x8aa44bc2, 0x81f12d55, 0xf37f35cd, 0x7f68679f, 0x3fc481db, 0xe02a5a41, 0x817f8130, +0x919b6c25, 0x14749815, 0x9d7f6c60, 0xc5777fa4, 0x7f8db49b, 0x524a623d, 0x2ef59cc0, 0xd18f4a24, +0x37707f92, 0x4af98126, 0x81b43c76, 0x7f965818, 0x7f81c786, 0x057f7f49, 0x1ab3ef1e, 0x1781eb8a, +0xf0909181, 0x656da3b4, 0x5181cbb9, 0x81e9a237, 0x8ace577f, 0xd19f2c0c, 0x86a7817f, 0xb7628100, +0x817fcfa2, 0x7f3cddf4, 0x744a96d2, 0x19967ffb, 0x7f7f813a, 0x81f2b043, 0x7d46cd81, 0x7f006e7f, +0x363a7c9b, 0x9f812dda, 0x77c77f1c, 0x8172274d, 0xa7ba8181, 0xff7f1ea8, 0x817f4e7f, 0x377f9ad7, +0x81447f7f, 0xf81f3623, 0x58ce7f7f, 0x8a4a208f, 0x4d938115, 0xb0ac7029, 0xf72b23cf, 0x467f9dd3, +0x81819130, 0x62bb7fc4, 0x84b4b190, 0x2781b070, 0xdc81cda4, 0xcb8145b3, 0xda812b38, 0xda307f7f, +0xafbc8158, 0x7fcb7205, 0x47e781ae, 0xc056245f, 0x81997f56, 0x51d9d6f7, 0x7f895c4e, 0xb7814beb, +0x7f658181, 0x757f81bf, 0xdb81816e, 0x582edcb2, 0xb9860711, 0x587f81ca, 0xa45a7f0b, 0x81e68a35, +0x7f7f3b0f, 0x3c233956, 0x7f5f3e44, 0x81ba915e, 0x9ea4dd81, 0xc323e1bc, 0xb1ac0322, 0x5e7f0e2b, +0x6b2fb26b, 0x0e92812c, 0x667fc670, 0x7faccae4, 0xce8181be, 0x815e68c5, 0x273e7f69, 0x10f3e868, +0x47a8b82c, 0xc8817f10, 0x617281b6, 0xe4a44714, 0xe19d4b04, 0xe8818333, 0x81e481fb, 0x58ab1e34, +0xa68c8168, 0x815f4c12, 0x181be656, 0x817f147f, 0xc87f7f8b, 0x7f7fa1f9, 0x7c787f7f, 0x7f7ff8ae, +0xcf9200a0, 0x7f65e492, 0xc37fbb89, 0xa234a37f, 0xa27f7fb4, 0x818a6fb3, 0x04817fb5, 0x2e7f636a, +0x905db630, 0x812bbc0d, 0x8f1beb7f, 0x7f83bb5c, 0x81b96698, 0xbebc8152, 0x6669b181, 0x7f81452e, +0x6861ca81, 0x8806d040, 0x4570d28b, 0x812681fe, 0x817e3b48, 0xcd3ddb3c, 0x21777fa4, 0x7f3d81cd, +0xae0d8104, 0x8134b9fe, 0x7fb87fdc, 0x81ac7f6a, 0xbeec5c5d, 0x60cc4dfe, 0x967f82fc, 0x3a8187a2, +0xe9577fa9, 0x818181a5, 0x8181b4aa, 0x81204f7f, 0x81708146, 0xbe08d127, 0xdec25eca, 0xc3e0db2f, +0xf7817f4f, 0x7f5da797, 0xc9c44f90, 0xbc5b84c5, 0xda44a681, 0xcc9b810a, 0x38c5c3c1, 0x674294d4, +0xe47fe181, 0xa4817f54, 0xcd06b70a, 0x5b3d7f7f, 0x7f137fbc, 0x38769bd6, 0x817f811d, 0x6bc4046f, +0xba9a7f2d, 0xca91ea2a, 0xe881d10b, 0xf9810daf, 0x42e57f81, 0xc87c147f, 0x4f817f7f, 0x66c7bd81, +0xb991549f, 0x816febec, 0xdc50fd71, 0xaab9aa83, 0x7fbd3952, 0x7f50c6cf, 0x1481f8a2, 0x3ff381ab, +0xf6b0470a, 0x81e12ebc, 0x817f5f39, 0x3c7fb836, 0x79813781, 0x7fefa2c9, 0x7e81030d, 0xb849bf3e, +0x81613b40, 0x08817f32, 0xbd5ffe64, 0xd688be81, 0x0166eab3, 0x81077f0a, 0xe5817f56, 0x91b83696, +0xac8181d6, 0x5732a410, 0x40818181, 0x81915bb3, 0xa87f8145, 0x816a06ce, 0x033f7d7f, 0x86438281, +0x3afc477f, 0x9cdf8102, 0x786d4383, 0x81ecfe7f, 0x89a7471d, 0x349c7fd7, 0x7f377f3a, 0x51d55daf, +0x62337f61, 0x547f910e, 0x4bea99d2, 0xbfa9557f, 0x7f2ee96a, 0xcb9ccccd, 0x0fc3a681, 0x687f30fd, +0xb07f4530, 0x83814334, 0x955bf3c6, 0x7f909981, 0x7b4e617f, 0x7659def0, 0x05565d7f, 0x81f271cc, +0x31ca8581, 0x1afd6755, 0x818181ff, 0x7f8181ef, 0x7f7f77ce, 0x1d6692cf, 0x73c57f7f, 0xf083d53f, +0x707f9c09, 0x1ce6813b, 0x17bcb0c6, 0x15b59ba7, 0xde7f817f, 0x7f925801, 0xeba91d81, 0x568190e4, +0x7f817e72, 0x8d7ffa3c, 0x82f281fb, 0x548a7f7f, 0xe9777fa8, 0x407f16cd, 0x7e8167a8, 0xc5812c04, +0xeae8e181, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0xf481c67a, 0x89b07fc3, 0x2e4d7b44, 0x9181818e, 0x83b88158, 0x8181739b, 0x4aa97f6a, 0xb36d7fa6, +0x075bb9b8, 0xcf839c97, 0x20ba9176, 0x117a6a6d, 0xc35b571e, 0x8181fb5f, 0xec3a7fa9, 0x3eec7fcb, +0x53622f7f, 0x8681a07f, 0x0ff39b65, 0xb6c82420, 0xefb381d6, 0xd5390961, 0xf1887f0f, 0xbec2abad, +0x64d57f81, 0x2281057b, 0x1f7fe936, 0x99038181, 0xc1450b81, 0x65cc5386, 0xb886acc1, 0x109dde08, +0x6417d0dd, 0xcb7f7f76, 0x9681b397, 0x53c39e81, 0x437fd46e, 0x751c3481, 0xdd4c813c, 0x38817f65, +0xf0b39281, 0x31357fcc, 0xf37fae31, 0xba8181e7, 0xefc1e23b, 0xa8650162, 0xbb815838, 0xd6eeba30, +0x8de2db81, 0xc01ca018, 0xed2dfb81, 0x7f84731f, 0x40633968, 0x557f2c0f, 0x5f5c6374, 0x41b02181, +0x447f6e09, 0x8d56b3a6, 0xe47f81bd, 0xbe712a04, 0x3e7e78b2, 0x470a8128, 0x298981b3, 0xe65e2866, +0x5bad817f, 0x78884acf, 0xb521287f, 0x81df4b81, 0x33818184, 0xb3d65f30, 0xc2ed7fbc, 0xc4811ccd, +0xec92a11d, 0x5bc5817b, 0x57ad8103, 0x7f6226e4, 0x07c2d87f, 0x3d7fb181, 0xff73974c, 0x08815f61, +0x50616fbc, 0x923ec16c, 0x2181c73a, 0xb8f88181, 0x047f7f7c, 0xe87fa53d, 0x399ca775, 0xef139839, +0x7f217f43, 0x811415c8, 0x37f6b9c6, 0x7f7f817f, 0x11c6ad81, 0x81b3c88d, 0x0ca47c91, 0x024a7604, +0x2dda2f7f, 0x811c5281, 0xc27f814b, 0x7f37a317, 0x42433381, 0xddfbc9da, 0x23737f7f, 0xe3acaecd, +0x1d846dc3, 0x7f1bd4e3, 0x4055d37f, 0x7f1e9a81, 0x31d29025, 0x6d90cf64, 0xdf532fea, 0x1a3dc07f, +0xf997ee19, 0x597f5460, 0x2a0b5b30, 0x5751a381, 0xd27f8489, 0x7f1f9ae4, 0x2bd07f7f, 0x5281923c, +0xbb7fae6b, 0xd344db7f, 0xb0431bc3, 0x33744f44, 0xd2996981, 0xce048861, 0xd57f36c1, 0x1181819f, +0xe7592aef, 0x7fbe817f, 0x177f7f04, 0x2cc78381, 0x27558149, 0x7f64746e, 0x2bb97dc7, 0x10aba652, +0x02f70c7f, 0x7fd8ae81, 0xcd81f07f, 0x9352c65e, 0x344a817f, 0x4f282881, 0x3e2d23fe, 0x69517fb3, +0x7fd1e4a6, 0x3bf47f7f, 0x33909ff6, 0x81aba30c, 0xaa8181dd, 0xf6f36f81, 0xb57f7fa4, 0xe8b6a572, +0xb0ac3a81, 0x81817413, 0x104de181, 0xef977fcf, 0x2054acbe, 0xa9013b81, 0x4a7f34fe, 0x49747f6d, +0x5ad74c74, 0x0e582cc5, 0xa7ea83c9, 0x7f7f750e, 0x38847f81, 0x41817281, 0x3344debd, 0xd229728c, +0x027f839f, 0x5d9fc4ff, 0xdb1e7f51, 0x439681ca, 0x1b81b581, 0x7f777f9f, 0xd6bfb07f, 0xb050814a, +0x1981648e, 0x2cca7f56, 0x30814ab8, 0xc48143a5, 0x5bab3c8b, 0x7f97249c, 0xf7f09eb9, 0x074aa055, +0xc86bbe54, 0x815ed4b2, 0x7fd0b28f, 0xd7e07470, 0xcf815cc6, 0x7e7f7f63, 0x81dfe09b, 0x4098da7f, +0xc1d5a339, 0x7928937f, 0xa481bf6e, 0x81accdd5, 0x3b9c5cb7, 0x817f81a8, 0xc49b8a56, 0xec817f93, +0xb7a9306f, 0x43555d59, 0x5f815d7f, 0xb79e7fc8, 0xc9d17fa5, 0x7f9999b8, 0x73467f81, 0x38627fd7, +0x7a9138ab, 0xbd46b67f, 0x71b5823e, 0x5c81fbb3, 0x3d508f7b, 0x8173c630, 0xd4816172, 0x3f81d92d, +0xcd6b9d7f, 0x72ee607f, 0x851bd2ce, 0xbc4e7fde, 0x1c6c8193, 0x9252b181, 0xcc217f96, 0x296d7f98, +0x5957ae75, 0x817f14bb, 0xbf7b8191, 0x44d87f81, 0x352a84fa, 0x524a882e, 0xae66ac81, 0xbc7fc592, +0x2cb1dc81, 0x42e56887, 0x7981b76a, 0xa5667333, 0xd2ab7e7f, 0xe37f817c, 0x26817f7a, 0x23570c5a, +0x1a685290, 0x814bb476, 0xf6527bb0, 0x7b810805, 0xe63cd272, 0xc1d97fb5, 0x1e8124d3, 0xd34f157f, +0x367f82a6, 0x14819e1e, 0x39226087, 0x818681da, 0x8121417f, 0x81186358, 0x57e88e9c, 0x077f6e37, +0x96087f7f, 0x9a82c77f, 0x0781c4b8, 0x81217f81, 0xc565447f, 0xab41447f, 0xf7915a1f, 0xe58b5a81, +0xcb817d13, 0x81819181, 0x0d4cc8ee, 0x63b4f36e, 0x587f81b3, 0x47947981, 0xcc7f7f88, 0xa3816781, +0x2581a481, 0xc750aa5c, 0xcd440151, 0x4d684b81, 0xd8a6b50f, 0x4f7fa9b5, 0x3f1b93cf, 0xb58f812e, +0xaacd817f, 0x136d7fa9, 0x81b581b9, 0xb75d7fd5, 0xd29a8133, 0x337f8ec0, 0x75ac901a, 0x0a6d7fb2, +0x2d208186, 0x8186b97f, 0x0cb42af4, 0x03ac9e7f, 0x1167a77f, 0x7f19fad8, 0x2d4781b9, 0xf97f6658, +0x9081bc81, 0x6f7f4087, 0x8e527f8d, 0x6873d688, 0x817fbc88, 0x4b81608a, 0x22814ea0, 0xe97f81c5, +0x10bb3de5, 0x818115ed, 0x421b3081, 0xa17f6e52, 0xf46d7f09, 0xd87f8185, 0x68a79c2b, 0x7f7f837f, +0x4d7f4967, 0xc8507f81, 0xc67881f1, 0x0dad027f, 0xad7f8125, 0xa47fb3aa, 0xc74ceb01, 0x373974a7, +0x4e7f8153, 0xa87c423b, 0x2ad75081, 0x4a817f35, 0x357f5517, 0x55816581, 0xcb7fa97f, 0x7b797f7f, +0x504ee981, 0x7adecc6b, 0xce338197, 0xc49ab9bd, 0xe3877f3e, 0x7f817f81, 0xea3624fa, 0xfd889681, +0xfd7f7ff9, 0xa6732f2f, 0xa81b81e0, 0x817f7fae, 0xbcb4b010, 0xc37f7ff4, 0x9a52819e, 0xc730eaec, +0x437f817f, 0x8865815a, 0x53337f64, 0x2d817f8f, 0xd7e2417f, 0x2b6ec637, 0x3d819d41, 0xe4877f52, +0x26a26a63, 0xdc74a867, 0x71433c4d, 0x94b02e35, 0xd6a155f5, 0x3cd37f73, 0x5c89997f, 0x569e7f03, +0xe0adb53a, 0x49d78e81, 0x9c7f22a4, 0x01385b17, 0x9081db0e, 0x3a8c46a9, 0x1e016448, 0xfebdc287, +0xf072dec5, 0x9f816429, 0x4bb0817f, 0x5a0ac21a, 0xf74ca8aa, 0x82811b7f, 0x1c08bf81, 0x1fe36842, +0xbc7f8181, 0x373b7f21, 0x097f8150, 0x54935b9a, 0x3a7f1381, 0xceb77ff1, 0x55ab4e7f, 0xcb1ab47f, +0x9f17477e, 0x7374b97f, 0xb2512a9d, 0x8148d0d2, 0xc057444f, 0xe94f8162, 0x7f816e40, 0x1da3d14d, +0xb458eb23, 0x7f907a47, 0xc8657f81, 0xc4f36ba1, 0xbc8181e6, 0x81efd081, 0x53607fa6, 0x51368181, +0x32b84fdc, 0x8127b27f, 0xd2247f5c, 0x56f77669, 0xecfdfe9d, 0x697fc681, 0x7f7f2781, 0xdb7f371a, +0x1b8181fb, 0xea78f17f, 0x5a187f7c, 0x7f7c705e, 0x2b58816c, 0x757f607f, 0xc87fc97f, 0xec5b66a6, +0xb89574c0, 0x77ccb739, 0xe1fd2aa7, 0x7f45d8cf, 0x20283a95, 0x5f45909a, 0xe581748f, 0xf97fab7f, +0xc8e0357c, 0x73c7da4b, 0x7f0e812e, 0xba01c67c, 0x0324fb81, 0xea2ee62f, 0x253e88ae, 0x077f81cc, +0x817fc781, 0x94f17fff, 0x0ed6f0f4, 0x84677f8f, 0xb598462a, 0x81a87fec, 0x3dbaa47f, 0xbd62fd81, +0x277f81ae, 0x0f43ab6c, 0x07c1617f, 0xc7bc8181, 0xba2460fa, 0x6d0df181, 0x92dfa27f, 0xf7d1033d, +0x53b71860, 0x81b28146, 0x6564b13a, 0x7fcf0981, 0xf9004e81, 0x10229f55, 0xdb7f7f05, 0x281845aa, +0xca6ddfab, 0x6481d77f, 0x4e4120ab, 0x81006b47, 0x518dce81, 0x907f8145, 0x4d5e7f2a, 0x064076ad, +0xdc1c8e81, 0x354b2545, 0xc8426706, 0x4447b281, 0x21daad30, 0x6d21a202, 0x04249781, 0x19ac7063, +0x3a7f00a0, 0x81f41c2f, 0x523c5b36, 0xb6817f43, 0x3f817f86, 0x811234c1, 0xd77fd8a3, 0xc2f2927f, +0x557f7fb1, 0x41d9e881, 0xeec06dde, 0xe4633caf, 0x3c72fd19, 0x9cffc57f, 0x4b7c8659, 0xcbf6e57c, +0xd9bb7f7f, 0x5fdd819f, 0x69ecd881, 0x7f817f90, 0xe68181f7, 0x10811439, 0x2a7a2c37, 0x66438157, +0xbc634044, 0x3fe98163, 0x81ac8166, 0x8143811d, 0x364f63d5, 0x377f4e81, 0x24755bb8, 0x17adfc5a, +0xdedea8e0, 0x2e6e9e14, 0x0e816188, 0xe56cbe81, 0xb9c85cca, 0x817fb120, 0xae7f8181, 0xe37f7f7f, +0x0827d377, 0xdc7f815d, 0x9a2a5625, 0xb2ab4736, 0x817f2274, 0x817fb29c, 0xdcece2cf, 0x5bddd988, +0xcc1e817f, 0x24793971, 0x40ea214f, 0x326f7e64, 0x1a9e3ed0, 0x268c1a7f, 0x314b8548, 0x0f7f8701, +0x347f7c7f, 0xb9ca13bb, 0x3492187f, 0xc67fa67f, 0x7f49814e, 0xaa7f91d1, 0x18d53577, 0xc88b53a1, +0x20af691f, 0xe6853f81, 0xcf5d08bf, 0x81705981, 0xe30d47f0, 0xa8c17fa5, 0xdb817fb4, 0x12818172, +0x83433d81, 0x4a7ff6e4, 0x327f937b, 0xbb6509d3, 0x2c02fc7f, 0xf3b7feb4, 0x415d60ab, 0x136551d0, +0x0e818181, 0x832eb165, 0x3c7f7f82, 0x817770ae, 0xf9b3a525, 0xddcb3d02, 0x008b6181, 0xdbe55445, +0xec8137cb, 0xf5e8f76c, 0x61815ac9, 0xc8810fb0, 0x35547799, 0x0d81a035, 0x1c3eea42, 0x88d77f96, +0x2948677f, 0x0852db6e, 0x42367f74, 0x0d7f867f, 0x3a7fff81, 0xd1d1d271, 0x02a981e5, 0x2bfd2988, +0xd3818199, 0x4e7f4010, 0x09297fd4, 0x7f816b68, 0xe60a05ca, 0xeb81c97f, 0xa2cf5e81, 0xfd7f7f52, +0x07057f53, 0x8672147f, 0x389bd281, 0x3a855c79, 0x41328181, 0x21fe5e81, 0x073f7fc3, 0x3654f170, +0x24bc7f7f, 0x4f816ee1, 0x67427f7f, 0x8140676d, 0xef53e53d, 0xa48bce41, 0xdb5832d8, 0xf64a58dd, +0x3d3e70ab, 0x53ab4a16, 0x137f2023, 0x74569bad, 0x017f8138, 0x81902bed, 0xbdca8f24, 0xa34d8193, +0x81cf7f90, 0xab43b32b, 0x003f7f49, 0xbf3d4570, 0xbce67f72, 0x7c197fc0, 0x5376d47f, 0xd53d7fa8, +0x7e9bb47f, 0xd72aa0f2, 0xde818138, 0x5cc3a981, 0xd57d9c5d, 0xecb2d6a9, 0xf97f81e4, 0xb67f8f33, +0xb92b577f, 0xc483f97f, 0x469d9481, 0xd4868181, 0x7031e886, 0x95b4ca8f, 0xc6d4bd93, 0x117f227f, +0xb67e7fe9, 0x7fb35540, 0x07d01f81, 0xb4ce9b40, 0x1b118681, 0x7f657bfc, 0x2bbe817f, 0x287f176e, +0x6c96bb02, 0xcfff9e59, 0xec7f647f, 0x817f81b6, 0xabd131be, 0x819081e8, 0x4724b87f, 0x5f817f3e, +0x62b99368, 0x81818110, 0xc99ad3b2, 0x25816734, 0x0702743f, 0xe1b4812f, 0xcb787f81, 0xac864257, +0xcb928181, 0x55e27b81, 0x26f281a4, 0x63898181, 0xbbe1e54a, 0x58d2f881, 0x49e3cc81, 0xf7ea287f, +0xe9827f82, 0x81a69a5c, 0xdd81d292, 0x7f935122, 0xd17fdc3f, 0x321a8fa8, 0xfa819a81, 0x1fdb7f79, +0xc6cf9b09, 0xe181547f, 0x0acacab3, 0x0e7f72ca, 0xf562e081, 0x13d3827f, 0x5692de7f, 0x35b04a5d, +0xdd818e3a, 0x6b7f6160, 0x29303878, 0xc7687fbf, 0x38508173, 0xa6bd37be, 0x0d339881, 0x5e198614, +0x67f745e5, 0x81081a7f, 0xa3c77d84, 0xaadb7f81, 0x578d7ff0, 0x75a0b54e, 0x0d01587f, 0xa1cb624b, +0xf94181b7, 0x819d3a81, 0x37258135, 0x5aa87f3b, 0x0e7f7fd0, 0x5b7f817f, 0xda4ce9b1, 0x40917f81, +0xc57f7f3e, 0x4115b391, 0x627b816f, 0xa8575f49, 0x99ec8124, 0x33db907f, 0x4973c07f, 0xc9b71548, +0xde548177, 0xb608bfcc, 0x1d0787bd, 0xd37f793a, 0xfe847f7f, 0x7f3f4111, 0x3dd5de99, 0xbedb817f, +0x4c65c24d, 0x5d2c813e, 0xd901b37f, 0x974c5c92, 0x7f7f7f7f, 0x812b7f81, 0x0057836f, 0xf0778181, +0x2881f3c5, 0x35147581, 0x817fb31a, 0x7e7ae063, 0xfa1e4184, 0x7f1e787f, 0x529c5123, 0x553b9a81, +0x12de4220, 0x1fa92f8f, 0x4d66a52a, 0x554a2f40, 0xad3f637f, 0xbc81257f, 0x04f8976a, 0x2aa1fc7f, +0x3881b3cb, 0xcfc6ee25, 0xa37f5f60, 0xffddf08e, 0xce817f2a, 0x824f4685, 0x17b8a353, 0xa97f8a81, +0xfc81dc7f, 0x7f81a5b0, 0x26fab529, 0x811085c4, 0x031c5a76, 0x817d7f95, 0x1a8181cc, 0xe6aa0f7f, +0x1e454a7f, 0x1c98818c, 0x2b5ef528, 0x7f2b7f5a, 0x4ec7adbe, 0x6981f15d, 0x7f1d4b23, 0x0f337fb9, +0x1db1b85f, 0xdb7fbe3d, 0x1eb3aff5, 0xba2a7f96, 0xd6a0761a, 0xa97f117f, 0x341781f3, 0x9981817f, +0x56dd7f44, 0x431981b3, 0x115d7f7b, 0x7f5a818b, 0x943248aa, 0xb78a45db, 0xc2046981, 0x7a330a6f, +0xedb7817f, 0x81198df6, 0x29e37f9d, 0x74818143, 0xe5147f86, 0x7fac4869, 0x39e2bb4a, 0x422b983d, +0xdb949bec, 0x2c7f5181, 0xe7539781, 0x7f818ca1, 0x2e815f79, 0x2b199727, 0x28ea3181, 0xe65c7fa5, +0xfda2810a, 0x7b227ff5, 0xc259caa9, 0xdf9e7f4e, 0x1c7fbc49, 0xce892118, 0x083b81e0, 0x7081b06f, +0x0c7f507f, 0x637fae31, 0xf2967f40, 0x810181fa, 0x0878b37f, 0x98817f4d, 0xdaa946cb, 0xae813eb4, +0x67bb97b4, 0xe156b2a5, 0xc97f6dcc, 0xa90bd681, 0x14817f81, 0x7f816058, 0x1d587f76, 0xea425425, +0xd62e4781, 0x477fdca5, 0xcb667f7f, 0x8169daa2, 0x631cc881, 0x1381fb76, 0xf27f817f, 0xb297434b, +0x3b1d7bae, 0x89b8d27f, 0x947f8bbe, 0xbc81d770, 0xf94f0724, 0x3cdc817e, 0xf981817f, 0x1b75245a, +0xab20818c, 0xf2811c81, 0xfc78c27f, 0xb77f0881, 0x838132d7, 0x20818381, 0x127f815a, 0x8e9e267f, +0x1fa87fc5, 0x1da2bf47, 0x0f81939b, 0x118ba016, 0x52c07c6d, 0x697fdd00, 0x0e617f92, 0xab812b56, +0xce754988, 0x5e8181db, 0x22814b35, 0xfda981f5, 0xa981e85b, 0x477f93e0, 0x1e73d681, 0xdc748147, +0x01a1479b, 0x6d4e8164, 0xd37f8b7f, 0xbf6b7f6b, 0xc68110bc, 0x81819156, 0x257f497c, 0xb07fa0c3, +0xcac10d73, 0x39b9284d, 0xad817fd5, 0x7f8a5781, 0xe37f7f2d, 0x7f7fe7d7, 0x186d2c39, 0xc4813181, +0x0ac8a47f, 0x569d4cd2, 0x7f3d597f, 0xb87fbe59, 0x35117742, 0xc18c7fa7, 0xb03bd7ce, 0xd77f6829, +0xec3a7281, 0x81285968, 0xf8a69dd1, 0x81e38181, 0x4e1dbe77, 0x783d4e9e, 0x50bc7f85, 0x4181dd76, +0xa61a4bac, 0x627dd07f, 0xa08168a3, 0xfefdde94, 0xcdc19d05, 0x39817f7f, 0x18d78181, 0x631681cd, +0x91d2e1ac, 0xd2bfce81, 0x01f1817f, 0x810681d0, 0xccc18c5a, 0x7f0b617f, 0x563a8345, 0x917f26e1, +0x87147fad, 0x88161a8c, 0x1481928f, 0x5c96b841, 0x1b7ff206, 0xf1bf4759, 0xc6efdc5b, 0xa1078181, +0xf52b428b, 0x817f7f10, 0xdcd1c062, 0x0c863afa, 0xe26a27b1, 0xbd5a967f, 0xe8047fa4, 0x5be68181, +0xef7b2e81, 0x60f47ff1, 0xf9761c7f, 0x8f297a94, 0x2c81096a, 0x7fc40ec0, 0xbb7fb3b8, 0x4494cae8, +0x5847c916, 0x7d728183, 0xe74d7f81, 0x818d426b, 0x0006ba4d, 0xb4bd9ba6, 0xe00a813e, 0x68032681, +0x18877f81, 0xb46281d3, 0x3381113e, 0x6a7f57f7, 0xe2a7194d, 0xcb7f7fca, 0x62c66981, 0x5181e082, +0x6acdde81, 0xbcbbf27f, 0x3e3ea181, 0x81e6c17f, 0xf854097d, 0x307d58c8, 0x2e9e81a7, 0x93ce81ce, +0xcad4d5a2, 0x1272b081, 0xb84ccf3b, 0x007f9d7f, 0xe294efb0, 0x830c7fb7, 0xca07b17f, 0x3b9a88bc, +0x09350bd4, 0xae007f48, 0x3a6a8181, 0x38777f48, 0xce7f7f12, 0x9781570e, 0x3766817f, 0x4b607f7f, +0xe38c2477, 0x9cc64b35, 0x157f0742, 0x6881d981, 0x53c88c81, 0x9f8974ae, 0xa77f8319, 0xfb527fbc, +0xdf865f22, 0x407f8117, 0x5f5a4c24, 0x14207f77, 0xbfe8e081, 0x667f7f58, 0x31f17f37, 0x467f307f, +0xed7f7f48, 0x2bc96b81, 0x04816572, 0x8137a081, 0xea407b02, 0x7f06814a, 0x097c819a, 0x3ef22003, +0x077d1469, 0x32e06181, 0x1e7fa7de, 0xc7508658, 0x2d58ddad, 0x921d7e79, 0x3af381d7, 0x31b971a9, +0xa7b38de5, 0xf274be04, 0x9690d41d, 0x857f4dff, 0xcf7f5cb0, 0x8117bc7f, 0x1b7f4a91, 0x288181d6, +0x3f4d7b12, 0x7f7f979c, 0x4581b06d, 0x7faf7f81, 0xaf1dac81, 0x4d068bb7, 0x367f50bc, 0xdb7fbd41, +0x4f7f7f38, 0x81ac15f8, 0x2c3725a9, 0x98577f4e, 0xb20f8197, 0x6f816981, 0x26c87aa2, 0x1781c2a1, +0xf5449d24, 0x14cc2651, 0xe33d7f7a, 0x4d7f7ff2, 0x3554512e, 0x25c23b96, 0xde7f8d7f, 0x44bc34b3, +0x2a7f3050, 0x1aec3857, 0xada98181, 0x2644d0ee, 0xcbc2333e, 0x7f8e7f87, 0x4aa03181, 0x11b736b7, +0x3abb2381, 0x2f96817f, 0xe1476381, 0x83ca2b85, 0xfc3281ba, 0xa0d62222, 0xefbe354f, 0xa022be81, +0xd77fb435, 0x17538367, 0xee85d87f, 0xa4a66071, 0xc8402561, 0x818a277f, 0x9d4ddd7f, 0x23bf327f, +0x32ec1b81, 0x8c138d68, 0x237f1a15, 0x4a0e2b7f, 0x5b7f4981, 0xba747f29, 0xa1d37848, 0x0c7f03df, +0x923f1d7f, 0x8faa7c7b, 0x35068194, 0x60d2f465, 0x27606681, 0x7fc37f8a, 0x31bd3bd5, 0xcf58445f, +0xdcce241b, 0x88d9816c, 0xb9b2a7d2, 0x2205817f, 0x01ec81af, 0x4581c8a8, 0xdb0f1b6f, 0xca0628cd, +0xf63881c7, 0x7fa3697f, 0x5246567f, 0xb77f7fa1, 0x297f897f, 0x4cb8c84b, 0xce9e977f, 0xbf811e0a, +0x219d746e, 0x7adcfa81, 0xbcce1a61, 0x28987681, 0xfa7faeeb, 0xdb454581, 0xa5a5919d, 0xbf85a57f, +0x34b68581, 0x925c812d, 0x189a4c7f, 0x3ad03756, 0xf8c97f13, 0x97227f81, 0xfd7f070e, 0x7c7fd41e, +0xfe81b327, 0x81a6627f, 0x0016cf64, 0xded56665, 0xcc5fbd10, 0x81112f30, 0x4c6c877f, 0xefb9b53f, +0x33bc2181, 0x5e797cc7, 0x2a10c047, 0x917a7f81, 0x3a81817f, 0x8197ccbd, 0x73df4f32, 0x264f522b, +0xc1e9528f, 0xbb494681, 0xfbde6eab, 0x4f7f280b, 0x987a7f1d, 0x167f1f7f, 0x0b817f81, 0x8124a87f, +0x927f8181, 0x7f81ad86, 0xc779dc81, 0xd0f681fb, 0xcfd8fb81, 0x19cf4981, 0xb87f4bf5, 0x4893529b, +0x0040eb60, 0x7fc78bae, 0x474455dc, 0x0b7f7f7f, 0xa92f7f70, 0x7f3d9c73, 0x137fc699, 0xb4a3c6fa, +0xcb7f5c1d, 0x9c259c99, 0xdf5183d9, 0xa62181a9, 0x9a273761, 0xa5e8cd8b, 0x41b4bf1b, 0xa9818110, +0xb27f7f7f, 0x0e6bb7c4, 0x2e3e7f90, 0xa4c68172, 0xf73f7f7f, 0x81818164, 0x5299a157, 0x0a297f5c, +0x9a46f024, 0x8178d77f, 0x02818181, 0x8181025c, 0xd96828bb, 0x77644a05, 0xb77e4427, 0x407fa650, +0xab7f7f56, 0xaeeff9fe, 0x2f871d3b, 0xb8d7dd43, 0xe8817f7f, 0x7f4d0a27, 0x036a5b7f, 0xd4a3e67f, +0x9122dd23, 0x627fb09d, 0x055c6534, 0x0689a1b3, 0x4f484a7f, 0x7f77b1e5, 0xc47f9d06, 0xd381c37f, +0x3b819a7f, 0x4381827f, 0xed81817f, 0x95c3917f, 0x1b1e9ff3, 0x4c98cf7a, 0x81927f81, 0x703fd644, +0xed5db0d6, 0x23225674, 0xf95a8187, 0xe6ac7f32, 0x488150e5, 0xf6b5e281, 0x29810449, 0xfdada97f, +0xa6a19781, 0xfe36879a, 0xc96ac845, 0x3bf937bf, 0xa2812172, 0x84203765, 0x5f9e3403, 0x017f6a34, +0x2cc6607f, 0x99877f7b, 0x2ad981d2, 0x81933581, 0xdf81338a, 0xd478056f, 0xfa7fa571, 0x1d818352, +0x17c9db81, 0x9c7f7f52, 0xdf9bd4b1, 0x417f98a5, 0xca5517df, 0xc44b22d0, 0x58727f9e, 0x28817fd5, +0x477f2fb3, 0x8b3f7f59, 0x0940765f, 0xa970b281, 0x0a0a7828, 0x81319e7f, 0x2dc34181, 0x4081668d, +0xe7817fcb, 0x3641753a, 0xc17f3d81, 0xb88154b1, 0x73246eb5, 0xb0db8186, 0x09a2ec6a, 0xd82e2c71, +0x336f7f81, 0xd6817f4e, 0x43814a81, 0xc579811f, 0xc0812803, 0x447fa1e0, 0x81b01b7f, 0x752152b3, +0x4f15817f, 0xf97f61c0, 0xc99bd9b4, 0x2281307f, 0x607f8181, 0x7342b07f, 0x06bd9181, 0xb8b8bac5, +0x3b3ca425, 0x8175e89b, 0xa50d7f4a, 0xbb56b781, 0x817fca6c, 0xac0fa7e8, 0x8e7894b7, 0x1d05cf50, +0xf34f5b7e, 0xb65332e5, 0xc76e9375, 0x66d058a1, 0xe5a09f7f, 0xe493eaa5, 0x05fb81b5, 0x33817f14, +0xee418105, 0x81a16f2c, 0x367fa849, 0x7aedbecc, 0x100f81c4, 0x81b2fb3c, 0xcc7fb181, 0x1b3ff096, +0xd3b03a81, 0x4b815f75, 0x497f9c7f, 0x81015625, 0xa0817f81, 0xb0b8167f, 0x7eb7a3fe, 0xe97f6585, +0x555a877f, 0x4b7f8181, 0x12a95e4a, 0xcb6f5afe, 0xd87b5fad, 0xbe655fa0, 0x8b81300b, 0xccb26381, +0x39868173, 0xd6817fa0, 0xcf684e81, 0x7f3bae43, 0xfd8c20a8, 0xcfa0a31f, 0xc0307f81, 0x4e7f5d84, +0xeb941e81, 0x6f77d070, 0xfa0b7f7f, 0x3cb88193, 0xf6818163, 0x3d5f8fbf, 0x40c4c73b, 0x4b7f4e4b, +0xae968181, 0x818181b2, 0x81148181, 0x6b325381, 0xeae5582e, 0x7630307f, 0x1c0f6ba9, 0xa54a0457, +0xd37fc693, 0x2345817f, 0xe44f1e7f, 0x3e4cc9cc, 0x3081babf, 0x817f45c7, 0x5265e57f, 0x427f81ca, +0xe451817f, 0x8190b7de, 0xf0718981, 0x7f9ad381, 0xb9645da5, 0x7f7bed3e, 0x5ce37af9, 0x307f4ecb, +0x0a7f8135, 0x81ffa7a0, 0x534fa1a5, 0x7f42d641, 0x0e501e61, 0x7f7f6edc, 0xc859c081, 0x377f85f7, +0x0df9ab94, 0x817f7f7f, 0xfe7f7f81, 0xc1327ff4, 0xe84c842c, 0x7f1e5bc4, 0x1da77cf1, 0x1e817f53, +0x5f9ab07f, 0x86209e4a, 0x204e81cb, 0x33c9928d, 0x275a2d8d, 0x7f810281, 0xfe7ff7ba, 0xfab77fea, +0x658145e5, 0x73d57fd6, 0x01ec097f, 0x6d83207f, 0x49810cb5, 0xb5d9ce39, 0xf9e0436a, 0x629c81ed, +0x48cf2c8c, 0x7fe37f7d, 0x5b257d81, 0x765f50b6, 0x2cc98403, 0x817fbebf, 0x127114c7, 0x347f287f, +0xd2d72990, 0x81099d9a, 0x8143cac5, 0x8c452fc2, 0xd27f8f6e, 0x61957f25, 0xbc6fc54d, 0xce81f31f, +0xd381787f, 0xac0e5156, 0xf081afc4, 0xb47f6f2f, 0xef811a81, 0x015ee99c, 0xd46de9d9, 0x30655a6d, +0x09b2b87e, 0xaf13753a, 0xc6dd2a7d, 0xb9bd8190, 0x81537f7f, 0x669ff8ed, 0x5b810c81, 0x42e5a566, +0x0c4174cd, 0xbdf9b7bc, 0x1c118e65, 0xe28f9b2a, 0x8c207f3a, 0x1e7f7ff4, 0xd6b57f1c, 0xf0131edf, +0x38c9a381, 0x7f40817f, 0xb735817d, 0xa1a864de, 0xd99cbf57, 0x811a81f5, 0xb381e381, 0xe88e7fb6, +0xfb4a5b89, 0x7fc006a3, 0x1c81653d, 0xbceaa09c, 0x8d7799d7, 0x4b407a81, 0xdc81bca1, 0x3437d76c, +0xd79fe5c9, 0xd9adeea8, 0x163e8164, 0x2cc1b281, 0x3e81e9f1, 0x936504c9, 0x047f6f7f, 0xf981a776, +0x203b2d68, 0x81dde8d1, 0x13e9e9bd, 0xe04108a3, 0x50304281, 0x486d7fb7, 0xbbff81ce, 0x31d43aa3, +0x21813c7f, 0x81c8f989, 0x7f481024, 0x0a979b6a, 0x010081d7, 0x7792e641, 0x0654d37f, 0xa53da946, +0x96c781d2, 0x41524ca6, 0x2ab2d581, 0x1c65bb79, 0xaaa494af, 0x81e09681, 0x3d7facf5, 0x04c3818d, +0x2c1461c7, 0x788581d9, 0x1d3f2081, 0x49dfaec2, 0xfdf07f61, 0x40396d32, 0xc8835fbf, 0x7fb19f54, +0x2fcc8142, 0xe7dcb4e1, 0x73c3a807, 0x7fc68137, 0x061e81ac, 0x81c70c7f, 0x1d817f73, 0xfdbace27, +0xa5fa4081, 0x9b5575ca, 0xee81cc4b, 0xe85a7f53, 0x8daff2e0, 0xd1d1067b, 0x98ec3662, 0xa5818253, +0xf8266d26, 0x38da26ac, 0x3f98912e, 0x4a10f144, 0xd845a345, 0x811bd786, 0x4b137f7f, 0x3427fa81, +0xe981eb9f, 0x8f4ce6e7, 0x277f8181, 0xb0a5b03b, 0xd0633788, 0x4781b17f, 0xb27f3ee7, 0xd37f30b6, +0x21c3a15f, 0x0492a4d0, 0x097a6c7f, 0x047f1c81, 0x994b7f81, 0x818ea981, 0xd46d7fca, 0x2b8a5381, +0x9d424d91, 0x814d7f81, 0x32aa367f, 0x28ff1b73, 0xbea1dbde, 0x76db7f86, 0x248a8d37, 0x294b8136, +0x6e8142d6, 0xec65687f, 0x66297f57, 0xdfb563b3, 0xd352ee79, 0x0689d45c, 0xc58123cb, 0x0b62bf8c, +0x53c6d80b, 0x7fbc9773, 0xe4596e21, 0x2584812f, 0x1793b781, 0xaa815927, 0xb8659035, 0xccbe2281, +0x5eabb4c9, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0xb339946b, 0xd7497238, 0xc36e8181, 0xd0816d3e, 0x7f7f2681, 0x6709bf21, 0x92e3bf81, 0xe54358cc, +0x6bb5a661, 0x3a7f81f9, 0x8140f0fb, 0xc5a67f1c, 0x8d3381a5, 0x817f81c4, 0x72817fd5, 0xa72c76fd, +0x68dc8b11, 0x9b5f4098, 0xc0f43965, 0x7f0ecf02, 0x7cfdd720, 0xe1a0ac15, 0x087fc3c6, 0x81b66e6a, +0x410a893b, 0x81c491f4, 0x81d87033, 0x2056562a, 0x7f526cd8, 0x817fe90f, 0x81f78178, 0x81d93134, +0x3699e0a2, 0x7fa181c9, 0x81f64081, 0x8f816b2e, 0x816ece59, 0xd548fd2c, 0xc4ee3919, 0x6d847f16, +0x81e423c5, 0xfade4a40, 0x9e29eeb9, 0x7fe450f5, 0x5207177f, 0xf17f710b, 0x8cf291ad, 0xd09f9cf0, +0x7f022d20, 0xb7ea81c6, 0xbb89cfa5, 0x1c6fd006, 0x7f8481dd, 0x81020914, 0xb14c81e9, 0x4d9b4f08, +0x7f6064c8, 0x817f7fcc, 0x6b9819e9, 0xb253e7df, 0x724368fa, 0x816fe408, 0x7481a204, 0x0e6fec58, +0x70449617, 0xf5a1da3f, 0x7f446e7f, 0x1005abce, 0x38664981, 0x8240472c, 0x10f52409, 0x818186dc, +0xf7810be2, 0x9c2ad304, 0x81817f62, 0xcc40a43d, 0x5e6b81e7, 0x7ff0e2fb, 0x812a7f7f, 0xe3017f51, +0x817f24dc, 0x85c49997, 0x863c7024, 0x7fc0ce3b, 0x637f8c81, 0x814c9dbe, 0x8ea08156, 0x2cf47f9f, +0x5b55a9b1, 0x107d7546, 0xcd0381b9, 0xd8617f42, 0x7f81b8f1, 0xb506abd3, 0x745f2b2b, 0x7f81b7d5, +0xbf7fdb20, 0x1ca3a610, 0x7fe5e9af, 0x20b37f08, 0x810e6f81, 0x9b7fe6ad, 0x1aa35d81, 0x46032cc7, +0xad7f7f77, 0x8bd48245, 0x3dffaa81, 0x507f91d9, 0x907f9a25, 0x404b81b7, 0xc67f7f97, 0xb1eae864, +0xed2a18c5, 0x2ea85ef3, 0x8153434e, 0x816daaf3, 0x7f03ad67, 0xa0ec264e, 0x640ca1e5, 0xf0818f2b, +0xcbd27ff7, 0xa6a39206, 0x4ca811b9, 0x4c57a068, 0x8647e4c9, 0x7f438ad2, 0x7f945029, 0xc168e72c, +0x7fed4781, 0x81f706cd, 0x62810667, 0x177f7f32, 0xb105d09c, 0x81fa7f16, 0x7f11fa53, 0xc8a6430b, +0xececbb81, 0xe82644a6, 0x3e82f381, 0x28817feb, 0x90c18135, 0xd0f5810f, 0x507f0ba3, 0xfeb38808, +0x8e6c66ea, 0x70a0c4c8, 0x8146912c, 0xb8e01cb1, 0x2b817f51, 0x64818207, 0x9f3a4e81, 0x7f4aece1, +0x84aeafb6, 0xa17f8188, 0x9a7fe759, 0x1cbf5681, 0xad7f6bad, 0x818eebdf, 0x438181e9, 0xa7d07f22, +0x75dbd318, 0x81817f3f, 0x5457bd92, 0x4b204ffc, 0x7f1c637f, 0x8b4c7ffe, 0xa4d97fe8, 0x6194ca37, +0xb06c7f06, 0xad2f1ac0, 0xaef07f7f, 0x15818101, 0x22bf8d7f, 0x597f35cd, 0xb155ab0c, 0xc2838b0f, +0x0f81f4e6, 0x181f7f81, 0x31e1817f, 0x6a2846c0, 0xede8048e, 0x2952c3e3, 0x09187fda, 0xce411cce, +0xbbc197b5, 0x8181464a, 0x161922d0, 0xc37f292f, 0xbd736439, 0xc881b6fb, 0x81ba557f, 0x9da521f0, +0x3e7f52dc, 0x3b3a813c, 0x7f544bc3, 0x46b1b3fc, 0xbeb18e34, 0x418156e1, 0xab22e318, 0x7fbe66e3, +0x2237bfb9, 0x010081be, 0xc159b8df, 0xf449b264, 0x017f9e82, 0x9dbd7ff1, 0x09c7cbd7, 0x81bc537f, +0x817fe1c7, 0x7f177f60, 0x7474bcdf, 0x3c7f6f70, 0x7f7f987f, 0x81812ecc, 0xca783f94, 0x64cc4da9, +0xa0ba1d0a, 0x7aeec524, 0x43bb6b81, 0x1a338981, 0x206be419, 0x65819d5a, 0x7f81d73f, 0x81a67f36, +0x22810442, 0x7fd4cef9, 0x6067ad85, 0xd6d17f30, 0xea7f8166, 0x457f7f84, 0x817f1bc8, 0x6b86be00, +0x7f8bc555, 0x519d5bfa, 0xd07f97e4, 0x505c81d4, 0xf1eb8115, 0x72bdd002, 0x9da57f77, 0x817f12cc, +0x81c03652, 0x441872fb, 0x554bdf67, 0x2e46f0d1, 0x81817f8d, 0x7ff5ce55, 0xd2ded97f, 0x817f67af, +0x0539c336, 0xa37f2f7f, 0x8150637f, 0x8156d068, 0x827fbda5, 0x9f176b2a, 0x81878c0d, 0x81117f07, +0xc0d15559, 0x869c81c4, 0x7fa498da, 0x46d5d102, 0x81f7a581, 0x816eca47, 0x67ad9d95, 0xccb49c2e, +0x297c5440, 0x817f7cda, 0x9f55bd08, 0x2b7fc490, 0x817f817f, 0x377a8158, 0x91b5bf7f, 0xc87f4f21, +0x977fc1d9, 0x7f3088f4, 0xe938b6ca, 0x41707434, 0x81a27fd5, 0xcf818116, 0x817f887f, 0x09cf3bc9, +0x814edf19, 0x46206509, 0x34a36c81, 0x313fc7da, 0x92f34a81, 0x21e942d8, 0x365c16d1, 0x68907ff1, +0xcb8190e3, 0x5e7f7fa3, 0x6fb78c98, 0xa1444250, 0x812bb37f, 0x83a26f0d, 0x739bab07, 0xcb2cea1d, +0x668174ef, 0x288b8965, 0x95849a7f, 0xdc0981b0, 0xd875297f, 0x8153d5b0, 0x9d81e6dd, 0xae81d259, +0xd981813d, 0x7f64ed35, 0x1f459b0f, 0xb8ebe4e5, 0x72812e81, 0x81909b54, 0x02f54fb3, 0xa87281f4, +0x7f6381cc, 0x7f3cab5b, 0xb1958152, 0x814b9bfa, 0x81312b5e, 0xc2977f23, 0x347fbb97, 0x3b057fe6, +0x40208144, 0x7f817fad, 0x54cb9deb, 0x37b34a28, 0x81781b81, 0x7f1b8130, 0x81e194f1, 0x882b7f1f, +0x81dfc90a, 0xe4a97fa6, 0x9e81ab40, 0x971c2535, 0x5a7f7f3d, 0x47708116, 0xab778160, 0x81ac57e3, +0xc2854dec, 0x81ea81a1, 0x8b347f7f, 0xe981d130, 0x93c70917, 0x817f812d, 0x81925016, 0x7f813953, +0x52a5a5fc, 0x7a33d8e1, 0xcbd80281, 0x79df8101, 0x797afc87, 0x7581bbf0, 0x81567f94, 0x472ad4cd, +0x23e49434, 0x4c7ffc03, 0x41b0717f, 0xcc67b7bc, 0x7f7f48bd, 0x8c1c81b9, 0x3a460d72, 0x9eb5fa34, +0x03530615, 0x6c7f7754, 0x818167d3, 0x7fc22100, 0x7f3da881, 0xa24e7f61, 0x321bb411, 0xd981c827, +0x76e8814a, 0x69ca13fb, 0x1390b881, 0xb1d08157, 0x7f4669a2, 0x7f237f13, 0x30268178, 0x0eb87f34, +0xee1aaa30, 0x816596e1, 0x31813181, 0x88888243, 0x817de70e, 0x84811713, 0xd74d7f52, 0x06b17f35, +0xed347f1e, 0x66d66058, 0x7f817fee, 0x035edde8, 0x819b18a6, 0x8e8198ee, 0x817fb5f5, 0x311e442b, +0x957f941d, 0x4e6aa649, 0x7f677f41, 0x8d876acb, 0x7e7bb58f, 0x44c27fce, 0x6f39ec5e, 0x7f7f81d2, +0xd43ba60f, 0x488150c8, 0x8181e9e8, 0xfe46ec02, 0xa2cc7fd8, 0x7f810781, 0x7f30c356, 0x22c97fc5, +0xbcc97f40, 0xf2b2ce1e, 0x7b59537f, 0x46d994ea, 0x4ae2273a, 0x7f36d92d, 0x818ad281, 0xbf7f2b7f, +0x8181abf8, 0x924f4eb0, 0xa62c81b6, 0x812b99aa, 0x447f817f, 0x2c7f5746, 0x0859b34d, 0x83e652db, +0x7fc781ca, 0x819e5cfe, 0x55c5d07f, 0xb4bb3dff, 0xc24b8178, 0x93d28123, 0x15e63a7f, 0x7fc87f40, +0x812fd94c, 0x991b8170, 0xb13593c6, 0x3b9a5119, 0x3b752581, 0x20b89686, 0xcac17f7f, 0x81237f6f, +0x99ec643e, 0x02813100, 0x81124724, 0x3455436b, 0x8155cc81, 0x68136028, 0x81517f5d, 0xc57481f6, +0xe4c88159, 0x81fba227, 0xaa814381, 0xb3818ebc, 0x457f8b51, 0x118181a1, 0x575b7fe7, 0x53571239, +0xa7d35d43, 0x4b577f89, 0x1f817fc2, 0x6c7f9609, 0x9714813b, 0x81d723b6, 0x6c7f377f, 0xa3cf6826, +0x08b27f45, 0xbf7f8184, 0x814de5a0, 0x8181a2fd, 0x167f3c70, 0x8a7fa9e4, 0x65ac4b7f, 0x7fd367c5, +0x81112d7f, 0x8170811d, 0x817f589e, 0x8b7f7f44, 0xba2ab081, 0x527f5bf6, 0x2faad6e3, 0xc335a82e, +0xb862c003, 0x81715776, 0x5081817f, 0xdd7585c0, 0xb7989d7c, 0x367f81b1, 0xa0818129, 0xc6f37f55, +0x5006be1d, 0x817faac3, 0x7e667f81, 0xaf2c7f99, 0x347f897f, 0x2bb1aac8, 0x8112eee8, 0x7f5c62de, +0x007f14d1, 0xa1a2140b, 0x95446419, 0x5e7fc82a, 0x7a7ff07f, 0x81fe7d31, 0x010a8149, 0x7f3e7fb6, +0x397f81ae, 0x7b7f7fba, 0x797f5981, 0x7f7f170a, 0xb9cf087f, 0x0be6cb28, 0xdd97cf4e, 0x92520625, +0x817f3ae4, 0x6c81c381, 0x440cc5ae, 0x2b7fb8b7, 0xe94a2081, 0x81967f4a, 0x7f817f9d, 0x7f81a7a1, +0x815b52cd, 0xd59d57f6, 0x899e827f, 0x7f7f9aa0, 0x52819e7f, 0xda3a2223, 0x8b81073c, 0x818132d0, +0x55638800, 0x9804bcb5, 0xc3810a7f, 0x6ddab6e3, 0x7fcae34b, 0x7f814a2d, 0xea7fed91, 0x3b9bb0d9, +0x87d97f1e, 0x16535739, 0xcc2f7f81, 0x7f98813a, 0x5873bbe6, 0x241781f3, 0xbcc7cb33, 0x7ff69847, +0x8d11c0dd, 0x9ac9d787, 0x66cff992, 0x7f7fc312, 0xaa4fe681, 0x39967f26, 0x201c7f7f, 0x9a7f4849, +0x5768a814, 0x814d731b, 0x81917f81, 0x6eea7d7f, 0xad7f65fe, 0x3d3e81d3, 0xae672981, 0x7f1c81a0, +0x9881053a, 0x81a16307, 0xf64c4063, 0xbc7b97d9, 0xa29dc0a1, 0x8161189c, 0x7478b3db, 0x87b330e1, +0x817f0598, 0xcf422b0b, 0xc29b7ac8, 0xc2817faf, 0x8185581c, 0x81d0b675, 0xc93f7fa9, 0x814e3a02, +0x815d6511, 0x5d81075f, 0xd768c13e, 0x89815437, 0xac810d85, 0x1a7fa5e9, 0x14a6bb81, 0x54531dfd, +0x81dafd24, 0x978133dd, 0x818130be, 0xb67f5bb3, 0x2c7f4179, 0x81b281f4, 0xf3a463fb, 0xda36c4d4, +0xb17f811d, 0x37ca75f1, 0x53257f7f, 0x3ef0208b, 0xdc71817f, 0x978181ef, 0x7f81d17f, 0x1197810f, +0x277f7f47, 0x7ff6b1b0, 0x4562fca6, 0x725d89e3, 0x888197cf, 0x05477fd3, 0x4988077f, 0x81cbb5d9, +0x047fcaa8, 0xdb7f7fc9, 0x81ccfe5f, 0xd91643ff, 0x7f3f7f7f, 0x03815ed4, 0x697c7f81, 0x815b814d, +0x287f0d32, 0x7881a041, 0x247f95f4, 0xc87048e8, 0x3d815e7f, 0x71ba7f0b, 0x50815075, 0x7e53dc30, +0x3f7755fe, 0x1d032571, 0x818cbaf0, 0xbda97f50, 0x7ffc0aad, 0xbf457f42, 0x66b48f81, 0x072c3f1c, +0x433d9cfc, 0x71077f76, 0xac2d5564, 0xdc523811, 0x7f54a538, 0x7f707f00, 0x8c10a6f1, 0xf4a581d3, +0xae875d1d, 0xd37f9747, 0x8124b869, 0x81a17745, 0x61bb8b7f, 0x227f8129, 0xa77fc381, 0x997f8119, +0x817f812b, 0x81a78836, 0xdd557fc6, 0x907f6418, 0xbfa9accd, 0x4f815762, 0x97818181, 0xa0524fa6, +0x815b5721, 0xc77f7f7f, 0x7f725efb, 0x437f614e, 0x81f15c6f, 0x7fb77f28, 0x473f7a1c, 0x7f81d71b, +0x7f5133e9, 0x7cd17f32, 0xfb7572e3, 0x16c35e62, 0xc961a649, 0xdec47c37, 0x6d442844, 0x386b4616, +0x9583c437, 0x81d57fae, 0x8114357f, 0x818f9db4, 0x7ff181d7, 0x773fb601, 0x4032fb29, 0x8613b71f, +0x7f81b3f4, 0x1256f605, 0x297f817f, 0x9a7f2996, 0x5d81b87f, 0xc8907ffd, 0x817f7f35, 0xf6719ac7, +0x7b7f523d, 0x18d8ca2e, 0x2681b97f, 0xc02a810d, 0x34cda181, 0x32e081bc, 0x016f7f08, 0x461bdaec, +0x04330900, 0x633f61ac, 0xa3815204, 0xa534dda9, 0x7f81814f, 0x91a66329, 0x7f51aacd, 0x7ff681a7, +0xabe476e8, 0xb57f9cae, 0x177f27c0, 0x7f2f8b0a, 0x1281a9ba, 0xd991af49, 0xea527f82, 0x9be09d46, +0x28b2a2bc, 0x81c8a1d6, 0x5e1f4e71, 0x817f812e, 0x28b981cc, 0xb071cf04, 0x82cb8139, 0x81608181, +0x819381ae, 0x818f90c8, 0x817f64c6, 0xd169c7c9, 0x8f548381, 0x87f7b03e, 0x6657817f, 0x81be8102, +0x0d078c5f, 0x7ff97fd4, 0x7f814981, 0xb3bb7faf, 0x37c3dd6b, 0x4c6930b7, 0xb56ba479, 0xfc82520f, +0x3494f934, 0xa2016c53, 0xc513eeba, 0x8c283bcf, 0x7fa6047f, 0x54319236, 0x42b2c19c, 0xf965c947, +0x60777fca, 0x02767f47, 0x4921bb04, 0x81818fa9, 0x8eab7f5d, 0x81f4aa14, 0x1abdf97f, 0xbdaa220d, +0xac368139, 0xddb1c212, 0x81d9fd99, 0x90b77f3f, 0x029ec8ca, 0x7f93d9db, 0x8563627f, 0xc5b72d52, +0x2e6eacc0, 0x7f66815b, 0x50db375b, 0x4529813f, 0x7f8d817f, 0x44486f52, 0xbcd64c9a, 0x7fd2bf33, +0x43507f58, 0x7281a4d3, 0x7ca82149, 0x29817fd0, 0x653544b9, 0x88c681a1, 0xd93c2bfe, 0x530835cf, +0x5be26556, 0x7fb37fd1, 0x81e68174, 0xe88c0c42, 0xb63b817f, 0x394a81c9, 0x407f8107, 0x747f1f52, +0x91816dcf, 0x5f818161, 0x8514aebe, 0x7f496bac, 0x5e818181, 0xe8e358ee, 0xae1a7f0b, 0xa4c97f05, +0x77a77f52, 0x818160fd, 0x6bab7f81, 0xcec7e121, 0xf4810196, 0x4a7f68e3, 0xe56d5a7d, 0x813d2c6b, +0x7d0b7ff5, 0x7bbd6760, 0x67812b81, 0x188197d7, 0x8146057f, 0x811eff1d, 0x818ecc4d, 0x4a786245, +0x72593331, 0x2e81556a, 0x66aad481, 0x94818128, 0xde89ba81, 0x4df07227, 0x7f73ff8f, 0x7f44b981, +0x9bf9e707, 0x46811bff, 0x85a96a36, 0xc13b7f15, 0x6670ba65, 0x81fcefee, 0xae8187b9, 0x73c3baa9, +0x819e8123, 0x9cfbc6aa, 0x7f89c73a, 0x7fde9812, 0x97136581, 0xc38181e6, 0x81de9c0c, 0x4994ab0e, +0x30977f06, 0x59dda72a, 0xc9f87fd1, 0x8186b4d7, 0xba86de81, 0x98a24c3b, 0x12157f3f, 0xd7674b1d, +0x81816042, 0x8da8926e, 0x397f81f6, 0x3dd281b1, 0x7f6570ba, 0x7f875412, 0xd74f5be9, 0xab81f207, +0x7feaa863, 0xab81da0d, 0x7ae79023, 0x357f8111, 0x654a468e, 0x521c2306, 0x2f817f37, 0xdb818a41, +0x50b5b30c, 0x71817fdc, 0x7f7f7f04, 0xb13471fa, 0xd1843f7f, 0xb700ece6, 0x109064ea, 0x989b4b03, +0x27556fd8, 0x5ecee700, 0x5f5a2281, 0xa81c81dd, 0xac54b6bb, 0xe60f1ae8, 0x139dc4dc, 0x22818109, +0x2c7fc74f, 0xa69cf331, 0x9d272281, 0x7f9a6da8, 0xb1bb8f45, 0x81815a6e, 0x8315517b, 0x638181f0, +0x273db266, 0x90777f18, 0x047d763a, 0xbdf50a4e, 0x7f5f5c12, 0x817f7f53, 0xcb177385, 0xb2ca7f22, +0x818eaa17, 0x815d4c81, 0xa67fc281, 0x7fd856eb, 0x01becb84, 0xa8e2685d, 0x7fadbe7f, 0x1b7f6a05, +0xef47f495, 0x817f1b1b, 0x8c8d0b81, 0x57b9a5d4, 0xdc298159, 0x5e81a7e2, 0x81467f5b, 0x370f6df0, +0xc49c30a1, 0x7a545710, 0x974b81ff, 0x458c814a, 0x237f17dd, 0x427fb6da, 0x1a7fe5de, 0x817f8121, +0x817f1c96, 0x9c817f90, 0x0e7fd365, 0x818aa15b, 0x7fd302f2, 0xdb7c138e, 0x7fe68181, 0xbf818112, +0x7f2e7fc8, 0x81728101, 0x6482d830, 0x59e24b0b, 0x9b67c284, 0x6e811018, 0x81a42c32, 0xd87a4da2, +0x817f2d41, 0x3e2fbce2, 0xd1d44623, 0x7464b44c, 0xcc255f95, 0x56c93b38, 0xa87f1491, 0x04819c5c, +0x7f457fd8, 0x8d6bb866, 0x819c8181, 0xbc118166, 0x3d4c2a5f, 0x7a6d51fe, 0x79065067, 0x8181d1fb, +0xff6f2b08, 0xf5be7f35, 0x9fa567f5, 0xabb581b4, 0x81819a5e, 0x7f8187c5, 0xaa7f5669, 0xdf526044, +0x7f6ea5f5, 0x7f3a68c7, 0x4feb7fb5, 0x627f2850, 0x39f49181, 0x71ec22ea, 0xc17f4c44, 0x56f4db23, +0x7f08cc95, 0x977fde95, 0x7f4f7f81, 0x81e07f1c, 0x5e7c7fe9, 0x1c3534ce, 0x7f907f2b, 0x7f83418b, +0xeb81daec, 0xa6fc892a, 0x757f7ffb, 0x6281cc2c, 0xb826cecb, 0xc7810a2e, 0x60d1906e, 0x7f8175ed, +0xd085cb36, 0x88c4811a, 0x5e3e707f, 0xaba57fff, 0x099c127a, 0xbac67fb7, 0xa9817d82, 0x4f6c7faf, +0x43d17a0b, 0x92a13dc8, 0x81995d8d, 0xab7f90fd, 0x5f1f8189, 0x767f8101, 0xb7caa2bc, 0x7f8177cd, +0xe0655857, 0x81cfebeb, 0x81af6fc0, 0x57ac6dae, 0x437f8176, 0x513c60fb, 0x9ef3c081, 0x78d0d12e, +0xa40c7f0e, 0x438101c0, 0x81816405, 0x6f527f04, 0x8d0fa47f, 0x6d7fbb7f, 0x39187f74, 0x3c7f5f06, +0x677f7fdc, 0x8114250d, 0xe08373c7, 0x9c7f7dd8, 0x816341a3, 0xdf7f073e, 0x7f818174, 0xca84e8ff, +0xd6ff70ef, 0x7fa1a3c7, 0x7f7f8148, 0x81de7fa2, 0x4d323671, 0x5aaafbff, 0x05ac7f7f, 0xd58102e8, +0xad6c45e4, 0x81677f27, 0x818132ce, 0x7fa55a1c, 0x553029ce, 0x81a47f06, 0x81ca6189, 0x5cd581af, +0x70f34733, 0x676d431f, 0x81387f66, 0xef916df0, 0x72099950, 0x867f81fe, 0xaa7f657f, 0xd7521bef, +0x81d573c6, 0x7f76bcd6, 0xaba4c10b, 0xe394b7d7, 0x82579781, 0x88f5715a, 0xf7815cda, 0x65817727, +0xeab07f22, 0xbb5f7c40, 0xc0a48139, 0x95442a2d, 0x857f7d81, 0xcd7a81ee, 0x7971819e, 0x3a31f0fe, +0x537f6181, 0xc307bed6, 0x8936627f, 0x98b18136, 0xd07c817f, 0x7f7f8126, 0x133a607f, 0x98d40015, +0x368a8123, 0xe181501d, 0xc9817681, 0xa27b2e00, 0x47dd81e1, 0x0e7f81d3, 0xf3287fce, 0x7f3400c9, +0x8d65c745, 0x4c8148c4, 0x21817f7f, 0x81ac0af0, 0x4462c1fa, 0xa99e4e8e, 0x7f153ad8, 0x8168bb47, +0x8370e19d, 0x7f3281ca, 0xf2437283, 0x667f279d, 0x9cb4c17f, 0xde814315, 0x72d3a17f, 0xac113b66, +0x6c13abef, 0x7f816cb5, 0xc8c7c37f, 0xa7d1862e, 0x8b7fb87f, 0x9dd9a384, 0xb69b22f0, 0x157f710a, +0x9007c13f, 0x4518caf0, 0x836c4246, 0x03f981d1, 0x817f7f7f, 0x4581b92f, 0xb4718193, 0xa8815e05, +0x437f81fe, 0x52817fdd, 0xb1477f54, 0xc12e3cba, 0x7f81815f, 0x4e737f32, 0x817fc141, 0xa67f7f58, +0xd19dab37, 0x8b9eb14b, 0x7fe8be30, 0x905d7fba, 0x16ae520b, 0x81015d10, 0x71812450, 0x1de37f1b, +0x811fa6af, 0xdf77b8e9, 0x647ff1bf, 0x17be7fbb, 0x817f6224, 0x8115e99f, 0x7fca7f7f, 0xd48f0fa2, +0xda7fc4ac, 0x10649756, 0x2f89af26, 0xdb5f3ec4, 0x817fde7f, 0xbde94df1, 0x9a707fb7, 0x7f7a656b, +0xfad88c3f, 0x7ccc7f5c, 0x7f8cc477, 0x7fc26452, 0x28c7812f, 0x85b7a1ab, 0x818181ad, 0x404ce719, +0x540fe3e5, 0x705bf0fd, 0x6206a54a, 0x89486861, 0x75e25f81, 0x3c9bc93d, 0x3457c36d, 0x57ab2f25, +0x3cd9bd13, 0x81769cbe, 0x87557fae, 0x025c7109, 0x05457f64, 0x5f427f13, 0x5c7f26be, 0xb9818130, +0x34e63305, 0x8a81ebcd, 0x673326ba, 0x05647ff7, 0x818b8181, 0x8c81782a, 0x4bc0b30f, 0x759749c7, +0x399e99f6, 0x7fdb4313, 0x9c7f5cae, 0x6cc2a611, 0x811dab81, 0x7f7f413b, 0x4fb37081, 0x818c8164, +0xac4c01d8, 0x1f63813e, 0x81db7881, 0x8194404b, 0xcc9bc55f, 0x542900eb, 0x87ff34c5, 0x507f7fa2, +0x7fd97f03, 0x79e428e3, 0x2f7f7c6c, 0x57813f13, 0x7f7f817a, 0x61c40926, 0x68a9457f, 0x63ea9f2f, +0xea81a8ec, 0xfd81301c, 0x81cd8181, 0x812cf7fc, 0xc996b697, 0x81895df8, 0x8d4ae3ca, 0x4c5de834, +0x81643d19, 0x202babce, 0x55a89081, 0x64ba81f9, 0x7481db7f, 0xa0eba6bf, 0x81c12ee3, 0x8181a75d, +0xc8e9939c, 0x0981295a, 0xf22e527f, 0x817f3314, 0xc4f17f5e, 0x8c83651a, 0x7f74a17d, 0xbdd28405, +0xb6bc4795, 0x388c7cdd, 0x957f7f7f, 0x1db7ec29, 0x68f59c45, 0x8142eaa1, 0x81f0b543, 0xa48163fe, +0x81d08134, 0x7d673f5d, 0x8ecdcf94, 0x6e56195a, 0x7f222687, 0x7fd8fb10, 0x81aa977b, 0x22014258, +0xd541be81, 0x817f62fb, 0x707fea79, 0x7f817f60, 0x4b7f3c7f, 0x60a4032c, 0x1c68c2ae, 0x7fc27f19, +0xd5a55cd4, 0x09a651f4, 0xb5447f7f, 0x7f815bfe, 0xe681d381, 0x1781f900, 0x8eb6b0d7, 0x1d90fdf8, +0x528147c4, 0x816b7f46, 0x7f6381c3, 0xd5d2088b, 0xcce4ae52, 0xdf4ec304, 0x967e5e1f, 0x567f1439, +0x317f810a, 0x815f814b, 0xe13ee67f, 0x7f8177a3, 0xac69c42a, 0x7f5837e7, 0x61b481c8, 0x777f812f, +0xcea38116, 0x6b6b7f37, 0x8feac838, 0xf9457f3c, 0xe5107f74, 0xcc957f56, 0xa3ab7f81, 0x81817f6e, +0xc5817f2b, 0x197d0d99, 0x7ad0c84a, 0x81ada865, 0x81b31c81, 0xee7f9b54, 0x815a3406, 0x3f617fce, +0x3d4db8c2, 0x5ac39e72, 0xd390812f, 0x27c6ede7, 0x6f69d981, 0x2981f141, 0x7faa2681, 0x787f6668, +0xc24ec00e, 0x7f008142, 0xfa811aad, 0x2d567fb0, 0x5adb7f65, 0x8a864fc5, 0x6d817f85, 0x307f7f1f, +0x5c817f47, 0xcc7f43b6, 0x9bb08159, 0x496f7f2f, 0x41e383c2, 0x4e7f7f88, 0x71ce63e4, 0xe07f4c7f, +0x857c58ad, 0x8599641d, 0x655c687f, 0x97a64fde, 0xdd42707f, 0x29ced6b6, 0xdf25c153, 0x7f047fbb, +0x7c5f462e, 0x61d08548, 0x7f7fab7f, 0x48b1ecb0, 0x93c63349, 0x039c7fe3, 0x817fce1a, 0x29bb7112, +0x7f3bbaf0, 0x81818d2f, 0x7f5b3d41, 0x1fa18160, 0xc481af81, 0xfc818130, 0xdae2bd73, 0x4d3c81cc, +0xa47f5f0a, 0x5e3ad1a6, 0x1a816eab, 0xb307dbd0, 0xa8798b88, 0x7f81252a, 0x6d7f2899, 0x4d7fa214, +0xc375762c, 0x7fb655ab, 0x5bb98185, 0x1381682c, 0x5281ca7f, 0xc56d7fc8, 0xa81d8170, 0x8daa50f4, +0x8bbf7f03, 0xcf7fb0c2, 0x3b733caf, 0x7f756442, 0x8c81353c, 0xf7e82f0f, 0xa5a620f2, 0x8ae39fea, +0xada55ad4, 0x74f1de4c, 0x37714753, 0x538193e3, 0xb1811481, 0xe5bd81cb, 0x70fc4ea8, 0x888473eb, +0x30b57f1a, 0x90c3f010, 0x81817577, 0xea8c4ae4, 0x6977547f, 0x549481a8, 0xf27f54b8, 0xf87f7fd3, +0xef81843e, 0x5a437fbb, 0xa77f8181, 0x77354c05, 0x9781667f, 0x7fc2ae7f, 0x0b81b868, 0x1267b5e3, +0xa0de743b, 0xb4cce348, 0x8192819d, 0xf0817faf, 0xecc781ab, 0x7f57af2a, 0x499e13c0, 0x7f3d632a, +0xac96fcc8, 0x81727829, 0x7f7ff281, 0x81819bb4, 0xc5bfd358, 0x76553120, 0xb57dbcb5, 0x9b425d12, +0xb0814cc1, 0xea3d393b, 0x1dde5e27, 0x03a6f127, 0x39a9a981, 0x94304751, 0x3af97f00, 0xceaa49d8, +0x3c8124d1, 0x7f24b547, 0x2830e08d, 0x3d7a8f0e, 0x456ffc4c, 0x814e7f4a, 0x357f817f, 0x7f61bc51, +0xd7097f44, 0x213ecf47, 0x815281bd, 0xa9a58187, 0x0c3cbfb2, 0x18f92fa4, 0x307f7f52, 0x727f3361, +0x6cdc8e37, 0x7f9d9e24, 0x64466e7f, 0x2491328c, 0xd58181cb, 0xc97ff4b1, 0x1890627b, 0xa48168b3, +0x7fad44dc, 0x411b7f43, 0x6681647c, 0xc5171bcd, 0x4a6d7fd9, 0x7e369d2b, 0xf8124f61, 0xa71515f1, +0xb381ec09, 0xc4ab71c0, 0x9e2e977f, 0x7f7f0d58, 0x1cdb7b16, 0x81f87f81, 0x517f7f60, 0xf4a3ea33, +0x5c2112c3, 0xc67f7e06, 0x9d7f78c6, 0x811730e4, 0xa29b6170, 0x81817fd1, 0xb6637f87, 0x4f1c540d, +0xe57f7707, 0x277d5c28, 0x7f817f21, 0x5e31e1ab, 0x7f81cd81, 0x815daab2, 0xec5284bc, 0xb45d7b63, +0x7f4d815e, 0x0860494f, 0x81874034, 0x81d0f00a, 0x5b4274ea, 0xdce04fd7, 0x8188680f, 0x827f7fc7, +0x8137e715, 0x9f7f6699, 0x786e75eb, 0x8253202d, 0xb2813ddd, 0x7315a11a, 0x81c18164, 0x7f7f8141, +0x6870da25, 0x117f3efd, 0x7f1c5f1f, 0xcb6d63bf, 0x45561890, 0xc3df8136, 0x23275f32, 0x81817f5d, +0xa50f7fde, 0x7f3ba6ae, 0x37a4bbf4, 0x81a6fd1a, 0x9b9bb6ec, 0x8b2cff0a, 0x2035d781, 0xa1c47fce, +0x095dee2d, 0x7f3a2d51, 0x69727f7f, 0xec8aad61, 0xb0dbcc42, 0x2e706fee, 0x962a9d53, 0x81814554, +0x759026dd, 0xb38851cf, 0x817fa70b, 0x7f462fb9, 0xcc7faf23, 0xa2981632, 0x3e8d8160, 0x07815dd2, +0x82817f22, 0x5a755325, 0x45c77f81, 0x81653cac, 0x93f73dd9, 0x3d81c4cd, 0xf67aa181, 0x467e7f1f, +0xe6c10cfa, 0x9bd4c12f, 0x6c51ca85, 0x4d7f2835, 0xc0bb341b, 0x81812aed, 0x87a181be, 0x4f7f7f6d, +0x81c9e3c4, 0x117f95bf, 0xd457e634, 0xb84b567d, 0x5618f922, 0xda7fc1ed, 0x7f7f81d6, 0xa07da0b1, +0x2e627f0a, 0x814ee836, 0xb65bff90, 0x3e29b6de, 0xb1b78f5d, 0x1724a1f0, 0x81484221, 0xe16d7fb6, +0x578181f2, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 + +hard_output0= +0xb5bc6ac8, 0xf5373664, 0x1310345c, 0xd5bae4e7, 0x1fc9e83e, 0xebfdfded, 0x84bd86ab, 0xb7aabe00, +0x60b44fea, 0xb9067464, 0x30325378, 0xa9195955, 0xf70c6e5c, 0x90922632, 0xc90b1cdb, 0xf2f5fb69, +0x73056b63, 0x1a33bf3f, 0x17755b5c, 0xc58bff6d, 0x2f4390b2, 0x2869d508, 0xe7c7dfe8, 0x38552963, +0x21da5367, 0x07282b9b, 0xa4767105, 0x1e294251, 0xe350a940, 0xb8a6aa27, 0xed12d778, 0xf10d9ece, +0xab93527f, 0xcf2da7e7, 0x68f6d0b1, 0x811f4bca, 0x577b06b2, 0x3234f13e, 0x30bab7df, 0x8dc47655, +0xbb843bed, 0x86da3aba, 0x30950c97, 0xdd096d7a, 0xa871fd6c, 0x8bee4e6b, 0x8fea30d0, 0x6c05b4d2, +0xf3e144d3, 0xd24ebb1f, 0x065635e5, 0x8d3f2cf9, 0x536c6c6a, 0xfbb0a5d0, 0x3d707b42, 0xc44d5982, +0xa5f4ad8f, 0xf32c0970, 0x1bccf1a6, 0x05916020, 0xa64fb176, 0x5ede6a35, 0xaf4966da, 0x9df5e0e7, +0x75042abc, 0x9ef10481, 0x11ddcbc8, 0xa0f5518c, 0xd5c23418, 0x2393d558, 0xfbe7dfeb, 0xed1c64c2, +0x86a36508, 0xde2dfb1e, 0xb8d0fef9, 0x24505232, 0xc894e71c, 0xbcc752a0, 0x40b74e83, 0x90d23c8c, +0x728e4a61, 0x108f0b08, 0x66f522ee, 0xc258d851, 0x35a31c44, 0x11311b5b, 0xfd3d5be9, 0x5ae448ff, +0x4f64994b, 0x5b8247a9, 0x4021114d, 0x2f0b6e82, 0x5eaa9828, 0x50ac71c0, 0xfb86ee52, 0x0dc1ac9b, +0xbbd47645, 0x8f357115, 0x978ceea0, 0xd557db99, 0x99b30388, 0xfc9a8a1c, 0x0f75be1a, 0x50143e22, +0x8840989b, 0x738ec50e, 0xe6b2783d, 0xf67899c8, 0x27ebed69, 0x6c415a16, 0x3a6cc2dc, 0xcd4e4e5d, +0x6cb12b2e, 0xdb88d7c0, 0x79cd1582, 0xbc422413, 0xe72ad2f4, 0x8eaac30f, 0x0bd86747, 0x6d87f69d, +0x15d62038, 0x4b375630, 0x0d51b859, 0x16db2cb2, 0xf210603a, 0x0abeb833, 0x55c694d0, 0xe57ca43b, +0x0ba94428, 0x1398a406, 0xe47d3889, 0x5a20203d, 0x250d7a1a, 0xd930ffec, 0x03992e79, 0xf2759376, +0x024ec121, 0x91fc3a2c, 0xb7e11cc5, 0x4ff7d459, 0xb8700134, 0xd6e61758, 0x4eba0a32, 0xb747e3ec, +0x7073fad7, 0xded80f99, 0x331e2f1b, 0xfa1f1bed, 0x056424a2, 0x1d1d95e0, 0x550b9ec8, 0x51ee2a38, +0x19525153, 0xd70c4cd5, 0x0d6cd7ad, 0xe44d1cf2, 0x30dfecda, 0xdacd7fe8, 0x7321d795, 0xddf48ef4, +0xe271e6a4, 0x9c1feecb, 0x951fcd7b, 0x8acc5a03, 0x3fb83527, 0xe306de74, 0x7b9cd6ee, 0x8e140885, +0xd4c91e8d, 0xe8c39733, 0x0f02f87f, 0xfb06b1b9, 0x0dc9349c, 0xf76bae8e, 0x4f642a07, 0x3d48a9aa, +0xe3ea323a, 0xa1cd5c8a, 0x40aa0e70, 0x132042d3, 0xa9732f6c, 0xd15a00c4, 0x43d3b046, 0x9a51ebd4, +0xc46ee0ed, 0xe2a2148b, 0xf5c478f0, 0x1fb01cf3, 0xf4f321ec, 0xd973811f, 0x11ad11b9, 0x5c67adda + +e = +34560 + +k = +6144 + +rv_index = +0 + +iter_max = +8 + +iter_min = +4 + +expected_iter_count = +8 + +ext_scale = +15 + +num_maps = +0 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_POS_LLR_1_BIT_IN + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_sbd_negllr.data b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_sbd_negllr.data new file mode 100644 index 00000000..b5ef6246 --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_sbd_negllr.data @@ -0,0 +1,1224 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation +op_type = +RTE_BBDEV_OP_TURBO_DEC + +input0 = +0x0a45ef00, 0xb2bdc60f, 0xc810d206, 0xf21cc300, 0xb0c93da1, 0xaae019d7, 0x23ba3c29, 0xa61d1e09, +0x202906d1, 0xd2b61ef3, 0x59e7f241, 0x2ec8daec, 0x68015520, 0x0912f4a3, 0x01282329, 0xa1fd1439, +0x0701231b, 0x0fdcba0a, 0xbe97c6c3, 0x0438e652, 0xd5f644d4, 0xbd0e5f4b, 0xfcb6d0d4, 0x0825b15a, +0x362e2fc8, 0xbbd5abda, 0x0df8c12d, 0xd016b5de, 0x1e439a0c, 0x44c94242, 0x00e2e8c5, 0x0639d241, +0xef6aa40b, 0x90de9def, 0x43e0ea14, 0x31dd07cf, 0xbad70a05, 0xcffc228a, 0x00293bd7, 0x6038080c, +0xdaeb3e20, 0xdd260c4c, 0xaa41fb3c, 0x6eeef0d4, 0x5b02c0f3, 0x14d6ac69, 0xfef0ce44, 0xc5abccf2, +0x2a0f00c4, 0xd6cadb30, 0x38e212cf, 0x5e9abc29, 0xf105fce4, 0xde34e024, 0xb9d42365, 0x281eb22e, +0xb219c055, 0x6504b01e, 0x465602ab, 0x1233a4e7, 0xa9ebf4cf, 0x63c1ed3f, 0xdbe9b71f, 0xf994579c, +0x22f359b5, 0x3606379d, 0xae30fa1e, 0x1d6e290c, 0xc6e0440a, 0x2f129132, 0x1b317bf6, 0xcfdbc119, +0x32022857, 0xaef168dc, 0xa1251d37, 0x65eec43b, 0x9630e70e, 0xe9e1fd98, 0x3e02d8ab, 0xf348af3e, +0xf7b0f027, 0x2ff021db, 0xecae5bd4, 0x1b2f3e34, 0xbb2511c0, 0xd611ddad, 0xf951d2d6, 0xb1dd0191, +0xc325cf35, 0x58bdf4b7, 0x36094dd7, 0x07cd3b44, 0x02c4ae4d, 0x812af6d9, 0x32f63731, 0x31103329, +0xac0033ff, 0x817f583f, 0xc4817f88, 0x8d81790a, 0xa6a2817f, 0x7f81910a, 0x7fc5ed7f, 0xbf2f7f8d, +0x7ff14707, 0x49cc3bda, 0xb04d5a2d, 0xd8d392b0, 0x7f69d3e1, 0x83ae43ad, 0x99b7da75, 0x3a7f7fd3, +0xd46d6fd2, 0x4f7fc524, 0x137f812f, 0x587fcd8a, 0x81c481bf, 0x9caa6184, 0x0bc3a499, 0xe2901a23, +0xfeb59b6f, 0x7f818171, 0xf68ed39d, 0x8113a975, 0x81dd6681, 0x7ffe707f, 0x5836757f, 0x48a98122, +0x7f724081, 0x81588447, 0x59f8b799, 0x7f6dbab0, 0x817f8f09, 0x13c3ecb6, 0x7ff98d64, 0xc4911e81, +0x33922f9b, 0x05c9e85c, 0xa19071b3, 0xec1ca24e, 0x02a48181, 0x813b8196, 0x7fca7d51, 0x7f81814d, +0x00815911, 0x242a7f5f, 0x29d6ee98, 0x8a61887f, 0x4e815e7f, 0x773e811d, 0x8e65d1a7, 0x8d514860, +0x3081b248, 0x7f2393ab, 0x815d2e31, 0x817f4be2, 0x81818519, 0xa07f677f, 0x0defa853, 0x22811c4c, +0xbe81d32d, 0x2d7fdc7f, 0xd8f3195c, 0xf69bc13a, 0x7f7fdae1, 0x2a56717f, 0xd4811c7c, 0xbe218c81, +0x81a556d3, 0x3f43a8c4, 0x6a95b0cf, 0x427f79db, 0x959cbc5c, 0x9f00dd19, 0xa1e8c5d4, 0x59814997, +0x4e35eb7f, 0xf87f348b, 0x76749124, 0xab7f7f1c, 0xf0816c81, 0x8c62b45d, 0x744ec34f, 0x329a7fce, +0x9048b815, 0x0076dcd0, 0x812d2981, 0x98d5cb15, 0xd29652cc, 0x2cd15e33, 0xc2557f91, 0x03517f6a, +0xfc1cc134, 0x0f7f6a00, 0x81939c0e, 0x7f4677a9, 0x7f597f5e, 0x4f972a86, 0x81b2ab81, 0x207f7f69, +0x85b9b234, 0xc1857fac, 0xedaa7c08, 0x7f50057f, 0x7f1dc26d, 0x7fa47f81, 0x7ff57f76, 0xb3bca5cc, +0x6b31d550, 0xce3a6583, 0xdac01f7f, 0x81184838, 0x65239c09, 0xde415bf5, 0x7f7f3881, 0x097fdc7a, +0x81698160, 0xa8da57db, 0x3281530a, 0x628d057f, 0xad7381f2, 0x18a35579, 0x3d909f7f, 0xbf7f7f81, +0x8181eb36, 0x6bd45881, 0x578c7520, 0x417fdc89, 0x307f5bad, 0x667f8181, 0xe1b44f6e, 0x817f7f2b, +0xd55cb62e, 0x8aa0ab9a, 0x81e42e47, 0xbc925270, 0x77aa4a90, 0x56247581, 0x8ff5f670, 0x7d7f9cc7, +0x3a013a2e, 0x10810005, 0x815eab7e, 0x812fe1c6, 0x81f0b47f, 0x7f2b8c03, 0x7f7fc07f, 0x7e497281, +0xd1d70cd5, 0xd981b501, 0x0c63b57f, 0xd8a8ea7f, 0xdeaece7f, 0xa08107f0, 0x9f54e62e, 0xa8cfc431, +0x7f985ee1, 0x811ae887, 0x7f81df4d, 0xc081d1bb, 0x81a15a9a, 0x7f44bf05, 0x037f057f, 0xf22c7f9d, +0x7f7f7f7f, 0x81856e03, 0xbcb08381, 0x46bb372d, 0x7f7bd59c, 0x8157e815, 0xa1910b3a, 0x5c537f7f, +0x3e81717f, 0x4851ac7f, 0x813b7f83, 0x867fb25e, 0x098131ca, 0xb06fc77f, 0x7f60e97f, 0xa25f5c46, +0xe82eae48, 0x40bf987f, 0x817f81e2, 0xf5b2c831, 0x810597de, 0x045c7f4b, 0x81e33f36, 0x81aaf35e, +0x81817fc6, 0xe700587f, 0x93b37fea, 0x9f81fc97, 0x4def727f, 0x6bc07fe7, 0x7f811076, 0x81248175, +0x0a7e5cf6, 0xcd7f7f81, 0x7f49f37e, 0x8181917f, 0x7f70df81, 0xd17f81b5, 0x1f6b4ef7, 0xa21d6b8b, +0x7fb0461a, 0x5c027581, 0x81819bb4, 0x816ee78f, 0xad7f817b, 0xe481a10a, 0xa1cb8d81, 0x32813e3a, +0x947f7f81, 0x8184bff3, 0xa96cd072, 0xeb0c5881, 0x814ab385, 0xe3f67f7f, 0x584a81c9, 0xdbc5c781, +0x78816e49, 0x817f8a81, 0x7f3f7fff, 0x56d89f40, 0x4397e427, 0x7f29b20d, 0x7f597fad, 0x35817f65, +0x0a7f3b19, 0x61c97f81, 0xc6817a65, 0xaa5ceb7f, 0xfc427fef, 0xb8684c75, 0x447fbaa3, 0x4d7fae7f, +0x8196e3bb, 0xe1588132, 0x7ff27f34, 0x8173bff4, 0x7f705f6d, 0x7f7f817f, 0xa353354c, 0x7f58ca67, +0x7a5a7f9c, 0x1e81ee7f, 0x98d681d9, 0x9e60c443, 0x7f7f39a3, 0xc79bc87f, 0x3ae8827d, 0x9f812e81, +0x7f6f8170, 0xa987947f, 0x2481aa81, 0x538fdd7f, 0x237f1442, 0x14e99a84, 0x6e62b068, 0x7b810541, +0x73a57fad, 0x69977881, 0x9be9cd7f, 0x9a48537c, 0xfccc7f47, 0x7f40a457, 0xbc63b4b8, 0x74bb447e, +0x36817f12, 0xd371cd1f, 0x7f1ed17f, 0x36624bbe, 0x726c811b, 0x4abe5c58, 0x81de39c1, 0x7b5ae996, +0xaa818105, 0xa881bebd, 0x5e7f6edb, 0xa8d08165, 0xd495f17f, 0x7fd06098, 0x7fb157fc, 0xc0f661d3, +0xd09f812a, 0x7f8ef77f, 0x69477f00, 0xf45f8481, 0x0f7fa07f, 0x97f17f7f, 0x6ceaac7f, 0x7f7f8163, +0x7fbcbe98, 0x698b697f, 0x79c0da81, 0x8b81e681, 0x81817f36, 0x6b814881, 0x7f918ec6, 0xb67aed0e, +0xc5020b85, 0x81ee8c18, 0x7ff650a5, 0x814f6233, 0x508139a1, 0x814e6aa1, 0xa055487f, 0x3ed75817, +0x4f7f81da, 0x1ddcfa81, 0x7f7f1dbc, 0x825b6454, 0x2ee6819b, 0x73eb6861, 0xf7fe9516, 0xe3816dc5, +0x527f3f7f, 0x550d8fb9, 0xf3f82681, 0x5954b562, 0x7e7f20bc, 0x817f6b4f, 0x8108bc22, 0x17fb8113, +0x7b040f33, 0x586282d5, 0x078c5c7f, 0xe17fc493, 0xdd813545, 0x9ba17f6e, 0x7fccfad5, 0x916f7fa3, +0x21ce7f1f, 0x1a037f81, 0x8e4d0081, 0x7a360731, 0x167fd77f, 0x96762a84, 0xca5660e3, 0x45f0d87a, +0x7fd151ae, 0x9e7f22a6, 0x7f817f8c, 0x1501b9cb, 0x886c4184, 0xea627f6c, 0xc53e7f08, 0x1a63d6eb, +0x481b4a73, 0xca816575, 0x167f7f7f, 0x15814549, 0x6d985e81, 0xa60b1f62, 0xa1818181, 0x33af7f7f, +0x816c7ff7, 0x7f81b67f, 0x08602281, 0x9a81815b, 0xe6787f3f, 0x7aba4f81, 0x5015817f, 0x1d02d55e, +0xbaf3e826, 0xc34ca8cb, 0x503e7c82, 0x647a7f60, 0x7f810873, 0x81149d63, 0x7f818181, 0x7f7ff3d1, +0x11d8dce2, 0x7fe3634c, 0xb63c535b, 0x810e6181, 0x81277f54, 0x817fa2dd, 0x34c34476, 0x848a65f8, +0x96f911b0, 0x9aa76464, 0x8200816e, 0x6d7f56a6, 0xcdb9e57f, 0x874e836a, 0x93a363a6, 0x41c09883, +0x8144c072, 0x52a77fef, 0xe8ded5e2, 0xf6857f81, 0x81b79f4c, 0x7f7f71a2, 0x2b5852bc, 0x1f7b43b0, +0x4c5aa37f, 0x557f3d2a, 0xa17fb581, 0x7349d204, 0x4042ec2b, 0x89cd96bb, 0xd3368281, 0x7f7b8181, +0x0f28f5cd, 0xe8aa4d23, 0x88628b33, 0x8181817f, 0x758197af, 0x37b43b4a, 0xc081c1a5, 0x8e3c4881, +0x7fcb0983, 0x55fe14e4, 0x734c7f20, 0x89267c21, 0x7781a4b7, 0x7f7f4eb3, 0x81ac8fa2, 0xddc20981, +0x7f9b2659, 0x2d7fc65e, 0x818181e8, 0x6a176a89, 0xaa7fb68e, 0x5c5446d2, 0xda7dcc81, 0xc3a5e054, +0x7881477f, 0x7057146f, 0x00764b7f, 0x7fa87f7f, 0xa4ff8181, 0xf07f7f7f, 0xa37a5cd3, 0x28a881b1, +0x13e0cb4f, 0x8f7f9804, 0x3d343181, 0x567fda8a, 0x5828c7d0, 0x434fe70a, 0x7fd5fe78, 0x1f818181, +0x686103cf, 0x62aab5ee, 0x5506cb88, 0x5428ad7f, 0x51908c1a, 0x6d5db4ef, 0x3f7f686a, 0xa6a15a8d, +0x917f7f53, 0xb77fea86, 0x7f816aa2, 0x6ae581d7, 0x54ad634a, 0xd59ca328, 0x6d815bfa, 0x5f470046, +0x23818188, 0x817a9419, 0x7f57dc81, 0xb02a81ad, 0xe986757e, 0xd9810681, 0x81f7be26, 0xbcb5cf7f, +0x63776b56, 0x4358bc47, 0x528aa47b, 0x9e7f7181, 0xd0818100, 0x0d811a7f, 0x7f818155, 0xbe207fc3, +0xb70d393c, 0x812bbd48, 0x7f7f7d0e, 0x81b5ab00, 0x1ecb8681, 0x24c67fcd, 0x2fc7ed01, 0x5503a881, +0x7f3d7fb4, 0x9a81559e, 0x6508d5f4, 0x08b56b7d, 0x96d84e10, 0x7fac9ec1, 0x817ca281, 0xd9b01081, +0xaf81d3a7, 0x8102b34b, 0xa0f98181, 0x7f7f929e, 0x3cf6d023, 0x990ed581, 0x147f817f, 0x81483a81, +0x68375130, 0x487f1451, 0xa0973d81, 0x87d4d4f7, 0x90633e22, 0xbc7f45e0, 0x2c4ea77f, 0x0811896e, +0x8d7f52e9, 0x8120aeb0, 0x3d159d81, 0x93cb0097, 0xce81bb81, 0x74194855, 0x4e7f3f26, 0x817f819f, +0x4531814f, 0x517f7fbb, 0x4981a87f, 0x4c810f4e, 0x3239b67f, 0xa17f194c, 0x81e58e81, 0x7f4890e5, +0xb07ffa3c, 0x937fa181, 0x447fa516, 0xbab5000e, 0x887f7f7f, 0x8181b17f, 0x7f82d4ad, 0x7f828159, +0x81b35c57, 0x23bcb1dd, 0x819446c7, 0x7fbf9ea6, 0x817f7fc1, 0x33817f81, 0x5cfe7fb6, 0x7f81fc61, +0x91477f4c, 0x88994381, 0x65bb447f, 0x198d61ae, 0x817f9f7f, 0x557cbcb8, 0x81fab781, 0x1bab3f7d, +0x7f812298, 0x7f44fadc, 0x1dc1c665, 0x7fde3a8d, 0x94c37f2b, 0xc7d8eb23, 0xdaae8181, 0x129c68bc, +0x7f819111, 0x7f32ce2c, 0xf0d17f02, 0x7f6a817f, 0x5cd564e0, 0x7f7fc47f, 0xd6995c9d, 0x70a7d476, +0x3b7ff299, 0x7f2f237f, 0x4381ac1f, 0x0bd67fb2, 0x81ba594d, 0xa6777fd5, 0x881cc254, 0x8189977f, +0x7f813b3e, 0xc17f7f7f, 0x3baa3dc9, 0x81007f57, 0x7f810295, 0x81f7a1c3, 0x7f65de7f, 0x4ca87f53, +0x407f81b9, 0x7f436fa6, 0x399f9581, 0x7f177f86, 0xbc698194, 0x447fd081, 0x81663d37, 0x01faee9b, +0xc485ff7f, 0xed6ff281, 0x4c6713c4, 0x28b0487f, 0x7f88e8c9, 0x8169016d, 0x7f7f71dd, 0x474bf37f, +0xb756b143, 0x95caeed0, 0xec7f872d, 0x437f23e1, 0xb42b3030, 0x662f8d76, 0xd4818f71, 0x8161fd50, +0x8863663e, 0x78256b06, 0x661b549c, 0x8e817fb4, 0x06b98354, 0x5a905ec1, 0xec960281, 0x2c30bbe3, +0x937f7f63, 0x6ade543f, 0x7f7f9f5e, 0x98dac336, 0x81327f73, 0x677f6d46, 0x637f5c7a, 0x82478181, +0xedbc9014, 0x9a7f431b, 0xb2d25981, 0x7f898181, 0x81815ab1, 0x2d7fa683, 0x7f03b2b0, 0x89bc7f18, +0x81947fb8, 0x1995817f, 0x78e54c17, 0x816cb1c3, 0x1a2681e0, 0xac3a7f8a, 0xdff245bb, 0x422ada81, +0x1ed1e07f, 0x7f6f5276, 0x8481db81, 0x9a817fe4, 0x1e132dab, 0x5c7fdf7f, 0x7d81ac7f, 0x937f7fe0, +0x49568171, 0x3f81907f, 0x81466896, 0x7f667f2b, 0x577f9a7f, 0x7fd73c0b, 0xc946df59, 0xd72f767a, +0xb51d326a, 0xc77f4f7c, 0xbc814bf5, 0xdae47fab, 0x40a64f73, 0x898188ea, 0x1c751c97, 0x7f8ed37f, +0x81b9585f, 0x6eb6758e, 0x5e11b233, 0xf0a47f81, 0x81709065, 0x447fbcc6, 0x58d1819b, 0x9fa36e74, +0x817fa9f3, 0x9f506f66, 0x81cd047f, 0x187f8136, 0x9d810500, 0x64cd8eb0, 0x7f817f7f, 0xf850e2f6, +0x81b61869, 0x606a66c4, 0x81638118, 0x20dca772, 0x6287b381, 0x8e056874, 0x086d99b9, 0x54438172, +0xb281697f, 0x5c470c16, 0xb0340523, 0x7f81b96b, 0x4451b8a1, 0x7f7fe6db, 0x57887f3a, 0x14438129, +0x927758ce, 0xe27fa16a, 0x8c30bd81, 0x7b7fff47, 0x41e58175, 0x35812462, 0xe7ff4358, 0xb4c04079, +0xa87c108b, 0x81aae151, 0x3d797f8e, 0x33007f3c, 0x27c2715a, 0x188c7fba, 0xb481e081, 0xe026d96e, +0x402d8185, 0x7066ffcd, 0xc5914e7f, 0x977f655f, 0x7f68b5d9, 0xcad3235d, 0xc77f8e81, 0x1cb4817f, +0x7f473044, 0x15168159, 0x7f5c81d2, 0x7f99bd6f, 0xb7ab0006, 0xf2487f41, 0xd2de0f82, 0x6a81627f, +0x2e8e87fa, 0x25187fb3, 0x6d31b393, 0xdd3d4b81, 0xa942a11d, 0xd21281d0, 0xe6b0e17f, 0x8a5646ad, +0x907f8124, 0x7f81d2e2, 0xd8c8e17f, 0xeb8b7f81, 0x3ad8bb1a, 0x7f7fbf81, 0xcb30ddf4, 0xa09134d8, +0x8187d2bc, 0x4949168a, 0x53998181, 0xd5e21412, 0x81811cb0, 0x11349a98, 0xdab2afe4, 0xd97f7f7f, +0x7f7f8181, 0x327e7fd4, 0x97818114, 0x827b6517, 0x5f0aabbd, 0x7f905b81, 0xafdde37f, 0xcd40cf7e, +0x7f56817f, 0x95810db2, 0xaf7fd131, 0x8181811a, 0x9e7f8f73, 0xe250d481, 0x7f2abb90, 0x5f7d981f, +0xb3257f91, 0xec657f2f, 0xfa815e3a, 0x6e223481, 0x6c00eaa8, 0x7fb19394, 0x8799c181, 0x810e7fde, +0x7d478181, 0x7fdc76bd, 0x035f8102, 0x627f447f, 0x7ff7ff7f, 0x81265db0, 0x6e81ac2c, 0x8802d277, +0x81c3f781, 0xec81d259, 0x817b7d81, 0x7a5b7f7f, 0x4953ab64, 0x896a52a0, 0xb28111f8, 0xd25d8f0e, +0x7f818174, 0x813f7f3b, 0x7fff7f72, 0x0fa87f88, 0xbcfad248, 0x507dd360, 0x7fae817f, 0x81f27f81, +0x72fa4d1b, 0x5a21d2a1, 0x6381817f, 0x5fb55ab3, 0x8ec3816f, 0x257f4ab5, 0x4c74ef81, 0x19a3f27f, +0x752746c6, 0xe17f8181, 0x7f7c8199, 0xdb9981a1, 0xcb984bd7, 0x815ea881, 0xdc50e0d9, 0xb9366f24, +0x7f455dec, 0x817cb181, 0x424856f3, 0x67ab02a5, 0x0081657f, 0x105e8191, 0xeab817a7, 0x3a7f6d5a, +0x425a1503, 0xeaac317f, 0x7fbe2cd0, 0xd2819a6d, 0x34887f6b, 0xef7f817f, 0xc3577f8f, 0xb124a664, +0x9981167f, 0xb4946031, 0xb8817f7f, 0x81c570ed, 0xc9c88181, 0x3a4cdf81, 0xd9514781, 0x8948c0fe, +0x2d5a8111, 0x68817f7f, 0x6be87f81, 0xa62c0381, 0x857f7fba, 0xa8c4ab7f, 0x279cc57f, 0x98816e0b, +0x817ff70e, 0x21536d81, 0x875ca181, 0x008164f2, 0x96955a53, 0x57a53917, 0x6517faad, 0xdecd7f49, +0x81819c32, 0x7f7fc25d, 0xfd81b181, 0x9f818166, 0x9c7f107f, 0x7f15b67f, 0x81636692, 0x18f276b5, +0xbb9d08e2, 0x6c25797f, 0x417f43fc, 0xd0a37eca, 0xd4702b8f, 0x5c72ba00, 0x7f9dcdbe, 0xd4ad8122, +0x39f1a381, 0x9b419181, 0xac816ace, 0xfb7c81e6, 0xdd7f7fb9, 0x2a27bb61, 0x3781499f, 0x7f6a7f93, +0x4bbf81a5, 0x4a07e527, 0xab4f8143, 0xc2703e81, 0xb1813d81, 0x81747ba3, 0x11cb7f81, 0xd7a385fd, +0xc9d0c981, 0x647fe554, 0x811d3681, 0x18817fdc, 0x81811bef, 0xd381845e, 0x466539da, 0x1081617f, +0xb5b99261, 0x56cd7f8d, 0xaaa79076, 0x7f814a81, 0x7f7fe313, 0x7f7f8170, 0x5df9817f, 0x102fe72a, +0x0c7f79ae, 0x4368d9e1, 0x6b0af013, 0xc13ed920, 0x98ed6a7f, 0xc494d37f, 0x81816a35, 0xc27f7f4f, +0x451a7f3a, 0x4c812e15, 0x032f49b1, 0xbd7f02a1, 0x507f0d67, 0x9c5f007f, 0x8712c21b, 0x6ba7385a, +0x7fb62c58, 0x7f818bcf, 0x7f7fc57f, 0x4141698d, 0x1e817f5c, 0x813fbe97, 0x6f817fa8, 0x51637759, +0x7f4ed17f, 0x81ac2c28, 0xf1c75bfd, 0x1463c260, 0xcb7f7fcf, 0x7483675a, 0x40eb7f7f, 0xad7f81a2, +0x7f817f50, 0x66816683, 0x407f8c81, 0xaf8190b3, 0x81d347f2, 0x5a215a81, 0xca7f8132, 0xd0528ef2, +0x8129c3a0, 0xa4a7c52b, 0x0b816383, 0xbc7faf5d, 0x38d34e66, 0xff65a68b, 0x817999d7, 0x5a620463, +0x885a817f, 0xad947f81, 0x7f1c902c, 0xaa0e9003, 0xd7ccb971, 0x64a68198, 0xb2337f11, 0x81a4d5dd, +0x817f81fd, 0x7e814f81, 0x817f3da2, 0x8381fd08, 0x81a35a81, 0x7f0099e3, 0xfa1f5850, 0x6773efd6, +0x6e7e5268, 0x138e7fc5, 0x81449526, 0x815781c2, 0xca2faff4, 0xf37f7f64, 0xaa080369, 0x7fb07f5c, +0x81397fcd, 0x972dbbbe, 0x813f8e81, 0x812ed669, 0xbf76e310, 0x0be7f981, 0x81929781, 0x577f8644, +0x0cc7e94b, 0x7e1d2995, 0x7ace8119, 0x81814867, 0x617f505c, 0xb9ad7f3e, 0x3ad56e18, 0x81e5b75d, +0xca874729, 0x7f81ad9c, 0x5881813e, 0xbb240a46, 0xcc908863, 0x6fc2815b, 0xfa4bca7f, 0x8f817319, +0xfe819dae, 0x81814281, 0x867f21e9, 0xfdaa0500, 0xd08c9e41, 0x99ea3b94, 0x7fa9394c, 0x81337f7f, +0xb57f23f9, 0xc59a8181, 0x815a7fa4, 0x16c9c781, 0x9f17b97f, 0xb2817f89, 0x7fbd95a5, 0xb27f7f8e, +0xa5e0b5c9, 0xefabff4c, 0x7fb7ea7f, 0x9aec9841, 0xb0e1784b, 0x81b29d1d, 0x7fba1781, 0x81f7086b, +0x8129f27f, 0x818181ea, 0xc1ce8113, 0xfbfbaacf, 0x9d4c14ed, 0x40c1ea26, 0x6051cd3f, 0x6cd681be, +0x4c469beb, 0x17637f22, 0x81882850, 0xbc91d4be, 0x6e318122, 0x95628181, 0x6c7f7f33, 0x6c47537f, +0xa8d27f7f, 0x3de9b857, 0xc73a2c81, 0x4e7f8101, 0x3b7f817f, 0x3a78815c, 0x1a67771c, 0x5a8286a4, +0xf7b98166, 0xd3b37f81, 0x4eb17f2f, 0xc1098181, 0x969a7fc5, 0x0bfbfbb9, 0x316f8194, 0x7f6f8184, +0x484ec57f, 0x364a007f, 0x1f81817f, 0xc5b9ac81, 0xbb81b97f, 0x1f82647b, 0x2c4b4600, 0x89813d0c, +0x75ffc374, 0xad7f5e23, 0x0af481a5, 0x3d0e3eee, 0xa49481cb, 0x39815284, 0x8187f438, 0x1a7c7f7f, +0x817fa8b5, 0x7fca781e, 0xb401d779, 0x424c7fda, 0x3c429944, 0x488d7f7f, 0x78966481, 0xaf687f5d, +0x89052b37, 0xbf7f407f, 0xe07fce79, 0x9fdf45d3, 0x8181de83, 0xe67f7f5a, 0x7fe6a3fa, 0x38705b81, +0x81518232, 0x7d5d9978, 0x270372c9, 0x818568a2, 0xca7f66c8, 0x43a06a3c, 0x831b4281, 0x853a7fb0, +0x8178cf45, 0x7f7f7fbc, 0xbdb69d20, 0x525c5392, 0xc081d97b, 0x18217f16, 0xb6ec7659, 0x0861819b, +0xc48b819e, 0x817f38f2, 0x7f7f787f, 0x7f365c81, 0x64a9f87f, 0xe4e67dac, 0xd1580073, 0x81817f54, +0x597f2ced, 0x575cabd0, 0xce81d781, 0x819c81a4, 0x7f5aceb2, 0x7f57ac33, 0x81e4d87f, 0x1df5d24d, +0x7f4f7f7f, 0x7feb81e8, 0x6a81816a, 0x7f668144, 0x3431b13e, 0x3cc6b82b, 0x815dc38c, 0xc65781b0, +0xae897f81, 0x9751537a, 0x817f0e88, 0xb15e69ea, 0x6f7f7f90, 0x24e681a4, 0xd97fa74f, 0x1881fa61, +0x2e743b95, 0xeb817f7f, 0x7fcd677f, 0x317f7fec, 0x9246f573, 0x2e7fae81, 0x7eb72b7d, 0x4d8144bd, +0x3f7fcb81, 0x7f619f9d, 0x51e7b87f, 0x81688e5c, 0x0d3d55d4, 0xb9b0817f, 0xaec17fd2, 0x017f2ffb, +0xa8af3550, 0x81187f3d, 0xa033427f, 0x38c83442, 0x7f7f8181, 0xe38a35cc, 0x1f00c081, 0x7f297f7f, +0xd17f7f81, 0x6e81c637, 0xc4387945, 0x26839381, 0xd481e281, 0x0b72bf12, 0x057f9e7f, 0x5eef2c81, +0x9216b507, 0xd7906948, 0x637f627f, 0x0a11bfaa, 0xd605bd9e, 0x81ea9c81, 0x72f03c3d, 0x4f819f7f, +0xc563d7b7, 0xad5a3899, 0x7ff15105, 0x7f692e81, 0xbf79c6c9, 0x3c738181, 0x7f4a0597, 0xecb5d588, +0x7f7f2b3c, 0x888144dc, 0xbd7f6e61, 0x6b6e81e5, 0x5d5b1caf, 0x4c743b73, 0x7f727f2f, 0x667f3551, +0x8fc63881, 0x81565a6c, 0x61817f3d, 0x0aa5d083, 0x81215ea2, 0x60461f02, 0x0d7fc1ee, 0xe77f813b, +0x40d9767f, 0x1c817fa8, 0x7f815e7f, 0xa5447f7f, 0x2ac45607, 0x81417f7f, 0x00aceb81, 0x8181cfe4, +0x817f81aa, 0xaa3eda6e, 0x7fbd4adc, 0xf8bb5deb, 0xf0867fe7, 0x3f33a481, 0x819e1bb7, 0x81458b81, +0x9573148c, 0x986d1ee0, 0x45deb557, 0xcf7f688f, 0x23d5b1a9, 0xc3fdfeb0, 0x3fad633c, 0xe4be21bb, +0x81819acf, 0x567f1571, 0x5a2c130b, 0x7f14e77f, 0x37d38908, 0xe55da5d4, 0xf2e16033, 0x23db8181, +0x7f8160bf, 0x177fa17f, 0xa00f4dc8, 0x47818153, 0xf7817b81, 0x529681a0, 0xeca34413, 0x7f49a081, +0x2c9c81c3, 0x8181ae14, 0x8125425f, 0x39b1123a, 0xe4e781b8, 0x4f83e39f, 0xd298507f, 0x49e92f7f, +0x26bac981, 0xf0a27f49, 0x81d281a8, 0x993dc681, 0x48c19a95, 0x98af7a6f, 0x3baa107f, 0x899d3c00, +0x7f814467, 0xc6e821df, 0x05226d8c, 0x9a813aba, 0x8d818117, 0x81688142, 0xd9244ff9, 0xdb0da3d0, +0xbef62430, 0x55868181, 0x8176d83f, 0x33813ef2, 0x941a6534, 0x81d0741b, 0x4e37927f, 0x6b2515e2, +0x8181aa81, 0x9581605b, 0x678113a0, 0x6e7f2681, 0xc1819f81, 0x813ca072, 0x7f81b97f, 0x3d95817f, +0x81233f81, 0x911e90e0, 0x76a6d5d9, 0xc6b5057f, 0x5262b67f, 0x81812681, 0xa68110fa, 0xc74a6049, +0x478eef7f, 0x8ea225d1, 0x77fd31f8, 0x7f568422, 0x81207f81, 0x94628118, 0x81a47fc0, 0x7f637f7f, +0xa8898caf, 0x8125b97c, 0x3de89f9a, 0x7f81bb71, 0x24183337, 0x88c0437f, 0xe97f6bb5, 0xa0dc007e, +0x36dcd381, 0x887cd84a, 0x0a817381, 0x9b818176, 0x46327ffa, 0x487f926d, 0x07508107, 0x7f7ffe06, +0xcefe7f7f, 0x028172d9, 0x7f7381da, 0x81f97f58, 0x81818d0d, 0x06597fcb, 0x817f7ff5, 0x8e999e7f, +0x7fd22e8f, 0x81766ae5, 0x817f7f86, 0xca81bf7f, 0xfd144e10, 0x947ff681, 0xa9817f4f, 0x7f187748, +0x81819081, 0x8170ca74, 0x7a5d9c79, 0xbe483f81, 0x6f7f9fa0, 0x7fb0505b, 0x7fc7f15f, 0x81793081, +0xbc7bb830, 0xce7ff56d, 0x81e02681, 0xa5968113, 0xa47fd381, 0x81a68197, 0xf4ca2a7f, 0xd220acb6, +0x7f837ae6, 0x5c812609, 0x847fd94f, 0x6adf9afc, 0xb17fc822, 0xfd1b82ec, 0x8dfbf181, 0xbe008215, +0x1dfd1081, 0xd1768cc8, 0x76818144, 0x7a917f9b, 0x0c6aada1, 0x16d3654e, 0xea1a8d7a, 0x9508c955, +0xbfe88c4c, 0x593fa4d6, 0x8c45818e, 0x7f7f557f, 0x7f1dcc7f, 0x984c8181, 0x6b817f7f, 0x6c8131d9, +0x4c8185e0, 0x678457da, 0xa6bb81f1, 0x79c0ca67, 0x7fe2c433, 0x257b8381, 0x37323c7f, 0xd768283d, +0xdde15d81, 0x7996d7be, 0x7fd38110, 0x81706b66, 0x7feb685e, 0x817fba69, 0x8182ec99, 0x53c67fac, +0x7f7fc1da, 0xf8afcee6, 0x09816181, 0x8a4f9e6c, 0x5b94c2c6, 0x812b3b97, 0xd0237f7b, 0x81817f64, +0x81768181, 0x7f24b359, 0x1f326247, 0x7f7f5e38, 0x81267f42, 0xff8781be, 0x85817758, 0x7f172281, +0x81599137, 0x22423760, 0x81c8e277, 0x7f46f381, 0xbb668184, 0x7f547f7f, 0x4aa27e87, 0x217f8c35, +0x925eb643, 0x48b89ac8, 0x7f9d8181, 0xffe3a47f, 0x3e810ac3, 0x7f2c7fa0, 0x847f4681, 0x8106818d, +0x81787f10, 0x3e2a5fa5, 0x81a38181, 0xfa7f712b, 0x74dd166f, 0x0d2819a4, 0x557f426d, 0x7f818158, +0x1b817f93, 0x927f0e81, 0x7f1b9a56, 0x78c78898, 0xbb07493f, 0xd4b37f1b, 0xae8aafef, 0x7fc2c3a0, +0xb1bb3d19, 0x1189816f, 0xc2813c7f, 0xc27ff3be, 0x7f3bc183, 0xd1817c7f, 0xc6817fa9, 0x0f9ad87f, +0x7f10c96f, 0x517f7fca, 0xb9958181, 0x9870315a, 0x1c091752, 0x7f0a4746, 0x509ec654, 0x168190cf, +0x8e3d0000, 0xc0817f69, 0x7f8a6f85, 0x7fe8b634, 0x2631815a, 0x67942d85, 0x5a7f8b38, 0xa2a07e1d, +0xfa816e81, 0x3c81bc7f, 0x813b7fed, 0x81aae601, 0x7f81ba4e, 0x81817fa7, 0x32453b81, 0x8e8aa579, +0x7f81d1e2, 0x78a4847f, 0x81819b57, 0x13137779, 0x81c5557f, 0x7fd18195, 0x049928a6, 0xedd7897f, +0xa0816adf, 0xb0815ab9, 0xb2c73281, 0x7f01a4b5, 0x5ebacf94, 0x7fbb8181, 0xa97f7fd1, 0xec9d5661, +0x3da381ec, 0x4445a98f, 0x291ca97f, 0x9d948d9b, 0xf9f0d592, 0x8b1fcd81, 0x81d54fea, 0xdb81600e, +0x69a781c0, 0x817f78f0, 0x818181a4, 0x36810aa0, 0x646f9f7f, 0x70c25f34, 0x9471817f, 0x3e5081db, +0x5cc37f74, 0x7f0a814d, 0x990ba07f, 0xe55babcf, 0x59813a5b, 0x26b96e7f, 0x199c8125, 0x812e0db3, +0xe581b498, 0x8174a97f, 0x440f7ff0, 0x812d487d, 0x227f9881, 0x7fcb77b8, 0x157fc3e9, 0xdbfcff7f, +0x81085d64, 0x44a9bdaf, 0x42998481, 0xb885b098, 0x94817f94, 0x81d3367f, 0x48394f81, 0x817a697f, +0x64671052, 0xbd81153c, 0xd8637f02, 0x558cd081, 0x8485d766, 0x9ea7c1ae, 0x81b2afc6, 0x81657f40, +0x62d4f31b, 0xbd7f7f8b, 0x7b909c60, 0x2f81734e, 0x5060ab22, 0x10b62281, 0x1d83812a, 0xc7aa8866, +0x31444bc6, 0x815d6281, 0x822f8f58, 0xb75aa477, 0xc21f9a27, 0xd3545a36, 0xad30af19, 0x6b813f7f, +0x0000188d, 0x7fa14165, 0x177a549f, 0x8f9e037f, 0x1cd8f7f1, 0x1e7fb9d0, 0x557363c3, 0x7f9c1a26, +0x769fda1c, 0x7f7f8192, 0x157f7e7f, 0xcb61817f, 0x4a9d3d64, 0x8148aa58, 0x637fd2d0, 0x308481d2, +0x95a8324b, 0x7f7f3681, 0xbe407f4b, 0x2b769128, 0x8156657b, 0x7f7f7f7f, 0x45c97f0c, 0x81577f8d, +0xe90f8181, 0x817f689e, 0x2781f97f, 0x7fd1a966, 0xa07f6a14, 0xddbb7fab, 0xa53f57fe, 0x56c77f81, +0x35e0837f, 0xde818164, 0x7f5bb62f, 0x7ffe814c, 0x4a8fe8cb, 0x7f7f5b81, 0x81137fbf, 0x8157a240, +0x81048171, 0xa9d17f18, 0x8458817f, 0x81424a81, 0x9d817fb2, 0x31ef5181, 0xd081507f, 0x567b6e81, +0x91a3276b, 0x7f257f81, 0x98d9a897, 0x8eb301e8, 0x7f8a11a8, 0xac7fb482, 0x1420dd00, 0xa629e6aa, +0x98628159, 0x4a815981, 0xf081eda7, 0xaf5ac540, 0xde387936, 0xf6f2cc49, 0x7fa88152, 0x4587a0b6, +0x0549f981, 0x7f6a817f, 0x7fde8f79, 0x7fd77f81, 0xbd58b276, 0x477f5db5, 0xa3d0817f, 0xb3811743, +0x378d5f7f, 0x889c8175, 0x00d9bffb, 0xa386f71b, 0x9b9b5a7f, 0x05301691, 0x077afd81, 0xc9813d20, +0x655b8118, 0xb1810fc4, 0x2ae18160, 0x7f818141, 0x81c2db39, 0x7f3cd124, 0x52c77f7f, 0xcfdb5141, +0x8113b964, 0x0c0bd10c, 0x8137637f, 0x56bf8149, 0x5fceb17f, 0x01857f81, 0xea81917a, 0xd67a3c7f, +0x5fd93fa4, 0x93720000, 0x31e08193, 0x957faa7f, 0x30e1b019, 0x557f19fc, 0x327f5c1c, 0x81e98181, +0x567f6f7f, 0x817f60bc, 0xe4f481a2, 0xb1f2aa31, 0xba5b7f36, 0x4d2bd790, 0x914d4e38, 0x817f7722, +0x10fb7fcd, 0x2f7f3026, 0x8c7f3c92, 0xbe81392b, 0x1cae7f78, 0xb6cb6581, 0x2f52db43, 0x49cb7f71, +0x7fe6dfe9, 0x232e7e66, 0x1497965f, 0x67316ca2, 0xaf71c384, 0x727fbd7f, 0x5e7f6ab9, 0x7b5181bd, +0x38e38170, 0x48cae97f, 0x4a81669c, 0x3d2d7f7f, 0x69508352, 0xac988181, 0xe093a587, 0xbd907753, +0x817fb87f, 0x894ac8c0, 0x9f7f9ccc, 0x6336a2de, 0x7f7fd741, 0x75b723c5, 0x396e7fc6, 0xc5c0819d, +0x813c7172, 0x816a7fa6, 0x95bdf475, 0x69b21f5f, 0x817f7f06, 0x4c90685e, 0x7fea7f7f, 0x7f958f0b, +0xf6a7937f, 0x5b6caa7f, 0x5fbf7f30, 0x4c6cb506, 0x7f097f95, 0x8181d881, 0x817f34b8, 0x937f7089, +0x830181a8, 0xa6b2817f, 0x4912930e, 0x7f8fd67b, 0xf69bd281, 0x7a5fd981, 0xc4aa7f7f, 0x817f819b, +0x9c05c2f5, 0x8f579881, 0x5aa4818e, 0xc14dc0bd, 0x7fce0981, 0x147fcd41, 0x817fe75b, 0xf8e1a54f, +0x817f8981, 0x1334585f, 0x810281db, 0x146db1bd, 0x25347f7f, 0x9cbf987f, 0xb6736d0c, 0x814ab481, +0x0fd1653e, 0x7fd67170, 0xe564ce22, 0xd05df37f, 0x10871f96, 0x9dc9e03e, 0x53bc03ad, 0xc47fd1c5, +0xaa4dd88d, 0x00004ad1, 0x8a35a85a, 0xcff89253, 0x64d0819b, 0x149ab17f, 0x819f7a33, 0x49fc81c7, +0x1881d433, 0x196171c4, 0x784dbdae, 0x8193148e, 0x75b63674, 0xdf81e4ab, 0xad299a6d, 0x8168b3d3, +0xeea6426e, 0x81c99981, 0xb17f647f, 0x814ec5d2, 0x2e3b16a8, 0x81597f81, 0xa97f8181, 0x2caec714, +0x387906fd, 0x818181be, 0x7f7fbcb3, 0xd55a81e6, 0x8134ae81, 0x7fc3817f, 0x9853ee56, 0x5881999d, +0x9ea98184, 0xfa8181a6, 0xceff5978, 0x688e505f, 0x7fbbb830, 0xc678d06a, 0x3c90e3ae, 0x4b6d7f23, +0x24cd7f58, 0x7f92c181, 0xb1a97fc2, 0xba872981, 0x66ec49a3, 0x7f7f26b5, 0x7f8163b8, 0xae7f7f7f, +0xde7f3781, 0x4147d381, 0xe191cb81, 0xd45b81c3, 0x474c9e91, 0x55b546db, 0x0e81b5a6, 0xb5b976aa, +0x7f7f7f3d, 0x968a81d7, 0x7f1f9333, 0xb37f7f7f, 0x2f7f6418, 0x2532a07e, 0x8b81bf44, 0x89818144, +0xa1572ffd, 0xfebf182a, 0xe54d7f5f, 0x107f8949, 0x00210081, 0xc53345a1, 0xa2818ff6, 0x81c181e2, +0x9b81f1d5, 0xd1061965, 0xb7a47ff0, 0x81817fb0, 0x18bc6181, 0x7ccc5981, 0xbe2b9dae, 0x038f8173, +0x7f3bec4d, 0x81ec81b0, 0x817b9a6a, 0x817cb47f, 0x7fa98fb5, 0xb0355e36, 0x8dad6109, 0x4b138116, +0xa3e19d81, 0x449ab793, 0x984acd21, 0x168aeb94, 0x85baac5d, 0x7fac8173, 0x81ad81d9, 0xa34abb7f, +0x81ca817f, 0x817f6074, 0x7fbc0000, 0xc79b108a, 0xc06990ca, 0x317f12cf, 0x9281e743, 0x90c681fd, +0x17b57f3a, 0xbdd05681, 0xfafcef3d, 0x6f0a0d81, 0x4337197f, 0x7f4881af, 0xb592de71, 0x44c24881, +0x536f9532, 0x21506996, 0x68489d79, 0x4aaf4312, 0x740c9447, 0x657f6673, 0x637f7f0f, 0x81205569, +0x8fb3fe9c, 0x4881995c, 0x8163cfb0, 0x17a4817f, 0x7f59ad84, 0x6d0c30d2, 0xbdc6b981, 0x1bfa8155, +0x7f22a8ca, 0x9c9a817f, 0x3bdba778, 0xf6817eb1, 0x07858141, 0xc037817e, 0x3f2c868c, 0x8181d07f, +0x02b704dd, 0x467f6bc8, 0x4881ab7f, 0xa25c814e, 0xcb4d8e6e, 0x477c7f88, 0xd991aecb, 0x9c81b243, +0x12d6b7e6, 0x03d745a0, 0xdf0bc9d8, 0x52637f85, 0xd557817f, 0x96342cc0, 0x81423981, 0x70657f81, +0xde8181cb, 0x8a81de2b, 0xf55d3e2f, 0x2d717f81, 0xba81289e, 0x99c27f7f, 0x81c5817f, 0x90be817f, +0x918ea3dc, 0x8de39c22, 0xce577f93, 0x3f7f8186, 0x9e817f7f, 0x816c8aeb, 0x3d818e2e, 0x8181be9f, +0x0fb9429c, 0x3c7f5155, 0x81275b24, 0xa47fda70, 0x84cc88c5, 0x70797f81, 0x812eaebb, 0x16a1314f, +0x7f7f817f, 0x38b61a4c, 0x7fb7388b, 0x7fb6e681, 0x5581987f, 0x9281c37f, 0xc45b7f7f, 0x8b35b97f, +0x7f7f7f81, 0xa23c0e3b, 0x7f9b2083, 0x9cc39283, 0x8843699c, 0xc07f81dc, 0xa1f98b7f, 0x81d245b5, +0x8181597f, 0x1aa93681, 0x000001df, 0x81574196, 0x7f3f3df1, 0x6fc8c7c4, 0x5e4d7f8c, 0x81b4dc44, +0xd56f7f8b, 0x8116a381, 0xb03f5f81, 0xf5020639, 0x7f02b25c, 0x55818143, 0xab658147, 0x357f1daa, +0x296ac18e, 0x8186e57f, 0x7464272d, 0x417863e1, 0xeae67831, 0x1a605547, 0x45ffb1d5, 0x7f066c81, +0x81bfb062, 0x4cb8f381, 0x81817f7f, 0x145a4bc0, 0x2e813d7f, 0xc6524d71, 0x81818181, 0xc9d681fb, +0x2a5db57f, 0x12c97fa7, 0x317f7f48, 0xf9b88131, 0x354c1389, 0xd77f8155, 0x811b4dbe, 0x9d3d457f, +0xf9dc2f28, 0xb01c8181, 0x5a81716d, 0x7f7f3d81, 0xd87f867f, 0x817f0581, 0x564f8181, 0x7f678181, +0x5cdd7f91, 0x07ad3f96, 0xb4dd9e34, 0x7f7fd50d, 0xa67f81b6, 0x01353420, 0x6481798a, 0xa5a68181, +0x9c7f816e, 0xafc73c84, 0x427b8d32, 0x1bbb817f, 0xd45c7fa5, 0xb0c2ec48, 0xb459aa7f, 0x70818181, +0x5ec983e7, 0x81294066, 0x81447f6f, 0x9b819f5e, 0x7fb1a47f, 0x81b8a081, 0x5f91c62b, 0x7f63de81, +0xdc813ff2, 0x817f81ce, 0x3d677f0b, 0x4da9dae5, 0xcc8190d9, 0x1d7fbbfa, 0x5ffdd2b7, 0x69624b8f, +0x50963e7f, 0x7f7fa281, 0xcc4b8149, 0x8102da7f, 0x3f7f9df5, 0x435732c5, 0x7f07d881, 0xb20fe07f, +0x4896ac46, 0x44b3ed39, 0x7f7f8127, 0x9e7f81e4, 0x8181b15f, 0x7c818132, 0x7f41980b, 0x299ea18d, +0x7f196381, 0x7fc95f36, 0xbecb7f35, 0xdaa80000, 0x292a7fe2, 0x3292947f, 0x7f98e9c5, 0xb77c984e, +0x74847f81, 0x917f487f, 0x537766db, 0x30b71c7f, 0x81237f81, 0x9d7f9542, 0xfc75737f, 0x72499c3a, +0x6885625b, 0x7f061c6d, 0x818e7f81, 0x5b814381, 0x81f17f4d, 0x4d81b6cd, 0xd2f7d778, 0xe5e9d5db, +0xf7f081c2, 0xb80b0481, 0x975658f2, 0xe8b1197f, 0x81634b1a, 0x7f6bd994, 0xdbd67f7f, 0x757f3406, +0xc4d07f7f, 0x63e7507a, 0x102b7709, 0xe08cf836, 0x1db26230, 0x1a814510, 0x40679bdb, 0xf336a87f, +0x7fc3bf7f, 0x818e69c8, 0xf67aa2a3, 0x75425b7f, 0xa4b654c1, 0x7f7f5ac1, 0x5f10488d, 0x57dc7f30, +0x7d817f81, 0xfc81237c, 0x388a81d6, 0xbce67f42, 0x7f7a7f56, 0xfbb7ff87, 0x467f4681, 0x22dc9b7f, +0xba939029, 0x4a44c1a5, 0xa4be71d3, 0x7ffb3589, 0xfefae9a6, 0xa1963e4f, 0x66b3d6b3, 0x81506f83, +0xee7f32ee, 0x7ffe7f7f, 0xbf505c53, 0xd769ca7f, 0x69c267d2, 0x7f9f8346, 0x3f7f7f7f, 0x48813f51, +0x7fed8115, 0x9e81aef0, 0x7f7f22e3, 0x819fa190, 0x55457f81, 0x7f813b81, 0x251b4a0d, 0x6476818e, +0xc4d8da21, 0x52fd2d4b, 0x81943c7f, 0x273ce481, 0x7d53817f, 0xcd9d9058, 0x754151d2, 0xcb9f365b, +0xac73ecd4, 0x7ffd9a4b, 0x577f2d92, 0x41c60481, 0x81297fa9, 0x9f7f850f, 0x8c513329, 0xe854c39f, +0x34bb4a20, 0xc37f5178, 0x0787cc22, 0x6c86717f, 0x75815a48, 0x2023777f, 0xd37fdd7f, 0x6a819cf7, +0xdc44f5e3, 0x258157bd, 0xe8e7813d, 0xb9118195, 0x9a5a4581, 0x2d8c517f, 0x0c728161, 0xaec769ac, +0x1f554191, 0xfc8be381, 0xca703b7f, 0xab813d5a, 0x3f7faced, 0x707a2181, 0x7f1b8c7f, 0x7fad7f99, +0xb307377f, 0x6c9d5b81, 0x7f3bfac6, 0x8182567f, 0x5a246cdd, 0x55408105, 0x277f7f84, 0x7fa09c52, +0x817fa355, 0x404fd360, 0xecbc4281, 0xc6897f20, 0x7881ae81, 0x407f81e1, 0xfb35587f, 0x9b67fb81, +0x7f8103ad, 0x36708155, 0xab81f381, 0xa3fe6417, 0xa97f7f98, 0x3cf7cd7e, 0xae1b4dc6, 0x8ac2f749, +0x7fb35e89, 0x07e0743b, 0x7f818135, 0x81a15481, 0x93819bbc, 0x81f68114, 0x194ccd81, 0x81527f7f, +0x583ba074, 0x91654d78, 0x4e81398d, 0x0c54567f, 0xe453d054, 0xd37f8175, 0xd57f81ea, 0x345e5b7f, +0x3589262c, 0xa8b886de, 0x811b8197, 0xbbc890ee, 0x8db0b9dc, 0x9fcb817f, 0x39817f7f, 0x55391fe5, +0x7f7f3f92, 0x5a7f55d9, 0x747f3c71, 0x43e31da0, 0x67dccd81, 0x81ddffaa, 0xc32a5881, 0xb9816d7f, +0x424d7f7f, 0xb97fa435, 0x7fdb7f81, 0x48fb8182, 0x8181407f, 0xba7fa88d, 0xa16d467f, 0x81cb8a81, +0x8181a66c, 0x8c747c1b, 0xb4810470, 0x8133dc82, 0xbc77727f, 0xee981437, 0x7f2e8981, 0x26be1939, +0x12da817f, 0x81a1da61, 0x1d2af46f, 0x7f3781f5, 0x29500000, 0x657fc67d, 0x81817f7f, 0x49e406cb, +0x0b787faa, 0x7bc2643b, 0xe4e4f0ec, 0xd28175e5, 0x7f425abc, 0x900c1828, 0x7fa85f7f, 0xe8365361, +0xc5d87fa2, 0x81baffab, 0x81818612, 0xbcb5af1b, 0x7a7e5d7f, 0x81b07f7f, 0xa27aba93, 0x817fd881, +0x818481b0, 0x7f8d1c22, 0xb48c869b, 0x977f9429, 0x9a8c7fc5, 0xb8d9b270, 0x6e340e88, 0x670f7f39, +0x7f4efebd, 0x41b5b57f, 0x7f277f0c, 0x5f177f23, 0x94818592, 0x68362b87, 0x6681ea7b, 0x7f187fd2, +0xa38131ae, 0xc97f698e, 0x25bd8155, 0xe37d88d6, 0x2d018f4d, 0x391e7f7f, 0x7f7f845a, 0xe8f8ee81, +0x4beb9d63, 0x7fd47ff8, 0x5e5781ec, 0x237ffe3a, 0x8e818168, 0xa1e781e7, 0x85de7f81, 0xbaaad281, +0xa7486481, 0x555e737f, 0xa2b28f7f, 0x70635a47, 0x7f837fd8, 0x64ea8641, 0xc20c7f81, 0x0b9e738e, +0x698e8145, 0x783a5a00, 0x7f376e81, 0x7f8cbd20, 0x7f548181, 0x81149934, 0x3dad456a, 0xb4561fed, +0x815d8127, 0xaea4bb24, 0x2187755a, 0x847ff0b7, 0x7fdb9039, 0x81887695, 0x8781e111, 0xe0de037f, +0xc879f7ca, 0x7fc18381, 0x7f7ff7bf, 0x958b7f81, 0xa65fe7ad, 0x3481817f, 0x9f897b7f, 0xa2bb7f7f, +0x31018170, 0x70e08c81, 0x4c1da62b, 0x16f27ff7, 0xdf0781ed, 0xbf4081c0, 0x3a233c53, 0xf8aed938, +0xef24617b, 0x812681ab, 0xa66e4db4, 0xef9b65b3, 0x00007f44, 0x7f7f997f, 0x7f341fc7, 0x0f9b2be4, +0x7ff67f81, 0x81817e2a, 0x4b4d7f81, 0xc0706505, 0x81fc60e1, 0x30d87f9c, 0x7fff9c0d, 0xa789757f, +0x3881d769, 0x6367bf81, 0x9bae7f8d, 0x8e81bbb6, 0x61e88127, 0xc96c7fbf, 0xdf813181, 0x7f7f7de7, +0x3e817f81, 0xb97f817f, 0x8b927f81, 0x5e74742d, 0x7c49817f, 0xbc1172b1, 0x81818181, 0x6dd2d4bb, +0x98f3e058, 0x7f81efde, 0xaeee76df, 0xd57fca01, 0xf5817f81, 0x26747f81, 0xdc3f8181, 0xc335c749, +0x7f2f7f7f, 0x7f02e67f, 0xfb706975, 0x8f4efde7, 0x819b414c, 0xb14cde70, 0x22e6812e, 0x3843892d, +0x6f81ce60, 0x50c12725, 0xb4e279d5, 0x7fb1ac7f, 0xbc7f8183, 0xc7767881, 0xb6e71854, 0xb37f7f77, +0x3d7f7f7f, 0x817f6824, 0x4981ae81, 0x7f811a7f, 0x58c37f74, 0xbef5a27f, 0x92a62541, 0x8643aa3c, +0x8f838181, 0x397fe4f4, 0x8a814677, 0x93e3affa, 0xa6fa21e9, 0x7a3e7f7f, 0x332a7872, 0xf28c8176, +0x227f573d, 0xbbbb6348, 0x7fefb2de, 0x43817f69, 0xc4c37b31, 0x747fa118, 0xacb3819c, 0x7f689f89, +0x74e0604b, 0x81b57f88, 0xe925a0cf, 0x81be2181, 0x126d34f1, 0x7648d74e, 0xd760b29b, 0x04d0fda1, +0x3b858181, 0x1be20981, 0x8127ac7f, 0x8a6b8181, 0x7f616ca2, 0x37e6e881, 0x7f9b82c0, 0x7f6e7f93, +0x8dc72420, 0x752f3d39, 0x5e7fd2e5, 0x7f25c372, 0xe97f2677, 0x307f0000, 0x81383b86, 0x713de0fd, +0xe47c817f, 0xf0214e8f, 0x8108349e, 0xe0c92881, 0x7fafb0e2, 0xe939b48c, 0x4855d8ae, 0xb581e469, +0x967feb6a, 0xba543d20, 0xc47f0c62, 0x2a28e6b2, 0xd27fb09a, 0xbadf7f7a, 0xbf98d54c, 0x5f54cfa3, +0x4881248d, 0xc97f7f53, 0x17816966, 0x3450877f, 0x4ff681c8, 0xfdb8786d, 0x0209730d, 0x7fbbba84, +0xc5994cff, 0x4d31cc44, 0x7fb99400, 0x239d757f, 0xa2bd517f, 0xd5558194, 0x93818191, 0x819b8190, +0x447f9289, 0x777f3e81, 0xc2c04a7b, 0x27c38eaa, 0x437a383d, 0x7f812432, 0xbd77707f, 0x7f81ea6d, +0x40256698, 0x81a95b58, 0x2fb47f81, 0x74b64d86, 0x70f07f23, 0xd7d6bbd5, 0x817f18a6, 0xe862327f, +0x3ff56cff, 0x7f7f8197, 0xcb75c381, 0x58954f57, 0x43818176, 0xa903811d, 0x7f6abbfa, 0x8cd7a77a, +0x55814473, 0x7f819e1a, 0x557f9e89, 0xa9c955e0, 0x648191b0, 0x81a97f51, 0x1dbc1181, 0x685a6b36, +0x7f2e5ef2, 0x59819afb, 0x3f30fd68, 0xe986702b, 0x810a2581, 0xfedf7781, 0x947ffb3a, 0xa9777f29, +0x9c547f3d, 0xe36d7f07, 0x81909281, 0x2b87817f, 0x07aa817f, 0xd9137f30, 0xb791d3ff, 0xb84869c5, +0x4db4e1ce, 0x81beed66, 0x1637adce, 0x10a1897d, 0xfd817fbb, 0x5771c343, 0x3b16dc58, 0x7fbf813f, +0x7f9ba269, 0x353a7e21, 0x14f07fa6, 0x7f81815b, 0x484bb301, 0x0000c27c, 0x59141b93, 0x7f637f3e, +0x3012937f, 0xf2a2b350, 0x1d7f7f7f, 0x81b6d4ed, 0x4fc3ba54, 0x3f7f107f, 0x02af3855, 0x598c814a, +0x3eb09fd7, 0x636832b6, 0x7f9e817f, 0x7f939c7f, 0x7fa99581, 0x7f817fb1, 0x387f3475, 0x9893f771, +0x52813581, 0xc5c75881, 0x7f787887, 0xb97981d0, 0x27436262, 0xfae6f21d, 0x81cacf5d, 0x41a5de81, +0x81817d81, 0x3b2f7f9e, 0x5dc0ade8, 0x3d0e8154, 0x81bd3a0a, 0x6e818153, 0x7f818123, 0x817fc575, +0x811b669b, 0x7f037929, 0x26acac81, 0x7fb3efb6, 0x82817fc3, 0x0c81f981, 0x607a677f, 0xc8258116, +0x0ae77a50, 0x816fba35, 0x7f256549, 0x04c57f89, 0x4381075f, 0x81cd3b56, 0x8ccdad7f, 0x7fa45cd5, +0x817fb67f, 0xb7698dbe, 0x5562298b, 0x688c257f, 0x9d7fde81, 0xe57f4e81, 0x7f81c9ea, 0x281a41f9, +0x7ffdfc6c, 0xaa32217f, 0x81a08166, 0x8181b1ae, 0xc447880e, 0x2b81367f, 0x2944a47f, 0xc6de9b49, +0x81816841, 0x54fa7fa8, 0x587beb7f, 0xe3475a81, 0x81dc2cb1, 0xa95dc06d, 0x8bd7477f, 0xb05d7f81, +0xb4c19d7f, 0x43817f53, 0xde7f5c7f, 0xe3d2817f, 0x8948aadf, 0xc17f7ffb, 0xd08188c1, 0x8181204e, +0xb3b6d781, 0x7381b381, 0x787f7681, 0xee640849, 0x657f8181, 0x811b329b, 0x5c728eed, 0x0c7f59c9, +0x4cc3b12c, 0x814f7e3d, 0x5f7f7f46, 0xf9817fd9, 0xb1b5ba76, 0x1f42609e, 0x3db30000, 0x400d7f46, +0x7f7f7f81, 0x45d36281, 0x81df9581, 0x7a7f7f53, 0x8181330a, 0x7ff5c3ab, 0x9e81b4bd, 0x7f70c27f, +0xac1666b3, 0xcf7f52e2, 0x817fea30, 0x8c4b3f7f, 0xa085bde5, 0xabae307f, 0x817f7fde, 0x17bc61b4, +0x914ccc4b, 0xe1e56bae, 0xac294f88, 0x75e562cd, 0xbfbd35cd, 0xed507f81, 0x81afcf81, 0x7fdc819b, +0x5a0385e8, 0x4f2aabf2, 0x569c7f3f, 0x7fbee1bf, 0xb0407fc4, 0x6b548214, 0xbcd6877f, 0x3d6e7716, +0x9a8134a3, 0x0a7f7f81, 0x29d73eca, 0xadbe7f65, 0x7fc1bb81, 0x7f63dc7f, 0x7f815430, 0x810e2366, +0x7f36057f, 0xda39d7d0, 0xe9855ddb, 0x7fffa2f9, 0x3b9a8181, 0x7f06b781, 0x844db0a2, 0x7b1d7fa3, +0x99a795ad, 0x7b819a6a, 0x3781819a, 0x867fc788, 0xa181817f, 0xfcc3637f, 0x747f5a5a, 0xf27f690f, +0x2f8a9c2f, 0x617f7fb3, 0x81b4b1c2, 0xa2818b96, 0x7f7f7fa6, 0x20707f81, 0x7fc9817f, 0x55577fc3, +0x777b40b9, 0xdffa3781, 0x38ba0e4e, 0x4f667d32, 0x9cf081ea, 0x8187d122, 0x79af9ebc, 0x647f99c2, +0xd1a47f79, 0xaba67f7f, 0x6b270e96, 0x90c07265, 0x817f4b81, 0x71f31f65, 0x2d818692, 0x65570681, +0x81f59b98, 0xe6b181ec, 0xc5b1a58f, 0x7fb2c981, 0x7fdb5914, 0xe37f81bd, 0x7fefd823, 0x9a7f9ccb, +0x631762c2, 0x7f4e81b8, 0x813d88c1, 0x9768c97f, 0xbb867f81, 0x2cd79456, 0x00004aa7, 0x8ec66e57, +0x7f7af878, 0x747f3cb4, 0x8c774fa4, 0x7f8df013, 0x8c5c7264, 0x3aade68d, 0x81819c30, 0xb0d38147, +0xbd816181, 0x4f415cbc, 0x7f21f710, 0xf7ba5565, 0x81df367f, 0x637f7f9a, 0x7f182e7f, 0x6f81996f, +0xca547fe5, 0x637f8db9, 0xccb4fe59, 0x55ca7f81, 0x7f7f7fae, 0xceffc677, 0x2955d0f8, 0x91a07f7f, +0x7f818181, 0xebc2f87f, 0x7f213646, 0x94a2a981, 0xd1819bdc, 0x7f866084, 0x2381ff8b, 0x75b58114, +0x447f162b, 0x5275e0a3, 0x37578cc2, 0x0dc1ec31, 0xb73681a8, 0xb081978a, 0x74429a2c, 0x698859dc, +0x937fb91d, 0x7fcdc06e, 0x81b4ffff, 0x7e2b168d, 0x4b649aa9, 0xe648be08, 0x44e37b7f, 0xc0fc27e9, +0x1b16521e, 0x81577fc5, 0x7f8152a8, 0x2981ee7f, 0xf08dd19f, 0x7d93ed81, 0x814435c5, 0xe6c67f29, +0x7ffc8111, 0x87f65881, 0x3f62563a, 0xa09a81f9, 0x490f5781, 0xc70d6284, 0xfb818181, 0x56817f38, +0x87420960, 0x0d2dedb3, 0x8ef18143, 0x4c5b4a62, 0x8f947f81, 0x667f8121, 0xcc817f81, 0xb567a44c, +0x7996b182, 0x736e9881, 0x81627f7f, 0x7f8e7f81, 0x725ee43d, 0x9f436a50, 0x7f055d88, 0x93498181, +0x21d08f7f, 0x26f17f3c, 0x581d939b, 0x90595ba7, 0xf5200e81, 0x4b91b78f, 0xcb7fc693, 0xe87fd04b, +0xaf81087f, 0x14ac4a01, 0x7f7d7f81, 0xe014dd7f, 0xc2816aed, 0x79760a93, 0x7f9b817f, 0x81480000, +0x844d73a5, 0x7f8181bb, 0x819dae3f, 0xc87f8e82, 0x9d81a23e, 0x7fbd813a, 0x8181702e, 0x4fab7a67, +0xd57f70b6, 0x817c53ac, 0x1355de7f, 0x4264437f, 0x81b0ff2f, 0x16d6e094, 0x7fc40f55, 0x7f9b7edf, +0x7f5a7f81, 0x7f996e66, 0x8e932881, 0x7f817f81, 0x6d4ca679, 0x815f871e, 0x81b4817f, 0x7fa97f81, +0x5e03827f, 0xbcd8b6a8, 0xc5683edc, 0xbb007fdc, 0xea266973, 0x498e467f, 0x86b1cc7f, 0x17814738, +0x81b7ae62, 0x56fea8f3, 0x5eac534d, 0x54f67f81, 0x24add4ce, 0x811a7881, 0x81cfc383, 0x9f2f81ef, +0x7f1f41fe, 0x819e3781, 0xcc73a3b0, 0xee148181, 0x7181d87c, 0x81c1c9e2, 0xfee4727f, 0x9b608139, +0xa2ec4ace, 0x7f810c7f, 0x9a7f2288, 0x46b169e4, 0x9b40817f, 0x547f2f7f, 0x81ef4f81, 0x243fffe3, +0x8181dd3f, 0x347f6596, 0x7f59c3fa, 0xa26cc7ac, 0xf7384881, 0xbdc0a118, 0x7f8173a8, 0xabf1b3e3, +0x7e3e8eac, 0x30c07bce, 0xbca0b981, 0x75816414, 0x81377f81, 0x7f7feaa1, 0xd8287f86, 0xfbd98bd1, +0x7f987fc1, 0x81b8347f, 0x8c7f37e2, 0x987f3c7f, 0xb5f681ab, 0x347f1adc, 0x1b815481, 0x91f17f60, +0xbf85a67f, 0x6d7f2374, 0x587f3c1b, 0x74b0ae9c, 0x97a64fb4, 0x1437697f, 0xc77f3b81, 0xf4e0bbad, +0xb6892b64, 0x5ed0e47f, 0xa55c8100, 0xbbe24e10, 0x50bb6551, 0x6db5349d, 0xaaae405f, 0x4d7f4f6d, +0x989e95a5, 0xcae9bf2b, 0x811e7f4d, 0x90538181, 0x7f9f096e, 0x4126a5df, 0x1369537c, 0x81a73581, +0x722f1409, 0x8b297c54, 0xf17f5081, 0xc22b4595, 0x7f6fde57, 0xdea96095, 0x7f98814f, 0x40f8fb81, +0x697fd77f, 0x35e07f33, 0x27459a7f, 0xc0818181, 0x3e817fb2, 0xdd5eae81, 0x8a8efd53, 0x13e91281, +0x2c486ba8, 0x7f6b447f, 0x7f818120, 0x1c496781, 0xf8e45993, 0x48817f81, 0x009db045, 0x7fd9c722, +0xab817fe2, 0x73bd7951, 0x687fa97f, 0x7f7f7fb8, 0x4f447ffb, 0xfc31d9c2, 0xc1d5d865, 0x526abd82, +0x7f6e7f47, 0x6b31817e, 0x8509817f, 0x5581fcbf, 0x7f9bd8ac, 0xcc22f37f, 0x547fa07f, 0xbd4fd2bb, +0x6f49a523, 0x835e896c, 0x65458e81, 0xd0e37fd2, 0x81587fe0, 0xd95fb08b, 0xd938d43f, 0x11e67fc6, +0x7fec3c2e, 0x7f8581d5, 0x017981b9, 0x812c8133, 0x307415cb, 0x2081bd7a, 0x994d5c83, 0x53812ab3, +0x7f8190bc, 0xad811645, 0x73c1ca14, 0x94c87d32, 0xbd4a7063, 0x7f442f7f, 0x06812617, 0xc481acc0, +0xc7bacc81, 0x81de5481, 0x7fa3167f, 0x4a3a385f, 0x2b817f37, 0xae912b7f, 0x32c4cfeb, 0xc3bf3bb1, +0xa4a63e50, 0x846c7b7a, 0x5c81816a, 0x75813daf, 0xd0665307, 0x6031117f, 0x507f5429, 0x29bfc44e, +0x81b69437, 0xa4c54d61, 0x81391b7f, 0x98347fec, 0xf7da5b06, 0x7e3d8b7f, 0x7f7f1abe, 0xa955d23a, +0xb7810000, 0xa1608164, 0x5f813cfb, 0x16602281, 0x5fb6814d, 0x3c3fd67f, 0x5dae83eb, 0x58ac2c2c, +0x097f5d25, 0x6081da52, 0x7fd4818c, 0xe181d13c, 0xc66d7f2c, 0xe94a00a3, 0x2cec12a0, 0xe84c6398, +0x64398162, 0xd056816f, 0x8147e081, 0x9ceb75c0, 0x7fbec481, 0x16d15734, 0x8158cd7f, 0x369c8172, +0x2a479b81, 0x7f4e9686, 0xb1494e0f, 0xe5816226, 0x057f7f55, 0x81dba929, 0x8f1890e4, 0x5e7f817f, +0x7f7f8162, 0xfced63c7, 0x370aadc1, 0x5f09b34f, 0x7fc0be25, 0x36927fec, 0x814d0ab6, 0xfd077f60, +0x8162f9ab, 0x8185597f, 0x2bac2f66, 0x28e6aa9f, 0x814dc1c6, 0x717f3841, 0xffd1078b, 0x4f5b8a12, +0x7f429a7f, 0x7f734d7f, 0x43817faf, 0x7f4e7f52, 0x232e0524, 0x7f4158e4, 0x327f7f7f, 0x89b464d8, +0x8130a3a7, 0x7fe6ac32, 0xd1818e81, 0x427f957f, 0x8150c60e, 0x3cb50481, 0x31955f7f, 0xec427f9f, +0x99eb5f69, 0x8ac893da, 0xf97da17f, 0x7f847f73, 0x7f97ce7f, 0x7f7fe806, 0x8984629e, 0x34539cba, +0x8a64a575, 0x25297f49, 0x1c509d07, 0xd5aa7f7e, 0x74817f79, 0x81819981, 0x95815a8b, 0x9518a19f, +0x3d9183ce, 0x817f00a1, 0x30306781, 0xc67f7f7f, 0x81594a49, 0x3d620f81, 0x3481bd81, 0xc3af8e63, +0xc249dc8b, 0xe5fa637f, 0x81185512, 0xa0b48307, 0x814c817f, 0xc6dac58b, 0x8b5c781a, 0x81982c81, +0x0000b269, 0x8105f78d, 0x81ad60f7, 0x81ffb8cc, 0x917ffed4, 0x104fc0a1, 0xfa38b45b, 0xb5318137, +0xbd781466, 0x0bd8068c, 0x72c57f91, 0x8181b48e, 0x7f81aedc, 0x43677f7f, 0x7f4f7f3a, 0x437f8178, +0xe99d0b81, 0x868792bc, 0x17727f57, 0xada05e06, 0x694d707f, 0x9081e59b, 0x7f3a8181, 0xb2bae481, +0x7f81dde5, 0x3e307f7f, 0xcaa07f3a, 0x2e1a8170, 0xed0f4881, 0x297f7f61, 0x81cc815e, 0x4fec813b, +0x1a4f02b2, 0xc6e66a4e, 0xe9817f6f, 0xc2c36a02, 0x302e9f60, 0x4ea28137, 0xb9d57f32, 0x462a7f81, +0x9085bb36, 0x7f7f817f, 0x3c714908, 0x70c9c17e, 0x5a7f204b, 0x8fbf6f70, 0x97880981, 0x0cbacfdb, +0xb80f6d81, 0xb6bb3aef, 0x81691db8, 0x1081e2df, 0x7f0504a0, 0x797d5e7f, 0x006de423, 0x7f19f16f, +0x1eb28181, 0x819f7f81, 0x7f6a84b9, 0x936537f2, 0x14a87f81, 0x7fa9cb81, 0x81813a8c, 0x8182c47f, +0x56758144, 0x81c55c97, 0x86de0b81, 0x818181d9, 0x7f386281, 0x7fbb2738, 0xffd18d81, 0x17e1eb34, +0x656449b7, 0x7f7fbe33, 0xd7f681b6, 0x77313c4f, 0x7dc9151e, 0x28debefb, 0x7f7f5c62, 0xa850b25f, +0x81166b59, 0x7aea7f05, 0x325d81a1, 0x7fd06471, 0x93137fbb, 0x43a51892, 0x3e326cfe, 0xab81a942, +0xb208d017, 0x817f0781, 0x08fcca86, 0xa31a7f81, 0xeb93206a, 0xd4fa21f4, 0x684f9029, 0x7f817f57, +0xdca7812c, 0x7f7f0000, 0xf2dc5a96, 0x7f62aa7f, 0x918d1c7f, 0xfbb5add6, 0x407fc0da, 0x4d5d9f7f, +0x935d8166, 0x817da9b1, 0x205d7f3a, 0xe08b4181, 0xd8bd7f7f, 0x4f33818c, 0x8105b781, 0x2f81cd81, +0xbaf8a48d, 0x2b81aa7f, 0x90b5810d, 0xbc62c181, 0x1592f72a, 0x4d81b5fe, 0x7f81e447, 0x99d22181, +0x303e3ea5, 0xa23078a5, 0xba8a7995, 0xd5282790, 0x6681453a, 0x7ff7ab7f, 0x7f4e8195, 0x8bc48181, +0x817fd481, 0x81b98142, 0x265a8182, 0x81f768a7, 0x85f11690, 0x7f957f7a, 0x109981a4, 0xeabba365, +0x577f9057, 0x5f7fae65, 0x817f8199, 0x718e3daf, 0xd6818181, 0xd1a1cc81, 0x97a48187, 0xd8d14520, +0x497b7f10, 0x7f81d781, 0xb77f747f, 0x7f813974, 0xc52a7f7f, 0x7a292226, 0x81602ef8, 0xe481cc7f, +0x0b4266a9, 0x4722287f, 0x7648747f, 0x9c861ec6, 0x4ba9efbe, 0x2063813f, 0x5b277f81, 0x54608181, +0x81b3ae7a, 0x5b812281, 0x6c306f81, 0x4fa0bcd5, 0x540c85d5, 0x2f81817f, 0xd2c9078a, 0x4281a39a, +0x3e7fa1d8, 0xa47fb881, 0x3efe9c7f, 0x7f5f6c23, 0x4681d46f, 0x49cb81c9, 0x7f4eaa68, 0x2e7f7fac, +0xbb497fd0, 0x3a4253a8, 0x91a6aaaa, 0x4f7f5a52, 0xf9375fad, 0x8b2a7f2d, 0x746e7f81, 0x7fb0cbe0, +0x5a917fd1, 0x5b65867f, 0xe9606f9c, 0xe9f8814e, 0x3045cf65, 0x937fad52, 0x760f5a81, 0x9be4ba50, +0xb59d81e5, 0x0000d77f, 0xd481bd81, 0x27224a55, 0x61467c81, 0x91d865df, 0xffa17fe4, 0x7f8a0c68, +0x1640fd52, 0x987f7f6e, 0x4d815a5a, 0x3081b65b, 0x7d7f6c8e, 0x5b26bf60, 0x44814281, 0x5a813427, +0x81f47a3b, 0xef9281eb, 0x81994ca6, 0x7081317f, 0x7f7fd49a, 0xfb8c8e7d, 0x546ad581, 0xd6817f16, +0x7f3e4b81, 0x4f4c4898, 0x817fe22f, 0x819a37c9, 0x387f1a55, 0x8cfddd03, 0x318aa9bb, 0xcb6e2d90, +0xa47f0d1b, 0xae81c204, 0x7f797f27, 0x0a7f65ad, 0xe4ac81af, 0xb2d74d81, 0xca0fad81, 0x357f69a8, +0xad71a581, 0x5be9d4de, 0xaebe8181, 0x95817fd8, 0x8f7aedb6, 0x0a81e59e, 0xa07f20eb, 0x7e66427f, +0x56769b04, 0x2e8149f1, 0x81f6f87f, 0xc3813768, 0xbcc28850, 0x6cbd3dac, 0x7fda99dc, 0x657f7fd5, +0x7fcf7f60, 0x81233698, 0x81daf17f, 0x867f7f7f, 0x7fda7f36, 0x0c7fae20, 0x7f817d78, 0x7fd09481, +0x818f30e0, 0x7f437c7f, 0xaecc2b3e, 0xcf427fca, 0xccfd2cce, 0xefd898bc, 0x7f5b81e2, 0x1d4b812c, +0x865871ae, 0x553ab4ae, 0x69817f7f, 0x811a745a, 0xa32a163d, 0x2e967f57, 0xff817f7d, 0x70d43e9a, +0x814681ae, 0x9f817f31, 0x819b8110, 0xfca3819d, 0xc4fc45b2, 0x56b2817f, 0x7c811d7b, 0x997f8181, +0xbe0dc3d8, 0x9f5b56a6, 0x7f8181e2, 0x5d59eb29, 0xa357e4c6, 0x8161817f, 0x3c7e7f32, 0x7fd07f06, +0x817f82ad, 0x93de8341, 0x92850000, 0x0c65c06f, 0x0a4d2817, 0x7705d752, 0xbcd8687f, 0xc4397f7f, +0x1b7ffd47, 0xada5012d, 0x7f8158e5, 0xba617e10, 0xa97d811f, 0x1fb61081, 0xac41e74e, 0x8c7ea7a3, +0x997f452e, 0xb5858149, 0x5c3cb0a0, 0xc8ffab38, 0x49d65d7f, 0xbbd07c81, 0x35b06b7f, 0xcc7f7f7f, +0x50c42881, 0x707f7fde, 0x7fd67f7f, 0x7f819981, 0xd4d6a781, 0xcb103b7f, 0x7f4fee9f, 0xb3997fe0, +0x7f9381a5, 0xbc7f9a28, 0xf4aa819f, 0x7fe862df, 0x316dd1f8, 0xb4a66f81, 0x98816581, 0xdb817f2e, +0x34819ee0, 0x746c817f, 0xdc9cd3e1, 0x8e7fab2d, 0xec368b2e, 0x7f8381c8, 0x817f7f7f, 0x814de179, +0xed5b7f4d, 0x274bdfa1, 0x5851250b, 0xec811a81, 0x7f365569, 0x57935681, 0x08757781, 0x196d813e, +0xa67581b5, 0x83a7d981, 0x73988163, 0x816eb57f, 0x2ce47e40, 0x5bef6481, 0xb131155f, 0xc2f9817f, +0x519b67b4, 0x7d2c7f59, 0x7fdb8150, 0x5c7fc881, 0xca285c9d, 0x7f7f7fe6, 0x39aabd59, 0xb9b494e6, +0x81401831, 0x74247792, 0xab7dfaab, 0x817fcd81, 0x819b25e3, 0x587f337f, 0x817fd238, 0x818a332c, +0x9db6bcc3, 0x1681c227, 0x7058306d, 0x50a27f64, 0x8181a4b2, 0x7fe2a57f, 0x8d5f4777, 0x7f7f8f81, +0x6e8381f7, 0x22518136, 0xae72d0d6, 0x7f9bba7f, 0x8117d27f, 0x792b7ff0, 0xe45892e0, 0x8e6f5c34, +0x3994817f, 0xa981afca, 0x0000a592, 0x7f811d81, 0x09548181, 0x0ea8127f, 0x7f7fb49d, 0xd6690b81, +0xa1846059, 0x81d15d81, 0x6ccaf481, 0x81ddef83, 0x7fccc681, 0xab502722, 0x4620e862, 0x3981de76, +0x7f81889f, 0x5b818154, 0x797222dc, 0x4b8153a6, 0xa470816d, 0x7fb2657f, 0x81150b64, 0x89dc1f57, +0xaa636e7f, 0xe59cba67, 0xb3b2da41, 0xc7818192, 0x7637d0d9, 0x1a78a78c, 0xaf5e3f5c, 0x8181a581, +0x56285469, 0xeed47f7a, 0x697ff6a5, 0x7f7b7f1e, 0x398181cb, 0x999fe4a0, 0xc17f8816, 0x5c815aa2, +0x78717fce, 0x7f7f842c, 0xf07f4c43, 0x7f818148, 0xbc66c134, 0x8168ce22, 0xaf1791a8, 0x3522ae40, +0x95afa97d, 0x435d4e69, 0x2ab59d7f, 0x8197c47f, 0x93cfe645, 0x8dba727f, 0x227f7f81, 0xb17feb7f, +0x702a7f81, 0xebd463b7, 0x5381e929, 0x817dba7f, 0x5c811a24, 0xfa7f814d, 0x817f8197, 0x2f7f704f, +0x36817f7d, 0xe87f0d7f, 0x54b07fb6, 0x81733686, 0x8fcb7fcf, 0xd888c623, 0x2de5ebc5, 0x8f6f6569, +0x7f7981f9, 0x908136b1, 0xa93a7fb5, 0x405f8141, 0x01a54dbc, 0xb6815763, 0x8ccc3ffc, 0x5681105b, +0x4a1498bf, 0x4c6f82e5, 0xa67f556c, 0x7f5d564f, 0xdb8132d0, 0x1e958139, 0xe386811b, 0x04845a40, +0x7fbd8109, 0x833b62ec, 0x81f40717, 0x701d8181, 0x8191ee44, 0xae2d9d7f, 0x3fa17854, 0xcbcad981, +0x73c2d681, 0x5f738681, 0xb87081dd, 0x817f0000, 0x037fcda9, 0x922caef5, 0x7cc7f91e, 0x9a7fbd88, +0x81819528, 0xf2cd7f53, 0xfd7c8170, 0xfb7fb997, 0x7f7f3f7f, 0xe4818154, 0x18c4414b, 0x4fa48da4, +0x81718181, 0x817c957f, 0x7f2e1582, 0x09bc81bf, 0x5e4b817f, 0xd5590d7f, 0x7f818b44, 0x88abcf7f, +0x39508181, 0x81ab86bf, 0xba58c325, 0x65bc83ed, 0x3402857f, 0x81c61e7f, 0x8bbab505, 0x81b2aba0, +0x8153ec32, 0x6803d681, 0x819d817f, 0xb6de31a4, 0x367f7fc2, 0xb1048db9, 0x6389815b, 0x7ffb7b01, +0x8f1b817f, 0xc1817f7f, 0x04187f64, 0x45cfac7a, 0x0f1b578c, 0x0f209f24, 0x33817f7f, 0x7f7f7f81, +0xac814781, 0x5a9d3dbf, 0x62a65581, 0xcb8a7353, 0x7f81c5b8, 0xba447fa1, 0x9081774d, 0x7af9ed38, +0xb6849bce, 0xacf07c18, 0xa1894581, 0xd7634242, 0x2d0e8174, 0xdbf799d9, 0x7fe7b446, 0x84110cf7, +0x6474da81, 0x8185e120, 0x9da4f123, 0xd07fce54, 0xa9cdf7af, 0x84db817f, 0x9e9a23b7, 0x81774c7f, +0x7f818152, 0x817f5243, 0x39058181, 0xbbc91e81, 0xe3d87581, 0x81b66581, 0x7fb06a61, 0x0bcd0fdf, +0x8188dee9, 0x1cd87f92, 0xf036977f, 0x977f4d61, 0x1da12581, 0x3a7fbee0, 0x87467fa8, 0x7fe27fa3, +0x89d37f7c, 0x397c7ff4, 0x57e64171, 0xc4819181, 0x93677f41, 0x65be2517, 0xbe6c7f7f, 0xaa7f7f81, +0x255d650e, 0x098181c9, 0xe8124525, 0xc73a4949, 0x24d14b47, 0x6730f681, 0xfe251c6e, 0xbc7fa092, +0x81917f5f, 0x81d1ab81, 0xd6128193, 0x13d62e52, 0x941c14f4, 0x25b452c6, 0x7f9c947d, 0x815d3f42, +0x81c8c9d0, 0x7f524663, 0x40ae7524, 0x2f81c77e, 0x815c8483, 0x7f7fb27f, 0x7fc37f44, 0xe07f9db7, +0x7b812117, 0x1c965b7f, 0x1822ad4b, 0xccb9e67f, 0xc5b18115, 0x7f8c377f, 0xd13939cb, 0x38e8147f, +0x4e722d21, 0x9e81ef00, 0x81c0fa81, 0x812d8158, 0x9d84a57f, 0xefc32797, 0x7fc9987f, 0xa37f8199, +0x81812681, 0x81818190, 0x89a9814c, 0x7945c381, 0xa57f816d, 0x7d65af81, 0x81bb7f72, 0x1c81cd7f, +0x6d3e4e7f, 0x6cbef90d, 0xca24894d, 0xb04892b6, 0x7f7f1e81, 0xf57f5965, 0x0769a7b0, 0x698562b9, +0x16817f7f, 0xabb94bb7, 0xc35c81f3, 0xb9b5728e, 0x8181641f, 0x81d2d2be, 0x9181bb37, 0xf82b9222, +0x7bdc7ff5, 0x9b812fa1, 0x8173f4ec, 0x94810185, 0x2bd00d63, 0x814c50dd, 0x9be376e5, 0xed7f90dc, +0x817bf98c, 0xe1df634d, 0x287f81ae, 0x27a4f115, 0x62101a81, 0x2723b481, 0x9c507f66, 0x44a01769, +0xbfd13025, 0x7f815b81, 0x5d5c817f, 0xb37f7fa5, 0x7f79b2c6, 0xa1e2847f, 0x813ac57f, 0x417f8b7f, +0x4b505b55, 0x22d47fbb, 0x7f886ad7, 0xf78c7f48, 0x53a5245d, 0xdfd37f1b, 0xb37f81c4, 0x907fc99f, +0xa393f1c0, 0x7f5f7015, 0x37be3fb3, 0x7f4c9e28, 0x7f770000, 0x9b7a0531, 0x7f35cdde, 0x5b4047cf, +0x81881773, 0xdc6e9ea5, 0x517f477f, 0xb2a77f81, 0x9981472d, 0xd4c57f81, 0x6ef2a77f, 0x81d481a3, +0x3d87307f, 0x214321bd, 0x7b7f7f8e, 0x997f1cbe, 0x266681ec, 0x3639f87f, 0x687f7fed, 0xf17f8191, +0x1558ae38, 0x815ac086, 0x81247f78, 0x7f612db7, 0x128d7fc9, 0x187fbf81, 0x4a898116, 0x3a6c818d, +0x7f7fdcf1, 0x60cb3e9c, 0x81bf627f, 0xe7c17fa1, 0x52247fd2, 0x814781dc, 0x6eb67f1a, 0x9d0b7f7d, +0xc2b238f8, 0x81b1427a, 0xa1555aad, 0x103c0c29, 0x97319c81, 0x05ab3a7f, 0x811f1d7f, 0x817f8195, +0x40bffc7f, 0x3aa38f4a, 0x46e17fcb, 0x6781fc31, 0xb725a5e4, 0x427f8cbd, 0x7f857fd4, 0xca1f7f9d, +0x2f77c6b9, 0xdde30a0e, 0x7f93fca2, 0xc6aa7fc7, 0x7f9e7f7f, 0x9b78012e, 0xdda0d07f, 0x0b4c7f83, +0x7fee4b44, 0x81640552, 0x40d573c0, 0x390efbce, 0x327fb881, 0xf5e69aec, 0x81e9c7d1, 0x81717f74, +0x81817d78, 0xac6eba86, 0x41a2d07f, 0x8981da45, 0x5264b681, 0x527f46f2, 0x949e7f52, 0x7fdd7fbd, +0x81678102, 0x87756c64, 0x3d2a81ca, 0xc87f8107, 0xd17fb64a, 0xa7b553b5, 0x8191812a, 0x55ddbf7f, +0x517f7b7f, 0x897aad7f, 0x63817f8d, 0xd9547f7f, 0x43437351, 0x84818181, 0x3a7f8127, 0xccbfdf7f, +0xe1191588, 0x81c80c65, 0x7ffcf571, 0xcc147b7f, 0x00007081, 0x2b7f7f7f, 0x889b3a2b, 0x5358e78b, +0xfcb917ab, 0x81c3f74d, 0xd57faa18, 0x69937f23, 0xd7321b81, 0x5db1ad81, 0x17577f0a, 0xf48118bf, +0x81822681, 0x2981e87f, 0xc16e6c7f, 0x891d38ae, 0xf37f813f, 0x81b96b55, 0x2fcd81b1, 0x23b5f981, +0x695c4d28, 0xa0810fab, 0x6cd5ea3d, 0xefc68ec4, 0xae32887e, 0xa27f0b17, 0x7f8baa97, 0x8116b3a1, +0xa393817f, 0x817019f0, 0x1d7f8193, 0x18f0d764, 0xb27fb7c9, 0xc4da637f, 0xcd586f56, 0x7f23810d, +0x572f25f3, 0x05df7f15, 0x817f815c, 0x8b147f84, 0x0f817f6b, 0x7f1f7f7f, 0xbccee4a8, 0xcb5ad7ed, +0x33a5f88b, 0xc7817ecd, 0x8781817f, 0x817e6444, 0x6697817f, 0xf9490257, 0x83d5f581, 0x7168b332, +0xf4ed4781, 0x17e0c1c4, 0x16b9c997, 0x8581247f, 0x81887b3f, 0x2281a5c7, 0xce437f0f, 0x994c6f81, +0xbcd09735, 0x9269d57d, 0x449a7e61, 0x8c6e7f81, 0x81b38c7f, 0x81db7791, 0x62602681, 0x3246817f, +0x81bb2881, 0x6d2510b4, 0xb07fbf69, 0x49ea817f, 0x938153e7, 0xf17f425b, 0x3b817f89, 0x36b48181, +0x837c1d0a, 0x853cbd2c, 0x2950527f, 0x388de1bc, 0xfa1c7f54, 0x7b507f7f, 0xad311c42, 0x8d8ae6c3, +0x9081677f, 0x58810e81, 0x984b4881, 0x4b9f407f, 0x819a00ff, 0x857faee2, 0xa06d9fb5, 0x448979b8, +0x9cc0f17f, 0x7f2fce7f, 0xa0b9ad7f, 0x428a7f7f, 0xdf562afa, 0x7f6f0000, 0xe04a5bef, 0xa7ad2867, +0x23818146, 0x7f7f0642, 0x7f481981, 0x7f810f81, 0x99a9db81, 0x64d437cd, 0xaf7fcb6d, 0x81815311, +0x723c81bd, 0xcc7fc729, 0x81a47e49, 0x9abce744, 0x7385eb5b, 0x7f7f5b7f, 0x2bb3819d, 0x81fd7f49, +0x7f5f8198, 0x7fb6a2f3, 0xe97fc33c, 0x436c79d3, 0x5ea67fff, 0x5a7ff2ac, 0x28aa713c, 0x8181c681, +0x7f7c8846, 0xaf817f39, 0x90bcc57f, 0x841c7f4a, 0x817f8181, 0x7f4e811b, 0xb53a81ce, 0x7f451a7f, +0x5f38027f, 0x7b81e4f3, 0x817f87c6, 0x8141318c, 0x532c81a4, 0x817fc84c, 0x919d33db, 0x29f2b781, +0x7f568139, 0x4658b1a6, 0x7f69342d, 0x81ab9582, 0x367fa301, 0x47468181, 0x7f8c8181, 0x81216a81, +0x9b577f7f, 0x7f447f7f, 0x72ef4649, 0x45418103, 0xee48bb81, 0xa7027c7f, 0x0ea4237f, 0x6b717cf4, +0xa296a17f, 0x17007f7f, 0x869835c8, 0x8a7f77ec, 0x5d7b8139, 0x32688f81, 0x7fdab0b3, 0x815c7f7d, +0x06a01fb6, 0x814981de, 0xa1c681d8, 0xdc6ff522, 0xd13081b1, 0x9c5a7ff5, 0x7f7f7f5c, 0x86faa148, +0x811a696b, 0xbb7fa2c5, 0x815779bf, 0xae3b7f48, 0x8c45d6de, 0x7f867f9a, 0x3e359b7f, 0xb7c48181, +0x78c27f95, 0xc4818181, 0x81cd7f3f, 0x558a8193, 0x7f4ca874, 0x4e1eb447, 0x2744355f, 0x9020ead4, +0x16e47ff6, 0x708123b7, 0xbeb61418, 0x27fcdd50, 0xde21e5d8, 0x9500a3db, 0xefc29f4d, 0x5ef0c542, +0xe02a3bc8, 0xe91a38bf, 0x24f81e3c, 0xe0fe4f11, 0x3be6891d, 0x7fef09ae, 0x16977ff0, 0xe8b74a18, +0x1a50fa2e, 0x24c04bf9, 0x39c84714, 0xbed7f6c1, 0xaeddab44, 0xa7f9ca2d, 0x275dc01b, 0x1df6e74b, +0xc3171107, 0xbcc93481, 0x1403f685, 0xcc1c0439, 0xb602ebaa, 0xe235d0e1, 0xf1afe3e3, 0x0814c025, +0xb4f93607, 0xa709c243, 0xbbfabdd8, 0xfd3e81e7, 0x2f9ae335, 0x1c1d52e9, 0x00f133a5, 0x23eee238, +0xc625eced, 0xefd56878, 0xe3cadc03, 0x585db90a, 0x024ace2b, 0xe3d804ef, 0xdf54d5a1, 0xc9e11709, +0xc3a20ccb, 0x18c0005f, 0x52424437, 0xccaba110, 0xc75736d6, 0xa8f1401a, 0x3186aa67, 0x0b1aaebe, +0xf952cf90, 0xfa4edd16, 0x9d72bee5, 0x2824f455, 0x9a3cb150, 0x6bbfe929, 0x6a6f5f9d, 0xbfa5385f, +0xf89828bc, 0x6b6d0baf, 0xcab514c5, 0xa9baf505, 0x24cff2c2, 0x1c2511d8, 0x3abccde9, 0x7f60deef, +0xbbf4dddd, 0x11366331, 0x0241c141, 0x5111c984, 0xc17f54da, 0xed4c1bb8, 0x0af6fb57, 0xfd2c28c0, +0xe790142d, 0x6bff6403, 0x7fd8cce3, 0x3c282cc0, 0xea48f68b, 0x3ecdd5e3, 0xb917f2e5, 0xd2b25334, +0xf65b10b5, 0xfdd0d4be, 0xe6e22cc9, 0xc59ec206, 0x3f3238cc, 0xbcbe2fd0, 0x2418c910, 0x3d07f7cc, +0xa25bf9cf, 0xdb81ebfc, 0xd35b2203, 0xde2d23cc, 0x3cd706d5, 0x0e34f6f5, 0xc2d20000, 0xfef1e4e0, +0xd26ad6e1, 0xfa130b0d, 0x324b211c, 0xc5dfc3a9, 0xf83ebec9, 0x0dd627c0, 0xcee99850, 0x4fcd1533, +0x04597ff0, 0x40d0ff25, 0x045cd181, 0x908f9ca1, 0xd0417f7b, 0x2f0a2c87, 0xfef998c7, 0xcc3370f3, +0xb0f4267f, 0x1bbe5072, 0xd8d6063a, 0xd058cb32, 0x448fffad, 0xa9b50064, 0x184ebdf7, 0xfe2e3538, +0x561f16a6, 0xe7f20181, 0x449b95f9, 0x0338f7b2, 0x4012bcae, 0xd67f6797, 0x4966f6f2, 0x1dcc60c0, +0xeecec631, 0x279f81c4, 0xc9f751be, 0x75994dc8, 0x01001ded, 0xb0ba1822, 0xbb14eff9, 0xb2dae837, +0x4cf69e23, 0xf35d6ad7, 0xf69e57c9, 0x3727d2e3, 0x31b3517f, 0xc1da575d, 0x30e2c1d5, 0x54d7beef, +0x293edf19, 0xeb37d80e, 0x296cee35, 0xeff14f04, 0x232d06de, 0xb2815853, 0x2c601508, 0xa5ecb6ff, +0xb407f524, 0x4ccd9a19, 0xe448b0c2, 0x01ebd4c6, 0x52fc03a1, 0x286afce2, 0xe4d45ebb, 0x2953101d, +0xca12d31f, 0x10cb00dd, 0xd2ae6347, 0x46e82f44, 0x45d64600, 0xae393c05, 0xf7219fb9, 0xeffe09d2, +0xedfbb5d1, 0x07070413, 0xd7d6ec37, 0xa0f7a621, 0x75bd023f, 0xc45b5d37, 0x19ca9b39, 0xd1ee50b7, +0x50062231, 0x301ca07f, 0xbeadd410, 0x1ce01d02, 0x51a5fbff, 0xd9104c7f, 0x79e4f23a, 0x33e47449, +0x1ceda8ea, 0xf6d65581, 0x418dd3e3, 0x9fc1e612, 0x54f747d9, 0x839acbce, 0x0000221c, 0x031434b6, +0xcc48960f, 0x1045ea23, 0xa8d7f8a1, 0xaf01243e, 0x2bf461c7, 0x9c2139dd, 0xd42bd5d5, 0xf8c2f5d5, +0xdeb61f4b, 0xf12ac9cd, 0x107f3209, 0x818d1d3c, 0xca34572c, 0x34da0052, 0xf9a951e2, 0xdf34d209, +0x0f8b37c1, 0xa7dee3d3, 0x1a390c98, 0x1d16e135, 0x33c3ad66, 0xd9e2cca4, 0xcbabcce4, 0x2eadd581, +0x81383b81, 0xc0db251b, 0x0a6e91c3, 0xdab3c725, 0xd2293bfc, 0x22d6abb5, 0xdb524adc, 0x30cf5f24, +0xb92527e8, 0x6000b7bf, 0xfefe1fe4, 0x2cf9035e, 0x2743f125, 0xd007b3ad, 0x2dd5e43a, 0x5a35e7b9, +0xea06e5b7, 0x39f3e1aa, 0x592614f3, 0x7fc3bab7, 0xf1aefe00, 0xf3e9b9fc, 0xcd81aee6, 0xae3e31cc, +0x95d8fbc7, 0x7f26bbf8, 0xf20e57e3, 0xf9eee307, 0xfde2bff2, 0x10e8f7db, 0xfbb0de50, 0xdfaa10e8, +0x5e18ee3a, 0x0520a445, 0xddd2bc9e, 0x13c97536, 0x33cf5159, 0xfac6d2f7, 0x18ca01e5, 0x112251da, +0x0211d9b6, 0x375feb63, 0x9a25b9cf, 0xfb5bf632, 0xe5b4a803, 0x95f55e8d, 0xdbede748, 0x39aed0bf, +0x5efd9c49, 0xcc7fd13c, 0xfba1a3d7, 0xa8a80206, 0x08f7e7d3, 0xd1fac77f, 0x32fb9272, 0xe1829834, +0x45408175, 0x34e4eec0, 0x0ca4ecae, 0x15e31638, 0x1d072d02, 0xee44d6ee, 0xafa5282c, 0x4d4d9f2a, +0xcdfc0f24, 0x9dfaf345, 0xbf3839c3, 0x3268a3e3, 0x2e4eacb5, 0x93dce12c, 0x4a484f3b, 0xdf7d0000, +0xeb113c3d, 0xd4bdf13f, 0xecc0f511, 0xd4cdf8c2, 0x42fc05f9, 0x53be2def, 0xb22e49cf, 0xead92e2e, +0xf956f1cc, 0x02c821e0, 0x1da533e5, 0x1fc50531, 0x34c30f37, 0x7ccba6e4, 0xab1afe2e, 0xb93bd67f, +0xea28a8a8, 0xf3ef282e, 0xac0c507f, 0xd0cbdd53, 0xd344ea1d, 0x472a1029, 0xed099f70, 0x1240edc6, +0x7f143244, 0xbae0d3d5, 0x7a4bddfd, 0x5f07d846, 0x1cdf4aaf, 0x4fc40ac1, 0xcfca381a, 0xb67fd847, +0xd381dde6, 0xdad40d1d, 0x64cb2d07, 0x171a8bc6, 0x11110cbf, 0x2c442dff, 0xbe90f52b, 0xd75500e5, +0xd8459ef9, 0xff0bc92f, 0x44a903c8, 0xb767d7f2, 0xc281fc02, 0xca534906, 0x25fdec32, 0x5f2aaeb2, +0x121b376c, 0xe3e41dd2, 0x12ecd9f8, 0xc5071a9d, 0xfaaeee7d, 0x183a1a57, 0xadcb921d, 0x1e33a3b2, +0x72e52634, 0xc8d4e81e, 0x3b1e0200, 0x321e1608, 0x49add232, 0x0516ff41, 0xc23181d3, 0xfa4e0151, +0xa63502cb, 0xda381204, 0x72d92da5, 0x7cd7ebff, 0xce08d106, 0x61c6f034, 0x55310f68, 0xed66c357, +0xc527d609, 0xdab11518, 0x41b808e5, 0x5f21e65e, 0xd4f6f036, 0xfc40008d, 0xaa7f19a0, 0xbff0ac1b, +0x78283b60, 0x1d0a4a03, 0xd6d0d016, 0xf1f23847, 0x58d93518, 0xd6d481b7, 0xaf11e02e, 0x5c74b67f, +0xd5734f27, 0x2fb07fc2, 0x29564eff, 0xcafae603, 0x1228f673, 0x3367ce30, 0x132d1342, 0xc80c10e9, +0x68ad07f9, 0x379c0c9c, 0x3a73c010, 0xc1a534bc, 0x69b0fc14, 0xf0d3ba81, 0x0d07bbe3, 0x3319fa45, +0x38815afe, 0xc1a67850, 0x7fe740fe, 0xc43fb638, 0xa0864249, 0x07a7dc33, 0x05e606d4, 0x3c6a81ca, +0x0cdb2635, 0x5dd3f756, 0xcbf09b70, 0x53b2a5b3, 0x5f035ab0, 0xfdda1fbd, 0x0510ac20, 0xa8611f44, +0x38ceb74c, 0x5048e2e5, 0x907f0238, 0xd9ad00d9, 0x7c247736, 0x8aabe3c6, 0xf5443d27, 0x7ff84622, +0x4bcc0a34, 0x797dc7e0, 0xf914e5f2, 0xa12df5d7, 0x0fdc23f9, 0x377f50c3, 0x8f47bf82, 0xb9948a4a, +0x8135ca9e, 0x523ace17, 0xd299fb23, 0x523b5407, 0x38b42a22, 0xadee2cd8, 0xee04b9c8, 0x2de3a5e2, +0x9f132faa, 0xa0030325, 0x019996f4, 0xd6c5562a, 0xf3e19255, 0x00ff2432, 0xe8f6cf36, 0xe55a7f14, +0x7079f06f, 0x1e11ff0b, 0xcbe89aa8, 0x6b363996, 0xe61dd6f7, 0x15133821, 0xf35940f9, 0xd9b139c1, +0x2ad6e10b, 0x2a29c0c6, 0x3c6ee3ce, 0x4b0a3624, 0x23cc10df, 0x17cdb502, 0xa46eaa3f, 0x42350300, +0xed66334e, 0x1d6fc255, 0x3213e4c5, 0x23d4a65a, 0x05b9a3e9, 0xbacd0c19, 0xc9c5b5b1, 0x8e12670d, +0x4aabbe2d, 0xb815e3c7, 0x5a2dd152, 0x3ef6551c, 0xf0a1b4f3, 0xb8b8459b, 0xc52dd72e, 0xb9f4b9f7, +0xbd05dcc8, 0xfae04029, 0xb16ad8df, 0x03d167d4, 0xaf08525b, 0xdbdf3117, 0x4192d163, 0x00a2caad + +hard_output0 = +0xb5bc6ac8, 0xf5373664, 0x1310345c, 0xd5bae4e7, 0x1fc9e83e, 0xebfdfded, 0x84bd86ab, 0xb7aabe00, +0x60b44fea, 0xb9067464, 0x30325378, 0xa9195955, 0xf70c6e5c, 0x90922632, 0xc90b1cdb, 0xf2f5fb69, +0x73056b63, 0x1a33bf3f, 0x17755b5c, 0xc58bff6d, 0x2f4390b2, 0x2869d508, 0xe7c7dfe8, 0x38552963, +0x21da5367, 0x07282b9b, 0xa4767105, 0x1e294251, 0xe350a940, 0xb8a6aa27, 0xed12d778, 0xf10d9ece, +0xab93527f, 0xcf2da7e7, 0x68f6d0b1, 0x811f4bca, 0x577b06b2, 0x3234f13e, 0x30bab7df, 0x8dc47655, +0xbb843bed, 0x86da3aba, 0x30950c97, 0xdd096d7a, 0xa871fd6c, 0x8bee4e6b, 0x8fea30d0, 0x6c05b4d2, +0xf3e144d3, 0xd24ebb1f, 0x065635e5, 0x8d3f2cf9, 0x536c6c6a, 0xfbb0a5d0, 0x3d707b42, 0xc44d5982, +0xa5f4ad8f, 0xf32c0970, 0x1bccf1a6, 0x05916020, 0xa64fb176, 0x5ede6a35, 0xaf4966da, 0x9df5e0e7, +0x75042abc, 0x9ef10481, 0x11ddcbc8, 0xa0f5518c, 0xd5c23418, 0x2393d558, 0xfbe7dfeb, 0xed1c64c2, +0x86a36508, 0xde2dfb1e, 0xb8d0fef9, 0x24505232, 0xc894e71c, 0xbcc752a0, 0x40b74e83, 0x90d23c8c, +0x728e4a61, 0x108f0b08, 0x66f522ee, 0xc258d851, 0x35a31c44, 0x11311b5b, 0xfd3d5be9, 0x5ae448ff, +0x4f64994b, 0x5b8247a9, 0x4021114d, 0x2f0b6e82, 0x5eaa9828, 0x50ac71c0, 0xfb86ee52, 0x0dc1ac9b, +0xbbd47645, 0x8f357115, 0x978ceea0, 0xd557db99, 0x99b30388, 0xfc9a8a1c, 0x0f75be1a, 0x50143e22, +0x8840989b, 0x738ec50e, 0xe6b2783d, 0xf67899c8, 0x27ebed69, 0x6c415a16, 0x3a6cc2dc, 0xcd4e4e5d, +0x6cb12b2e, 0xdb88d7c0, 0x79cd1582, 0xbc422413, 0xe72ad2f4, 0x8eaac30f, 0x0bd86747, 0x6d87f69d, +0x15d62038, 0x4b375630, 0x0d51b859, 0x16db2cb2, 0xf210603a, 0x0abeb833, 0x55c694d0, 0xe57ca43b, +0x0ba94428, 0x1398a406, 0xe47d3889, 0x5a20203d, 0x250d7a1a, 0xd930ffec, 0x03992e79, 0xf2759376, +0x024ec121, 0x91fc3a2c, 0xb7e11cc5, 0x4ff7d459, 0xb8700134, 0xd6e61758, 0x4eba0a32, 0xb747e3ec, +0x7073fad7, 0xded80f99, 0x331e2f1b, 0xfa1f1bed, 0x056424a2, 0x1d1d95e0, 0x550b9ec8, 0x51ee2a38, +0x19525153, 0xd70c4cd5, 0x0d6cd7ad, 0xe44d1cf2, 0x30dfecda, 0xdacd7fe8, 0x7321d795, 0xddf48ef4, +0xe271e6a4, 0x9c1feecb, 0x951fcd7b, 0x8acc5a03, 0x3fb83527, 0xe306de74, 0x7b9cd6ee, 0x8e140885, +0xd4c91e8d, 0xe8c39733, 0x0f02f87f, 0xfb06b1b9, 0x0dc9349c, 0xf76bae8e, 0x4f642a07, 0x3d48a9aa, +0xe3ea323a, 0xa1cd5c8a, 0x40aa0e70, 0x132042d3, 0xa9732f6c, 0xd15a00c4, 0x43d3b046, 0x9a51ebd4, +0xc46ee0ed, 0xe2a2148b, 0xf5c478f0, 0x1fb01cf3, 0xf4f321ec, 0xd973811f, 0x11ad11b9, 0x5c67adda + +soft_output0 = +0x81818181, 0x7f7f7f7f, 0x7f7f7f81, 0x7f817f81, 0x7f7f8181, 0x7f7f7f7f, 0x817f817f, 0x81817f81, +0x7f817f81, 0x7f817f81, 0x7f81817f, 0x7f7f7f7f, 0x7f7f8181, 0x7f81817f, 0x817f8181, 0x81817f81, +0x7f7f7f81, 0x817f7f81, 0x7f7f8181, 0x7f817f81, 0x7f81817f, 0x817f7f7f, 0x817f8181, 0x7f7f8181, +0x81817f7f, 0x81817f7f, 0x81817f7f, 0x817f7f7f, 0x7f7f7f7f, 0x817f7f81, 0x7f7f817f, 0x7f817f7f, +0x7f818181, 0x8181817f, 0x81818181, 0x7f817f7f, 0x7f7f817f, 0x81817f81, 0x7f7f8181, 0x817f817f, +0x7f817f81, 0x7f7f8181, 0x817f7f81, 0x81818181, 0x817f817f, 0x81817f7f, 0x7f7f8181, 0x7f7f8181, +0x8181817f, 0x817f7f81, 0x81818181, 0x817f8181, 0x7f7f817f, 0x7f7f7f81, 0x7f81817f, 0x7f81817f, +0x7f81817f, 0x8181817f, 0x7f7f7f81, 0x817f7f81, 0x7f817f7f, 0x7f7f7f7f, 0x8181817f, 0x7f81817f, +0x817f7f81, 0x7f7f7f7f, 0x7f7f7f7f, 0x7f7f8181, 0x7f7f8181, 0x7f817f7f, 0x7f81817f, 0x7f7f817f, +0x81817f81, 0x81817f81, 0x81817f81, 0x817f817f, 0x7f817f7f, 0x7f81817f, 0x81818181, 0x8181817f, +0x7f817f7f, 0x817f7f7f, 0x7f7f817f, 0x817f7f81, 0x7f7f7f81, 0x81818181, 0x817f8181, 0x7f7f7f81, +0x7f817f81, 0x7f817f81, 0x81817f81, 0x7f7f7f7f, 0x7f7f8181, 0x81817f7f, 0x7f7f8181, 0x7f7f817f, +0x7f7f817f, 0x81817f7f, 0x8181817f, 0x817f8181, 0x81817f81, 0x817f7f7f, 0x817f7f81, 0x7f81817f, +0x7f7f7f7f, 0x7f817f81, 0x7f7f817f, 0x7f817f7f, 0x81818181, 0x817f7f81, 0x7f817f81, 0x7f817f81, +0x817f7f81, 0x7f818181, 0x7f81817f, 0x817f817f, 0x7f7f817f, 0x81818181, 0x7f818181, 0x8181817f, +0x817f7f81, 0x7f81817f, 0x7f7f7f7f, 0x81817f81, 0x817f8181, 0x7f818181, 0x817f8181, 0x81817f7f, +0x7f7f8181, 0x81817f81, 0x7f7f817f, 0x817f817f, 0x81818181, 0x7f81817f, 0x817f817f, 0x81818181, +0x817f8181, 0x817f8181, 0x817f8181, 0x7f817f81, 0x7f7f7f81, 0x7f7f8181, 0x7f817f7f, 0x7f7f817f, +0x7f817f81, 0x7f81817f, 0x817f7f7f, 0x7f7f8181, 0x7f817f7f, 0x7f7f8181, 0x7f7f7f81, 0x817f7f7f, +0x7f817f7f, 0x817f817f, 0x7f817f81, 0x7f817f81, 0x7f7f8181, 0x81817f81, 0x8181817f, 0x81817f81, +0x817f7f81, 0x7f818181, 0x7f7f8181, 0x817f7f81, 0x7f817f81, 0x817f7f7f, 0x7f7f8181, 0x81817f7f, +0x7f7f7f7f, 0x817f8181, 0x81817f81, 0x7f7f7f7f, 0x817f817f, 0x7f7f7f81, 0x7f7f7f81, 0x81817f7f, +0x8181817f, 0x817f8181, 0x81817f81, 0x7f7f7f81, 0x7f7f7f81, 0x81818181, 0x81817f81, 0x7f7f8181, +0x8181817f, 0x7f817f81, 0x7f817f81, 0x7f817f7f, 0x8181817f, 0x8181817f, 0x81817f81, 0x817f8181, +0x81818181, 0x7f817f7f, 0x81817f81, 0x7f817f81, 0x7f817f81, 0x7f817f7f, 0x8181817f, 0x817f8181, +0x817f7f81, 0x817f7f81, 0x7f817f81, 0x817f8181, 0x7f7f7f81, 0x7f81817f, 0x81818181, 0x8181817f, +0x7f7f8181, 0x7f7f7f81, 0x81817f7f, 0x7f818181, 0x817f7f7f, 0x7f817f7f, 0x7f7f7f81, 0x7f817f7f, +0x8181817f, 0x7f81817f, 0x817f7f7f, 0x817f817f, 0x7f7f7f7f, 0x7f817f81, 0x8181817f, 0x7f81817f, +0x7f81817f, 0x81817f7f, 0x817f7f81, 0x7f817f81, 0x7f81817f, 0x7f7f7f7f, 0x81817f81, 0x81817f7f, +0x81817f7f, 0x81817f7f, 0x7f818181, 0x8181817f, 0x7f7f8181, 0x7f818181, 0x7f7f8181, 0x7f7f8181, +0x817f7f7f, 0x81818181, 0x7f818181, 0x7f7f8181, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, 0x7f7f817f, +0x7f7f8181, 0x7f817f7f, 0x7f817f7f, 0x817f8181, 0x7f7f817f, 0x81817f81, 0x817f8181, 0x7f7f817f, +0x81817f7f, 0x7f7f7f81, 0x7f81817f, 0x7f817f7f, 0x7f81817f, 0x7f7f817f, 0x817f817f, 0x81817f7f, +0x817f817f, 0x817f7f81, 0x81818181, 0x81817f7f, 0x817f8181, 0x817f8181, 0x817f7f7f, 0x7f7f7f7f, +0x7f81817f, 0x7f818181, 0x817f817f, 0x7f818181, 0x7f7f7f7f, 0x7f7f7f7f, 0x8181817f, 0x7f7f7f81, +0x7f7f7f7f, 0x7f7f8181, 0x7f817f81, 0x7f818181, 0x81817f81, 0x7f7f7f81, 0x817f7f81, 0x8181817f, +0x817f817f, 0x7f7f7f81, 0x7f818181, 0x81817f81, 0x817f7f81, 0x7f7f7f7f, 0x817f817f, 0x81817f7f, +0x817f7f7f, 0x81818181, 0x8181817f, 0x81817f7f, 0x7f7f7f7f, 0x817f8181, 0x7f817f81, 0x817f8181, +0x817f7f7f, 0x7f7f7f81, 0x817f8181, 0x81818181, 0x7f81817f, 0x817f7f7f, 0x7f81817f, 0x817f817f, +0x7f817f7f, 0x7f818181, 0x7f818181, 0x7f817f7f, 0x81817f7f, 0x7f817f7f, 0x7f7f817f, 0x81817f81, +0x81817f81, 0x817f7f7f, 0x81817f7f, 0x81817f81, 0x7f818181, 0x7f7f7f7f, 0x7f818181, 0x7f7f817f, +0x7f817f81, 0x81817f81, 0x7f7f7f81, 0x817f8181, 0x81818181, 0x7f81817f, 0x817f7f7f, 0x7f817f7f, +0x817f7f81, 0x7f7f817f, 0x7f7f8181, 0x7f7f8181, 0x81818181, 0x7f81817f, 0x7f81817f, 0x817f8181, +0x7f818181, 0x817f7f81, 0x817f7f7f, 0x7f817f81, 0x7f817f7f, 0x817f7f7f, 0x817f7f81, 0x7f817f81, +0x8181817f, 0x7f81817f, 0x81818181, 0x7f7f7f81, 0x81817f81, 0x7f818181, 0x7f7f7f7f, 0x7f81817f, +0x8181817f, 0x8181817f, 0x7f7f7f7f, 0x7f818181, 0x817f7f81, 0x7f7f817f, 0x817f7f81, 0x81818181, +0x817f817f, 0x7f818181, 0x81817f81, 0x81818181, 0x817f7f81, 0x7f817f81, 0x7f7f8181, 0x817f817f, +0x817f7f7f, 0x817f8181, 0x81817f81, 0x81818181, 0x817f817f, 0x7f817f7f, 0x7f817f7f, 0x7f817f81, +0x817f7f7f, 0x81817f81, 0x817f7f81, 0x7f7f817f, 0x7f817f7f, 0x7f81817f, 0x8181817f, 0x81817f81, +0x81817f7f, 0x7f817f81, 0x7f81817f, 0x81818181, 0x7f7f817f, 0x817f7f7f, 0x7f817f81, 0x8181817f, +0x817f8181, 0x7f81817f, 0x81817f7f, 0x7f817f81, 0x81818181, 0x7f81817f, 0x7f81817f, 0x7f81817f, +0x7f7f817f, 0x7f817f7f, 0x7f81817f, 0x817f7f81, 0x7f81817f, 0x81818181, 0x7f7f8181, 0x7f818181, +0x8181817f, 0x81818181, 0x8181817f, 0x817f817f, 0x7f81817f, 0x81817f7f, 0x7f81817f, 0x7f817f7f, +0x81817f81, 0x7f81817f, 0x7f7f817f, 0x817f8181, 0x81818181, 0x7f817f81, 0x81817f81, 0x7f817f7f, +0x817f7f7f, 0x7f7f817f, 0x7f818181, 0x817f8181, 0x7f7f7f81, 0x81817f7f, 0x81817f7f, 0x7f7f7f7f, +0x7f818181, 0x7f7f8181, 0x7f81817f, 0x7f7f817f, 0x8181817f, 0x7f81817f, 0x8181817f, 0x81817f7f, +0x817f7f81, 0x817f817f, 0x7f7f7f7f, 0x817f8181, 0x7f7f8181, 0x817f817f, 0x7f817f7f, 0x81818181, +0x817f7f7f, 0x7f817f81, 0x7f7f7f7f, 0x817f8181, 0x7f7f7f81, 0x7f7f8181, 0x81817f81, 0x81817f7f, +0x7f818181, 0x817f7f81, 0x7f818181, 0x817f817f, 0x7f7f7f81, 0x817f817f, 0x7f817f81, 0x7f7f7f81, +0x7f818181, 0x7f7f817f, 0x81818181, 0x817f7f81, 0x81817f7f, 0x8181817f, 0x817f8181, 0x8181817f, +0x7f818181, 0x7f817f81, 0x8181817f, 0x7f817f7f, 0x8181817f, 0x7f817f7f, 0x7f7f7f81, 0x817f7f7f, +0x817f8181, 0x7f7f7f7f, 0x817f8181, 0x81817f81, 0x81817f7f, 0x7f7f8181, 0x81817f81, 0x7f7f7f7f, +0x7f818181, 0x7f818181, 0x7f7f7f7f, 0x8181817f, 0x7f817f7f, 0x81817f81, 0x81817f81, 0x7f7f817f, +0x7f7f7f7f, 0x817f817f, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f817f, 0x7f817f81, 0x81818181, 0x817f8181, +0x7f7f817f, 0x81817f7f, 0x7f7f7f7f, 0x7f81817f, 0x81817f7f, 0x7f817f7f, 0x81817f81, 0x7f7f7f81, +0x817f8181, 0x81818181, 0x7f81817f, 0x7f817f7f, 0x7f7f7f81, 0x7f818181, 0x817f7f7f, 0x7f7f7f7f, +0x817f7f81, 0x7f818181, 0x7f817f7f, 0x7f7f8181, 0x7f7f8181, 0x81817f81, 0x81817f81, 0x7f7f7f81, +0x8181817f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x817f7f81, 0x7f7f8181, 0x7f7f7f81, 0x817f7f7f, +0x7f817f81, 0x81818181, 0x81817f81, 0x7f81817f, 0x7f7f7f81, 0x817f817f, 0x8181817f, 0x7f818181, +0x817f817f, 0x81817f7f, 0x7f7f7f7f, 0x7f817f7f, 0x8181817f, 0x7f7f817f, 0x7f7f7f7f, 0x7f81817f, +0x7f817f7f, 0x7f7f8181, 0x7f7f8181, 0x7f7f8181, 0x81817f7f, 0x7f7f8181, 0x7f7f8181, 0x8181817f, +0x817f7f81, 0x7f81817f, 0x7f817f7f, 0x8181817f, 0x7f7f817f, 0x7f7f8181, 0x7f7f8181, 0x7f817f7f, +0x817f817f, 0x7f81817f, 0x817f7f81, 0x8181817f, 0x81817f7f, 0x81818181, 0x817f817f, 0x7f7f817f, +0x817f8181, 0x7f817f7f, 0x7f817f7f, 0x817f817f, 0x81817f81, 0x7f7f7f81, 0x7f81817f, 0x7f7f7f81, +0x8181817f, 0x817f7f7f, 0x7f7f817f, 0x7f817f7f, 0x7f7f7f81, 0x7f817f7f, 0x7f7f8181, 0x817f817f, +0x7f7f8181, 0x7f7f7f81, 0x817f8181, 0x7f7f8181, 0x81817f81, 0x8181817f, 0x7f7f817f, 0x7f817f81, +0x7f817f81, 0x7f817f81, 0x7f817f81, 0x817f8181, 0x7f7f7f7f, 0x7f817f81, 0x817f7f81, 0x7f7f7f81, +0x7f7f817f, 0x7f817f7f, 0x7f7f7f7f, 0x817f7f81, 0x81817f7f, 0x81817f7f, 0x81818181, 0x7f81817f, +0x7f7f7f7f, 0x7f7f7f81, 0x7f817f81, 0x8181817f, 0x817f817f, 0x817f817f, 0x817f817f, 0x7f817f7f, +0x81818181, 0x7f817f7f, 0x7f7f817f, 0x81818181, 0x8181817f, 0x81817f7f, 0x81818181, 0x817f817f, +0x8181817f, 0x7f7f7f7f, 0x7f7f8181, 0x7f818181, 0x7f7f7f81, 0x817f817f, 0x7f7f817f, 0x81817f7f, +0x8181817f, 0x817f817f, 0x7f7f817f, 0x7f81817f, 0x7f7f7f81, 0x8181817f, 0x817f7f7f, 0x817f817f, +0x817f7f7f, 0x7f7f7f7f, 0x81817f81, 0x817f7f81, 0x7f817f7f, 0x81817f7f, 0x81817f81, 0x7f817f81, +0x817f7f7f, 0x817f817f, 0x7f817f7f, 0x817f7f81, 0x817f817f, 0x81817f81, 0x81818181, 0x7f817f7f, +0x8181817f, 0x817f7f7f, 0x81817f7f, 0x7f817f7f, 0x817f7f7f, 0x817f7f81, 0x7f7f7f81, 0x81817f7f, +0x817f7f81, 0x7f817f81, 0x7f7f8181, 0x7f81817f, 0x81817f7f, 0x817f7f7f, 0x81818181, 0x7f7f817f, +0x817f817f, 0x81817f81, 0x817f7f81, 0x81817f7f, 0x7f7f7f81, 0x7f817f7f, 0x81817f7f, 0x817f817f, +0x7f817f7f, 0x7f817f7f, 0x81817f7f, 0x817f7f7f, 0x81818181, 0x8181817f, 0x7f7f8181, 0x7f817f7f, +0x7f817f81, 0x7f7f8181, 0x81817f81, 0x817f7f81, 0x7f818181, 0x7f817f81, 0x7f7f7f81, 0x8181817f, +0x81817f7f, 0x817f7f7f, 0x7f818181, 0x7f818181, 0x7f81817f, 0x7f7f8181, 0x817f7f81, 0x7f817f81, +0x81817f7f, 0x81818181, 0x7f81817f, 0x7f7f8181, 0x817f817f, 0x817f7f81, 0x81817f81, 0x817f7f81, +0x81818181, 0x817f7f7f, 0x7f818181, 0x81818181, 0x7f7f817f, 0x817f817f, 0x7f7f817f, 0x7f817f7f, +0x7f817f81, 0x817f7f7f, 0x81817f81, 0x7f817f81, 0x7f817f7f, 0x817f8181, 0x817f817f, 0x7f818181, +0x817f7f81, 0x817f7f7f, 0x81817f81, 0x81818181, 0x7f81817f, 0x7f817f81, 0x7f817f81, 0x817f8181, +0x7f818181, 0x81817f81, 0x7f817f7f, 0x7f7f7f81, 0x7f7f817f, 0x817f7f81, 0x7f7f7f81, 0x81818181, +0x7f7f8181, 0x7f7f817f, 0x817f7f81, 0x81817f7f, 0x7f81817f, 0x7f81817f, 0x817f8181, 0x7f81817f, +0x8181817f, 0x81818181, 0x7f7f817f, 0x817f8181, 0x7f817f7f, 0x7f7f817f, 0x817f7f7f, 0x7f7f7f7f, +0x8181817f, 0x7f7f7f7f, 0x81817f81, 0x817f7f7f, 0x7f7f8181, 0x817f7f81, 0x7f7f817f, 0x7f81817f, +0x81817f81, 0x7f7f8181, 0x7f7f7f81, 0x7f817f81, 0x81817f7f, 0x7f817f81, 0x7f817f81, 0x7f7f817f, +0x81818181, 0x7f818181, 0x817f7f81, 0x817f7f7f, 0x7f81817f, 0x7f7f8181, 0x7f7f8181, 0x81817f7f, +0x7f817f81, 0x817f817f, 0x7f817f7f, 0x81818181, 0x7f817f7f, 0x81817f7f, 0x7f817f81, 0x81817f81, +0x7f7f7f81, 0x81818181, 0x81817f81, 0x817f7f81, 0x7f7f7f7f, 0x817f7f81, 0x7f818181, 0x7f7f8181, +0x7f7f7f81, 0x7f817f7f, 0x7f818181, 0x7f7f817f, 0x817f7f7f, 0x817f8181, 0x817f8181, 0x817f7f81, +0x7f7f8181, 0x7f817f7f, 0x817f8181, 0x81817f81, 0x817f8181, 0x7f81817f, 0x7f7f817f, 0x81817f7f, +0x81817f7f, 0x7f817f7f, 0x81817f7f, 0x81817f7f, 0x7f817f81, 0x7f7f8181, 0x7f7f817f, 0x7f7f7f7f, +0x81818181, 0x7f7f817f, 0x7f7f7f81, 0x7f817f81, 0x81818181, 0x81817f81, 0x817f7f7f, 0x7f7f817f, +0x817f7f7f, 0x7f7f7f81, 0x81817f81, 0x817f8181, 0x7f7f817f, 0x7f7f817f, 0x7f7f8181, 0x817f7f81, +0x8181817f, 0x8181817f, 0x7f7f7f81, 0x7f7f817f, 0x81817f81, 0x7f7f7f7f, 0x817f8181, 0x817f817f, +0x7f7f7f81, 0x7f817f7f, 0x7f81817f, 0x7f7f817f, 0x7f81817f, 0x817f7f81, 0x7f7f7f7f, 0x7f7f7f81, +0x81817f7f, 0x8181817f, 0x7f81817f, 0x7f7f7f81, 0x8181817f, 0x81817f7f, 0x7f818181, 0x7f817f81, +0x8181817f, 0x7f81817f, 0x817f7f7f, 0x817f7f7f, 0x8181817f, 0x817f817f, 0x81817f81, 0x7f7f8181, +0x7f7f7f7f, 0x7f81817f, 0x817f817f, 0x81817f81, 0x81817f81, 0x81817f7f, 0x81817f81, 0x81817f81, +0x7f7f817f, 0x7f7f7f7f, 0x817f7f7f, 0x7f818181, 0x7f817f7f, 0x8181817f, 0x7f7f8181, 0x817f7f7f, +0x7f7f7f81, 0x7f817f81, 0x7f7f7f7f, 0x7f817f81, 0x7f81817f, 0x7f817f7f, 0x81817f7f, 0x7f817f7f, +0x81817f81, 0x7f7f7f81, 0x7f81817f, 0x81817f81, 0x7f817f81, 0x817f8181, 0x7f817f81, 0x7f817f7f, +0x81818181, 0x817f817f, 0x81817f81, 0x7f7f817f, 0x7f7f8181, 0x81818181, 0x8181817f, 0x7f817f81, +0x81817f7f, 0x7f7f7f7f, 0x7f7f817f, 0x7f81817f, 0x81818181, 0x7f81817f, 0x817f7f7f, 0x7f817f81, +0x817f817f, 0x817f7f81, 0x81818181, 0x81817f7f, 0x817f7f81, 0x817f8181, 0x81817f7f, 0x81817f7f, +0x7f7f817f, 0x817f817f, 0x7f817f81, 0x7f817f81, 0x81818181, 0x817f817f, 0x7f7f7f81, 0x81817f7f, +0x817f817f, 0x7f7f7f7f, 0x7f7f8181, 0x817f7f7f, 0x817f7f81, 0x817f7f81, 0x817f8181, 0x8181817f, +0x81818181, 0x81817f7f, 0x81817f7f, 0x817f8181, 0x817f8181, 0x817f817f, 0x7f7f7f7f, 0x7f817f81, +0x8181817f, 0x7f818181, 0x7f818181, 0x8181817f, 0x7f7f8181, 0x817f8181, 0x7f817f81, 0x81817f7f, +0x7f7f7f81, 0x81817f81, 0x7f818181, 0x81817f7f, 0x7f7f817f, 0x7f818181, 0x81818181, 0x7f818181, +0x7f7f7f81, 0x81817f7f, 0x7f81817f, 0x7f7f8181, 0x7f81817f, 0x7f817f7f, 0x7f817f7f, 0x7f817f7f, +0x7f7f817f, 0x7f817f81, 0x7f81817f, 0x7f7f7f81, 0x81818181, 0x7f7f8181, 0x817f8181, 0x7f817f81, +0x7f817f81, 0x81817f81, 0x7f817f81, 0x817f7f81, 0x7f7f7f81, 0x817f7f7f, 0x7f81817f, 0x7f817f7f, +0x7f7f7f81, 0x817f8181, 0x7f81817f, 0x817f817f, 0x81817f81, 0x7f7f817f, 0x7f817f7f, 0x817f817f, +0x7f7f7f7f, 0x817f7f81, 0x817f8181, 0x7f7f7f7f, 0x7f817f81, 0x7f7f7f7f, 0x7f7f817f, 0x8181817f, +0x817f7f81, 0x7f818181, 0x7f817f7f, 0x8181817f, 0x7f7f817f, 0x7f817f7f, 0x817f817f, 0x817f7f7f, +0x817f817f, 0x81817f7f, 0x81818181, 0x7f7f7f81, 0x81817f7f, 0x817f7f81, 0x817f817f, 0x81817f81, +0x7f817f7f, 0x817f7f81, 0x7f7f7f81, 0x8181817f, 0x7f7f817f, 0x817f7f81, 0x817f7f81, 0x7f818181, +0x817f7f7f, 0x8181817f, 0x817f8181, 0x7f7f817f, 0x7f818181, 0x817f8181, 0x7f7f7f81, 0x817f8181, +0x8181817f, 0x7f817f81, 0x81817f7f, 0x817f8181, 0x817f817f, 0x817f7f81, 0x81818181, 0x81818181, +0x8181817f, 0x817f8181, 0x7f7f7f81, 0x7f7f8181, 0x817f817f, 0x7f818181, 0x817f8181, 0x7f7f817f, +0x7f7f7f81, 0x7f818181, 0x7f818181, 0x7f81817f, 0x81817f81, 0x7f818181, 0x7f7f8181, 0x7f817f7f, +0x8181817f, 0x7f817f7f, 0x817f7f81, 0x7f817f81, 0x8181817f, 0x81818181, 0x817f7f7f, 0x817f817f, +0x7f7f7f7f, 0x817f8181, 0x81818181, 0x817f7f7f, 0x817f7f81, 0x7f817f7f, 0x817f7f81, 0x81817f81, +0x7f818181, 0x7f817f81, 0x7f818181, 0x7f7f7f81, 0x81817f81, 0x7f817f81, 0x7f817f7f, 0x7f81817f, +0x81817f81, 0x817f7f7f, 0x817f817f, 0x7f7f817f, 0x7f7f8181, 0x817f7f7f, 0x7f7f817f, 0x7f817f7f, +0x7f81817f, 0x7f81817f, 0x817f7f7f, 0x7f817f7f, 0x817f7f81, 0x8181817f, 0x817f7f81, 0x7f818181, +0x7f818181, 0x817f7f81, 0x7f817f81, 0x7f7f7f7f, 0x81817f81, 0x81818181, 0x7f818181, 0x7f81817f, +0x81817f81, 0x817f7f7f, 0x7f7f817f, 0x7f7f817f, 0x81817f81, 0x7f817f81, 0x7f7f8181, 0x81817f7f, +0x817f817f, 0x81817f7f, 0x7f7f7f81, 0x7f817f7f, 0x7f7f7f7f, 0x817f7f81, 0x7f817f81, 0x817f7f7f, +0x7f7f8181, 0x7f7f817f, 0x8181817f, 0x7f818181, 0x7f817f81, 0x81818181, 0x817f817f, 0x7f7f7f7f, +0x81818181, 0x7f7f8181, 0x7f7f8181, 0x7f81817f, 0x8181817f, 0x8181817f, 0x81817f7f, 0x817f817f, +0x817f7f7f, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x817f8181, 0x81818181, 0x7f7f7f7f, 0x7f7f817f, +0x817f7f7f, 0x7f7f817f, 0x817f8181, 0x7f817f81, 0x7f818181, 0x7f81817f, 0x7f81817f, 0x81818181, +0x7f81817f, 0x817f7f7f, 0x7f7f8181, 0x81818181, 0x7f817f81, 0x81818181, 0x7f817f81, 0x81818181, +0x817f8181, 0x7f7f7f81, 0x7f818181, 0x7f7f817f, 0x817f7f7f, 0x7f7f7f7f, 0x817f7f81, 0x817f7f81, +0x7f81817f, 0x817f7f7f, 0x81817f81, 0x817f7f81, 0x81818181, 0x7f7f7f7f, 0x81817f7f, 0x81817f81, +0x81817f81, 0x81817f81, 0x7f817f81, 0x817f817f, 0x817f817f, 0x817f817f, 0x7f7f817f, 0x81817f81, +0x7f7f817f, 0x8181817f, 0x817f7f81, 0x81818181, 0x817f7f7f, 0x817f8181, 0x7f7f817f, 0x817f7f7f, +0x8181817f, 0x81817f7f, 0x7f817f7f, 0x817f7f81, 0x81817f81, 0x817f7f7f, 0x81818181, 0x817f7f7f, +0x81817f81, 0x7f81817f, 0x7f81817f, 0x7f818181, 0x817f7f81, 0x817f817f, 0x7f81817f, 0x81818181, +0x81817f81, 0x81818181, 0x817f7f81, 0x81817f7f, 0x7f81817f, 0x8181817f, 0x8181817f, 0x81818181, +0x81818181, 0x81818181, 0x7f7f8181, 0x817f7f7f, 0x7f7f7f7f, 0x81817f81, 0x817f7f81, 0x7f818181, +0x817f7f7f, 0x7f817f81, 0x7f7f7f7f, 0x7f7f8181, 0x7f81817f, 0x7f817f81, 0x817f7f7f, 0x81817f81, +0x7f7f7f7f, 0x817f7f81, 0x817f7f7f, 0x7f817f81, 0x81817f7f, 0x81817f81, 0x817f7f81, 0x81817f81, +0x817f817f, 0x7f817f81, 0x81817f7f, 0x7f7f7f81, 0x7f7f8181, 0x7f7f817f, 0x8181817f, 0x7f81817f, +0x817f8181, 0x7f7f7f81, 0x817f817f, 0x7f7f817f, 0x7f7f817f, 0x7f817f7f, 0x7f818181, 0x81817f7f, +0x81817f7f, 0x8181817f, 0x817f817f, 0x7f7f817f, 0x7f817f81, 0x7f817f7f, 0x7f818181, 0x7f81817f, +0x817f7f7f, 0x7f817f7f, 0x81817f81, 0x8181817f, 0x8181817f, 0x7f817f81, 0x8181817f, 0x8181817f, +0x81817f7f, 0x817f7f7f, 0x7f7f7f7f, 0x7f817f7f, 0x81817f81, 0x7f7f7f81, 0x817f7f7f, 0x7f817f81, +0x7f818181, 0x7f817f81, 0x81818181, 0x817f817f, 0x817f817f, 0x7f818181, 0x817f8181, 0x7f7f817f, +0x7f7f7f81, 0x817f7f7f, 0x7f81817f, 0x7f81817f, 0x7f81817f, 0x81817f81, 0x817f817f, 0x7f7f817f, +0x7f817f81, 0x817f817f, 0x7f817f7f, 0x7f81817f, 0x8181817f, 0x817f7f7f, 0x8181817f, 0x7f7f8181, +0x7f818181, 0x7f817f81, 0x7f7f817f, 0x817f8181, 0x817f817f, 0x817f7f81, 0x7f7f7f81, 0x81817f81, +0x817f8181, 0x81818181, 0x81818181, 0x7f7f817f, 0x7f7f7f81, 0x8181817f, 0x7f7f7f81, 0x7f7f7f81, +0x7f818181, 0x817f7f81, 0x7f817f7f, 0x81818181, 0x7f817f7f, 0x7f7f817f, 0x7f7f8181, 0x8181817f, +0x7f817f7f, 0x817f7f81, 0x7f7f7f81, 0x7f81817f, 0x81817f81, 0x7f817f81, 0x817f8181, 0x817f817f, +0x817f7f7f, 0x81817f7f, 0x7f818181, 0x81818181, 0x817f7f81, 0x81817f7f, 0x817f8181, 0x817f8181, +0x81817f7f, 0x8181817f, 0x7f7f817f, 0x7f7f817f, 0x7f7f817f, 0x7f817f81, 0x7f7f7f7f, 0x817f8181, +0x81817f7f, 0x7f818181, 0x7f7f7f81, 0x817f8181, 0x7f817f7f, 0x7f817f7f, 0x7f817f81, 0x817f8181, +0x817f817f, 0x7f7f7f7f, 0x81818181, 0x81818181, 0x81818181, 0x7f817f7f, 0x81818181, 0x7f81817f, +0x7f817f81, 0x8181817f, 0x817f7f7f, 0x817f817f, 0x81817f81, 0x7f817f81, 0x817f7f81, 0x81817f81, +0x7f7f8181, 0x7f817f7f, 0x817f7f7f, 0x7f81817f, 0x81818181, 0x7f7f817f, 0x7f7f7f81, 0x817f7f81, +0x817f7f7f, 0x7f817f81, 0x7f817f7f, 0x817f817f, 0x8181817f, 0x7f7f8181, 0x7f818181, 0x8181817f, +0x7f817f7f, 0x8181817f, 0x7f817f81, 0x7f81817f, 0x7f7f7f81, 0x7f7f7f81, 0x817f817f, 0x81818181, +0x7f817f81, 0x7f81817f, 0x7f7f8181, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f7f81, 0x7f818181, 0x7f7f7f7f, +0x7f7f8181, 0x817f8181, 0x7f7f7f7f, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, 0x7f7f7f7f, 0x7f7f8181, +0x817f7f7f, 0x81817f81, 0x7f818181, 0x81817f7f, 0x7f81817f, 0x7f817f81, 0x7f817f81, 0x7f817f81, +0x7f7f817f, 0x7f7f7f7f, 0x817f7f81, 0x7f7f8181, 0x8181817f, 0x817f7f7f, 0x7f817f7f, 0x817f8181, +0x7f817f7f, 0x817f8181, 0x7f818181, 0x81817f7f, 0x7f817f7f, 0x8181817f, 0x7f81817f, 0x7f7f7f7f, +0x7f81817f, 0x7f817f81, 0x817f817f, 0x817f7f7f, 0x7f7f7f7f, 0x7f818181, 0x817f7f7f, 0x7f81817f, +0x81817f81, 0x7f7f7f81, 0x7f818181, 0x7f817f81, 0x81817f81, 0x7f7f7f7f, 0x817f8181, 0x817f8181, +0x7f817f7f, 0x7f817f7f, 0x817f817f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x8181817f, 0x81817f81, +0x817f8181, 0x817f8181, 0x7f7f8181, 0x7f7f8181, 0x8181817f, 0x7f817f7f, 0x8181817f, 0x7f81817f, +0x7f7f7f81, 0x817f817f, 0x81817f7f, 0x817f7f81, 0x81817f7f, 0x817f7f81, 0x81817f81, 0x7f817f81, +0x817f8181, 0x7f818181, 0x7f817f81, 0x81817f81, 0x81817f81, 0x817f8181, 0x7f818181, 0x81818181, +0x817f7f81, 0x7f817f7f, 0x817f7f81, 0x817f817f, 0x8181817f, 0x81818181, 0x817f7f81, 0x7f818181, +0x81817f81, 0x7f817f7f, 0x817f817f, 0x817f7f81, 0x81817f81, 0x7f817f81, 0x7f7f817f, 0x8181817f, +0x817f7f7f, 0x7f7f817f, 0x7f817f81, 0x817f7f7f, 0x7f7f817f, 0x81818181, 0x817f7f81, 0x7f7f8181, +0x7f7f8181, 0x8181817f, 0x7f7f8181, 0x817f8181, 0x817f7f7f, 0x817f7f7f, 0x7f7f7f7f, 0x81817f81, +0x817f7f81, 0x7f7f8181, 0x7f817f81, 0x817f7f81, 0x7f7f817f, 0x817f8181, 0x7f7f7f81, 0x81817f7f, +0x817f817f, 0x7f7f7f7f, 0x817f7f7f, 0x7f81817f, 0x817f817f, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, +0x7f7f8181, 0x817f817f, 0x7f7f7f81, 0x7f7f817f, 0x817f8181, 0x7f81817f, 0x7f81817f, 0x817f7f7f, +0x7f817f7f, 0x817f817f, 0x817f7f81, 0x817f7f81, 0x81817f81, 0x7f817f81, 0x817f817f, 0x7f81817f, +0x7f7f8181, 0x7f818181, 0x7f818181, 0x817f817f, 0x7f81817f, 0x817f8181, 0x8181817f, 0x7f817f7f, +0x817f8181, 0x817f7f81, 0x81817f81, 0x7f7f7f81, 0x817f7f81, 0x7f7f7f7f, 0x81818181, 0x7f7f8181, +0x7f817f7f, 0x7f7f8181, 0x81817f7f, 0x817f817f, 0x81818181, 0x817f7f7f, 0x8181817f, 0x81817f81, +0x817f817f, 0x817f817f, 0x7f7f817f, 0x7f7f7f81, 0x7f818181, 0x7f817f7f, 0x7f7f7f81, 0x817f817f, +0x7f818181, 0x7f7f8181, 0x7f817f7f, 0x81817f81, 0x7f7f8181, 0x817f7f81, 0x817f7f7f, 0x817f8181, +0x7f7f8181, 0x7f817f7f, 0x817f7f81, 0x817f817f, 0x817f7f81, 0x81817f81, 0x817f8181, 0x817f7f7f, +0x7f7f7f81, 0x817f8181, 0x7f817f81, 0x7f817f81, 0x7f817f7f, 0x7f81817f, 0x7f7f7f7f, 0x81817f7f, +0x7f7f7f7f, 0x817f7f81, 0x7f7f8181, 0x7f7f7f81, 0x817f7f7f, 0x7f817f7f, 0x7f7f817f, 0x81817f7f, +0x8181817f, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x817f8181, 0x7f7f7f81, 0x7f817f81, 0x7f817f7f, +0x7f818181, 0x817f7f7f, 0x81817f81, 0x817f8181, 0x7f818181, 0x81817f7f, 0x817f8181, 0x7f7f8181, +0x81817f7f, 0x7f7f7f7f, 0x81818181, 0x7f818181, 0x817f817f, 0x7f7f7f81, 0x817f7f81, 0x817f8181, +0x7f81817f, 0x81817f7f, 0x817f8181, 0x817f817f, 0x7f818181, 0x7f7f7f7f, 0x7f818181, 0x7f7f817f, +0x7f817f81, 0x817f8181, 0x7f7f7f7f, 0x7f818181, 0x7f7f7f81, 0x7f7f817f, 0x7f817f81, 0x817f8181, +0x81818181, 0x7f818181, 0x7f7f7f7f, 0x7f818181, 0x7f81817f, 0x7f7f817f, 0x81817f81, 0x7f7f8181, +0x81818181, 0x817f7f81, 0x7f817f81, 0x7f7f8181, 0x817f7f81, 0x81817f81, 0x7f817f7f, 0x817f817f, +0x7f81817f, 0x7f818181, 0x817f8181, 0x81817f7f, 0x817f8181, 0x7f7f7f81, 0x7f818181, 0x7f7f817f, +0x817f8181, 0x817f8181, 0x7f817f81, 0x7f81817f, 0x8181817f, 0x817f7f81, 0x817f817f, 0x81818181, +0x7f817f7f, 0x817f817f, 0x817f817f, 0x7f817f7f, 0x7f817f7f, 0x817f817f, 0x7f81817f, 0x81818181, +0x7f7f7f7f, 0x7f7f7f7f, 0x817f7f81, 0x7f81817f, 0x7f7f7f81, 0x817f7f81, 0x817f8181, 0x7f818181, +0x7f7f7f7f, 0x7f7f817f, 0x7f7f7f81, 0x7f818181, 0x817f7f81, 0x81817f7f, 0x81817f7f, 0x81817f81, +0x7f7f7f81, 0x8181817f, 0x7f7f8181, 0x7f81817f, 0x7f7f7f81, 0x81817f7f, 0x7f7f7f7f, 0x7f7f817f, +0x7f817f7f, 0x81818181, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f7f7f, 0x817f817f, 0x81818181, 0x81817f81, +0x7f81817f, 0x7f7f8181, 0x7f81817f, 0x81818181, 0x81818181, 0x7f7f7f7f, 0x817f7f81, 0x7f81817f, +0x7f81817f, 0x7f7f8181, 0x7f818181, 0x7f818181, 0x7f817f7f, 0x8181817f, 0x81817f81, 0x817f7f81, +0x7f7f7f81, 0x7f81817f, 0x817f7f81, 0x817f8181, 0x7f81817f, 0x817f8181, 0x81817f7f, 0x7f817f81, +0x81818181, 0x7f817f81, 0x7f818181, 0x7f7f7f81, 0x7f7f8181, 0x8181817f, 0x81818181, 0x81818181, +0x81818181, 0x817f8181, 0x81817f81, 0x817f7f7f, 0x81817f81, 0x817f7f7f, 0x7f7f8181, 0x7f81817f, +0x7f7f7f81, 0x7f7f7f7f, 0x7f817f81, 0x81817f81, 0x817f817f, 0x7f7f7f81, 0x817f7f81, 0x817f7f7f, +0x7f817f7f, 0x81818181, 0x81818181, 0x7f7f817f, 0x7f817f81, 0x81817f81, 0x817f817f, 0x817f7f81, +0x81818181, 0x7f817f7f, 0x81817f7f, 0x81817f81, 0x817f817f, 0x817f7f7f, 0x7f7f817f, 0x7f7f817f, +0x7f7f8181, 0x817f7f81, 0x7f81817f, 0x7f817f7f, 0x817f7f7f, 0x817f7f7f, 0x817f7f7f, 0x7f7f7f7f, +0x7f7f7f7f, 0x7f7f7f81, 0x7f7f8181, 0x817f8181, 0x817f817f, 0x7f7f7f81, 0x81818181, 0x81817f81, +0x7f7f7f7f, 0x817f817f, 0x81818181, 0x7f7f817f, 0x7f818181, 0x7f7f8181, 0x817f8181, 0x817f7f7f, +0x7f7f8181, 0x7f817f7f, 0x817f8181, 0x817f8181, 0x81817f81, 0x7f817f81, 0x817f7f81, 0x7f7f7f81, +0x7f818181, 0x7f817f7f, 0x81817f81, 0x7f817f7f, 0x817f7f7f, 0x817f7f7f, 0x817f817f, 0x7f7f7f7f, +0x817f8181, 0x7f7f7f7f, 0x7f81817f, 0x7f817f81, 0x81818181, 0x817f817f, 0x817f7f7f, 0x81817f7f, +0x7f81817f, 0x817f817f, 0x7f818181, 0x8181817f, 0x817f817f, 0x81818181, 0x817f7f7f, 0x817f7f81, +0x7f7f8181, 0x817f817f, 0x817f7f81, 0x7f81817f, 0x7f818181, 0x7f7f817f, 0x817f7f7f, 0x7f7f7f7f, +0x7f7f7f7f, 0x81817f81, 0x7f81817f, 0x7f7f7f7f, 0x7f7f7f81, 0x8181817f, 0x81818181, 0x81817f7f, +0x7f81817f, 0x7f7f8181, 0x817f7f7f, 0x7f7f7f7f, 0x817f7f7f, 0x7f81817f, 0x817f817f, 0x81817f81, +0x7f7f7f81, 0x817f7f81, 0x817f7f81, 0x81818181, 0x7f7f8181, 0x81817f81, 0x7f817f7f, 0x7f818181, +0x7f818181, 0x81818181, 0x7f7f817f, 0x7f7f7f81, 0x81817f81, 0x817f7f7f, 0x817f8181, 0x7f7f817f, +0x81818181, 0x81817f81, 0x7f7f8181, 0x7f7f7f7f, 0x7f818181, 0x7f7f817f, 0x817f7f81, 0x817f7f7f, +0x81817f7f, 0x7f81817f, 0x7f817f7f, 0x7f81817f, 0x7f7f817f, 0x7f7f8181, 0x8181817f, 0x817f7f81, +0x7f7f7f81, 0x7f817f81, 0x8181817f, 0x7f7f7f7f, 0x7f817f81, 0x7f7f7f81, 0x817f7f7f, 0x81817f81, +0x7f817f7f, 0x817f7f81, 0x817f817f, 0x81817f7f, 0x7f818181, 0x7f81817f, 0x7f817f81, 0x7f817f81, +0x7f81817f, 0x7f7f7f7f, 0x7f817f81, 0x7f818181, 0x7f7f7f7f, 0x7f7f7f7f, 0x7f7f7f7f, 0x7f7f817f, +0x7f7f7f81, 0x7f818181, 0x7f818181, 0x7f818181, 0x7f7f8181, 0x817f7f81, 0x7f817f7f, 0x81818181, +0x7f81817f, 0x817f817f, 0x7f7f7f7f, 0x817f7f81, 0x7f7f817f, 0x7f7f817f, 0x7f7f8181, 0x7f7f7f81, +0x81817f81, 0x7f81817f, 0x7f7f7f81, 0x81817f81, 0x7f818181, 0x81817f81, 0x7f7f7f7f, 0x7f81817f, +0x81817f81, 0x7f817f7f, 0x7f81817f, 0x81817f81, 0x81818181, 0x7f81817f, 0x7f7f817f, 0x81817f7f, +0x7f7f817f, 0x8181817f, 0x81817f7f, 0x7f81817f, 0x81817f81, 0x8181817f, 0x7f817f81, 0x81817f7f, +0x817f7f7f, 0x7f7f8181, 0x817f8181, 0x7f81817f, 0x7f7f817f, 0x817f817f, 0x817f7f81, 0x7f7f817f, +0x7f7f8181, 0x7f818181, 0x7f7f8181, 0x81817f7f, 0x8181817f, 0x817f7f7f, 0x7f7f8181, 0x7f817f81, +0x817f817f, 0x7f817f7f, 0x81817f81, 0x7f7f817f, 0x81817f81, 0x7f7f817f, 0x81818181, 0x81817f81, +0x81817f7f, 0x7f7f7f7f, 0x7f7f7f7f, 0x817f817f, 0x7f7f8181, 0x7f7f7f7f, 0x7f817f7f, 0x7f7f817f, +0x7f81817f, 0x817f7f81, 0x817f7f81, 0x7f7f7f7f, 0x7f7f7f81, 0x81817f7f, 0x817f7f81, 0x7f7f817f, +0x7f817f7f, 0x81818181, 0x7f81817f, 0x7f817f7f, 0x81818181, 0x8181817f, 0x7f7f7f81, 0x7f818181, +0x8181817f, 0x7f7f7f7f, 0x817f817f, 0x7f818181, 0x81817f81, 0x7f818181, 0x7f81817f, 0x817f7f81, +0x81818181, 0x7f7f7f81, 0x81817f81, 0x7f7f8181, 0x7f81817f, 0x81817f7f, 0x8181817f, 0x7f817f7f, +0x7f7f8181, 0x7f7f817f, 0x7f817f7f, 0x817f817f, 0x7f818181, 0x7f7f7f7f, 0x7f7f7f7f, 0x7f818181, +0x81817f81, 0x7f818181, 0x81817f81, 0x7f818181, 0x7f7f7f81, 0x817f817f, 0x81817f7f, 0x7f7f7f81, +0x817f817f, 0x7f817f81, 0x7f817f81, 0x7f7f8181, 0x7f7f7f7f, 0x7f817f81, 0x8181817f, 0x817f8181, +0x7f7f7f7f, 0x8181817f, 0x817f8181, 0x81818181, 0x7f818181, 0x81818181, 0x7f7f7f81, 0x817f817f, +0x81817f81, 0x8181817f, 0x8181817f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x8181817f, 0x817f7f7f, +0x7f817f7f, 0x7f7f8181, 0x7f7f7f81, 0x7f817f81, 0x81817f81, 0x81817f81, 0x81817f7f, 0x8181817f, +0x817f817f, 0x7f817f7f, 0x817f7f7f, 0x7f7f7f81, 0x817f8181, 0x7f7f7f81, 0x7f817f81, 0x81817f7f, +0x817f817f, 0x817f8181, 0x7f7f7f81, 0x7f7f7f81, 0x81817f7f, 0x7f818181, 0x7f7f8181, 0x817f817f, +0x81818181, 0x7f818181, 0x817f7f81, 0x817f8181, 0x817f817f, 0x7f817f7f, 0x7f7f7f7f, 0x8181817f, +0x7f7f7f7f, 0x7f817f7f, 0x817f817f, 0x817f817f, 0x7f817f81, 0x81818181, 0x7f817f7f, 0x7f7f8181, +0x7f7f8181, 0x81817f7f, 0x81817f7f, 0x7f7f7f81, 0x817f7f7f, 0x7f7f817f, 0x81817f7f, 0x7f7f817f, +0x817f7f81, 0x8181817f, 0x81817f81, 0x7f7f8181, 0x7f7f8181, 0x7f7f7f81, 0x81817f7f, 0x7f7f817f, +0x81817f81, 0x7f817f81, 0x81817f7f, 0x7f7f7f7f, 0x7f81817f, 0x7f7f8181, 0x7f817f7f, 0x7f7f8181, +0x817f817f, 0x817f7f7f, 0x7f817f7f, 0x817f8181, 0x817f7f7f, 0x7f81817f, 0x7f7f817f, 0x817f817f, +0x7f7f8181, 0x7f7f7f7f, 0x817f7f81, 0x7f7f8181, 0x7f7f817f, 0x81817f81, 0x7f817f81, 0x7f817f7f, +0x817f7f81, 0x7f7f817f, 0x8181817f, 0x817f7f7f, 0x81817f81, 0x7f817f7f, 0x817f7f7f, 0x7f7f7f7f, +0x7f817f81, 0x7f7f7f7f, 0x81817f7f, 0x7f7f7f7f, 0x7f818181, 0x817f7f81, 0x817f817f, 0x8181817f, +0x817f7f7f, 0x817f8181, 0x8181817f, 0x7f817f7f, 0x7f817f7f, 0x817f7f81, 0x81817f7f, 0x7f7f8181, +0x81818181, 0x81818181, 0x7f81817f, 0x7f817f81, 0x7f817f81, 0x817f7f81, 0x8181817f, 0x7f81817f, +0x7f7f817f, 0x7f818181, 0x81817f7f, 0x7f817f81, 0x817f8181, 0x7f7f7f81, 0x7f7f8181, 0x817f7f7f, +0x7f818181, 0x7f7f8181, 0x817f8181, 0x7f817f81, 0x7f818181, 0x81817f81, 0x817f7f7f, 0x7f818181, +0x7f817f81, 0x817f7f81, 0x817f817f, 0x7f7f7f7f, 0x817f817f, 0x81818181, 0x7f817f7f, 0x7f817f81, +0x817f7f81, 0x7f7f817f, 0x81817f81, 0x7f81817f, 0x7f7f8181, 0x817f7f81, 0x817f817f, 0x81817f81, +0x817f8181, 0x7f7f7f7f, 0x817f7f7f, 0x7f81817f, 0x7f818181, 0x7f818181, 0x7f7f7f7f, 0x817f7f81, +0x817f7f81, 0x7f7f7f81, 0x8181817f, 0x7f7f7f7f, 0x817f817f, 0x7f817f7f, 0x817f817f, 0x7f7f7f7f, +0x8181817f, 0x7f7f7f7f, 0x7f7f8181, 0x7f817f7f, 0x7f817f7f, 0x7f817f7f, 0x817f8181, 0x7f81817f, +0x7f817f7f, 0x81817f81, 0x8181817f, 0x7f7f7f7f, 0x7f7f8181, 0x817f7f81, 0x7f817f7f, 0x817f7f7f, +0x81817f81, 0x817f7f7f, 0x81817f81, 0x7f81817f, 0x7f818181, 0x7f818181, 0x7f818181, 0x7f81817f, +0x7f7f8181, 0x817f7f81, 0x817f8181, 0x7f817f81, 0x7f817f7f, 0x81818181, 0x7f817f81, 0x81818181, +0x81817f7f, 0x81818181, 0x817f7f81, 0x81817f81, 0x81817f81, 0x7f7f7f7f, 0x81817f7f, 0x7f81817f, +0x7f7f8181, 0x81818181, 0x7f7f7f7f, 0x7f7f8181, 0x81817f7f, 0x81817f81, 0x7f7f7f81, 0x8181817f, +0x8181817f, 0x7f7f817f, 0x81817f81, 0x817f817f, 0x817f7f7f, 0x817f817f, 0x817f817f, 0x7f81817f, +0x8181817f, 0x817f7f7f, 0x7f818181, 0x817f7f7f, 0x817f8181, 0x7f7f8181, 0x817f8181, 0x817f817f, +0x817f7f81, 0x7f7f7f7f, 0x7f7f7f81, 0x8181817f, 0x7f818181, 0x7f7f8181, 0x7f81817f, 0x817f8181, +0x7f81817f, 0x7f7f8181, 0x8181817f, 0x817f7f7f, 0x817f817f, 0x8181817f, 0x8181817f, 0x81817f81, +0x7f7f817f, 0x7f817f7f, 0x817f817f, 0x7f81817f, 0x7f817f7f, 0x81817f7f, 0x7f7f8181, 0x817f7f7f, +0x7f7f7f81, 0x817f8181, 0x7f817f81, 0x817f7f81, 0x817f7f81, 0x81818181, 0x81818181, 0x81818181, +0x7f7f8181, 0x7f81817f, 0x7f817f7f, 0x7f817f7f, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x7f817f81, +0x7f817f81, 0x817f8181, 0x7f81817f, 0x81817f81, 0x7f7f7f81, 0x817f7f81, 0x817f7f7f, 0x817f8181, +0x7f7f8181, 0x7f817f7f, 0x7f818181, 0x7f7f817f, 0x7f81817f, 0x8181817f, 0x8181817f, 0x7f81817f, +0x7f817f81, 0x7f7f817f, 0x817f817f, 0x7f7f7f81, 0x817f8181, 0x7f81817f, 0x817f7f81, 0x817f817f, +0x817f817f, 0x7f817f81, 0x7f817f7f, 0x7f81817f, 0x817f817f, 0x7f7f8181, 0x7f81817f, 0x817f817f, +0x8181817f, 0x81818181, 0x81817f81, 0x817f8181, 0x81818181, 0x81817f7f, 0x817f8181, 0x7f7f8181, +0x7f7f8181, 0x7f7f817f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x81817f7f, 0x7f7f7f7f, 0x817f817f, +0x817f7f7f, 0x7f7f817f, 0x7f817f81, 0x7f7f817f, 0x7f818181, 0x8181817f, 0x7f7f8181, 0x7f817f7f, +0x7f81817f, 0x7f7f8181, 0x7f817f81, 0x817f7f7f, 0x7f7f817f, 0x7f818181, 0x7f7f817f, 0x81818181, +0x7f81817f, 0x817f8181, 0x81818181, 0x81817f81, 0x817f817f, 0x81817f7f, 0x7f7f8181, 0x7f7f7f81, +0x81817f7f, 0x7f817f81, 0x7f817f7f, 0x7f7f7f7f, 0x7f7f8181, 0x817f7f7f, 0x7f81817f, 0x7f818181, +0x7f818181, 0x817f7f7f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x817f7f81, 0x7f7f7f81, 0x817f7f81, +0x7f817f7f, 0x7f7f7f7f, 0x7f818181, 0x7f817f81, 0x8181817f, 0x817f7f7f, 0x7f817f81, 0x81818181, +0x7f7f817f, 0x81817f81, 0x8181817f, 0x7f7f7f81, 0x81817f7f, 0x7f817f81, 0x81818181, 0x817f817f, +0x7f7f7f81, 0x7f81817f, 0x7f7f817f, 0x817f7f81, 0x7f7f7f7f, 0x817f8181, 0x81817f81, 0x8181817f, +0x7f7f7f7f, 0x81817f81, 0x7f7f7f7f, 0x7f7f7f7f, 0x81817f7f, 0x7f817f81, 0x7f81817f, 0x81817f81, +0x817f817f, 0x81817f81, 0x81817f7f, 0x817f7f7f, 0x7f7f7f7f, 0x7f7f817f, 0x7f818181, 0x817f817f, +0x7f7f7f7f, 0x7f817f7f, 0x8181817f, 0x817f7f7f, 0x7f7f817f, 0x7f81817f, 0x7f7f7f81, 0x817f7f7f, +0x7f817f81, 0x7f7f8181, 0x8181817f, 0x7f818181, 0x7f817f7f, 0x817f817f, 0x7f7f7f81, 0x817f817f, +0x7f7f817f, 0x7f7f8181, 0x7f7f7f7f, 0x7f7f7f7f, 0x817f7f7f, 0x817f7f81, 0x7f817f7f, 0x7f81817f, +0x7f7f817f, 0x7f817f7f, 0x7f818181, 0x7f7f817f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x7f817f81, +0x7f817f7f, 0x817f817f, 0x7f7f7f7f, 0x81817f81, 0x7f817f81, 0x7f817f7f, 0x81818181, 0x817f7f7f, +0x817f7f81, 0x817f7f7f, 0x817f7f7f, 0x8181817f, 0x7f7f7f81, 0x7f7f8181, 0x817f7f81, 0x817f7f81, +0x7f817f7f, 0x817f7f7f, 0x817f817f, 0x8181817f, 0x7f818181, 0x7f7f817f, 0x7f817f81, 0x7f7f817f, +0x7f817f7f, 0x7f817f81, 0x8181817f, 0x81818181, 0x7f7f7f7f, 0x7f81817f, 0x81817f7f, 0x7f817f7f, +0x817f8181, 0x7f818181, 0x7f7f7f7f, 0x817f7f81, 0x7f7f8181, 0x7f818181, 0x81818181, 0x7f7f7f7f, +0x8181817f, 0x81817f81, 0x817f7f81, 0x81818181, 0x81817f7f, 0x8181817f, 0x7f817f7f, 0x7f7f7f7f, +0x817f7f81, 0x817f7f81, 0x817f7f7f, 0x7f7f7f81, 0x7f7f817f, 0x81817f7f, 0x7f7f8181, 0x7f7f817f, +0x81818181, 0x7f7f7f81, 0x7f7f817f, 0x7f7f8181, 0x8181817f, 0x7f818181, 0x817f817f, 0x7f7f7f81, +0x817f7f7f, 0x7f817f7f, 0x817f7f81, 0x8181817f, 0x7f81817f, 0x817f817f, 0x8181817f, 0x7f7f7f7f, +0x817f7f7f, 0x7f818181, 0x817f7f81, 0x7f7f8181, 0x817f7f7f, 0x8181817f, 0x817f8181, 0x8181817f, +0x81817f81, 0x817f8181, 0x817f7f81, 0x7f7f8181, 0x817f8181, 0x7f817f81, 0x7f7f7f81, 0x7f818181, +0x7f817f7f, 0x7f7f8181, 0x7f7f817f, 0x81817f81, 0x81817f7f, 0x81817f7f, 0x81817f7f, 0x817f817f, +0x7f81817f, 0x7f7f7f7f, 0x817f8181, 0x8181817f, 0x7f817f81, 0x7f817f7f, 0x817f7f7f, 0x7f7f8181, +0x7f7f8181, 0x7f7f7f81, 0x7f818181, 0x81817f7f, 0x8181817f, 0x817f7f81, 0x7f7f7f7f, 0x7f81817f, +0x817f8181, 0x7f81817f, 0x8181817f, 0x81818181, 0x7f817f81, 0x81817f81, 0x7f7f7f81, 0x817f7f7f, +0x81818181, 0x7f7f7f7f, 0x81818181, 0x81818181, 0x7f7f8181, 0x7f817f81, 0x81818181, 0x817f8181, +0x81817f81, 0x7f81817f, 0x8181817f, 0x7f7f7f81, 0x8181817f, 0x81818181, 0x81817f81, 0x81817f81, +0x7f7f8181, 0x7f7f7f81, 0x7f7f7f7f, 0x8181817f, 0x7f7f8181, 0x7f818181, 0x81818181, 0x7f7f7f7f, +0x81817f7f, 0x817f7f81, 0x7f7f817f, 0x817f8181, 0x817f8181, 0x7f7f8181, 0x817f7f7f, 0x7f7f7f7f, +0x7f7f7f81, 0x817f7f81, 0x7f81817f, 0x817f7f81, 0x817f817f, 0x7f7f7f7f, 0x81817f7f, 0x81817f81, +0x81818181, 0x7f817f7f, 0x81817f81, 0x7f7f7f81, 0x817f7f7f, 0x81818181, 0x7f7f817f, 0x7f818181, +0x7f7f7f7f, 0x81818181, 0x7f7f7f81, 0x81818181, 0x8181817f, 0x7f817f81, 0x7f7f7f7f, 0x7f817f81, +0x7f7f7f7f, 0x7f7f7f81, 0x81817f7f, 0x8181817f, 0x7f818181, 0x7f817f7f, 0x7f7f8181, 0x7f7f817f, +0x7f7f7f81, 0x8181817f, 0x81817f7f, 0x7f81817f, 0x7f817f81, 0x817f7f81, 0x7f7f817f, 0x81817f81, +0x817f7f81, 0x7f7f8181, 0x7f7f7f7f, 0x7f817f7f, 0x7f7f817f, 0x817f7f81, 0x817f8181, 0x7f7f7f7f, +0x7f818181, 0x81817f7f, 0x7f817f7f, 0x817f817f, 0x7f7f8181, 0x7f7f817f, 0x817f7f81, 0x7f7f8181, +0x817f7f81, 0x81817f81, 0x817f8181, 0x817f817f, 0x81818181, 0x7f817f81, 0x81817f81, 0x7f7f8181, +0x7f7f7f81, 0x817f7f81, 0x7f7f7f81, 0x7f7f7f7f, 0x81818181, 0x7f7f7f7f, 0x7f817f7f, 0x7f7f7f7f, +0x81818181, 0x8181817f, 0x7f7f7f7f, 0x7f7f817f, 0x817f8181, 0x817f817f, 0x817f7f7f, 0x817f8181, +0x7f7f8181, 0x7f7f8181, 0x817f817f, 0x7f817f81, 0x817f817f, 0x7f7f7f81, 0x817f7f7f, 0x7f7f817f, +0x7f7f7f7f, 0x817f8181, 0x817f8181, 0x7f7f817f, 0x81817f81, 0x7f7f8181, 0x7f81817f, 0x81817f7f, +0x817f7f7f, 0x817f817f, 0x7f7f8181, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x7f817f81, 0x7f81817f, +0x81817f7f, 0x817f7f7f, 0x7f817f81, 0x81818181, 0x7f7f817f, 0x817f7f81, 0x7f7f817f, 0x7f817f81, +0x81817f81, 0x81818181, 0x7f7f8181, 0x817f7f7f, 0x7f7f7f81, 0x81817f7f, 0x81818181, 0x817f8181, +0x7f7f817f, 0x7f817f7f, 0x7f817f81, 0x817f7f81, 0x7f7f817f, 0x7f818181, 0x7f817f81, 0x7f817f81, +0x7f818181, 0x81817f7f, 0x817f7f81, 0x8181817f, 0x7f7f7f81, 0x81817f7f, 0x7f817f81, 0x8181817f, +0x7f7f817f, 0x817f817f, 0x81817f81, 0x81817f81, 0x81817f7f, 0x7f818181, 0x817f7f81, 0x817f7f81, +0x7f7f7f7f, 0x7f81817f, 0x7f817f81, 0x7f7f817f, 0x7f818181, 0x8181817f, 0x81817f81, 0x7f817f81, +0x7f81817f, 0x817f7f81, 0x7f818181, 0x817f817f, 0x817f7f7f, 0x81817f81, 0x7f817f81, 0x7f817f7f, +0x81817f7f, 0x7f7f8181, 0x81817f81, 0x81817f7f, 0x8181817f, 0x7f7f817f, 0x8181817f, 0x7f818181, +0x7f817f7f, 0x817f7f7f, 0x81817f81, 0x7f81817f, 0x81818181, 0x7f7f7f81, 0x7f7f817f, 0x7f7f7f81, +0x817f7f81, 0x817f7f7f, 0x7f817f7f, 0x7f7f8181, 0x7f817f81, 0x7f817f81, 0x7f81817f, 0x81818181, +0x7f7f7f81, 0x7f7f7f81, 0x81818181, 0x817f8181, 0x7f7f8181, 0x817f8181, 0x817f7f81, 0x8181817f, +0x81818181, 0x8181817f, 0x8181817f, 0x817f7f81, 0x7f7f8181, 0x817f7f81, 0x7f817f7f, 0x817f7f7f, +0x81818181, 0x81818181, 0x81817f7f, 0x7f81817f, 0x7f817f81, 0x81817f81, 0x7f7f7f7f, 0x817f817f, +0x7f7f7f7f, 0x7f817f7f, 0x817f8181, 0x7f7f7f81, 0x817f7f7f, 0x7f817f81, 0x7f7f817f, 0x8181817f, +0x817f7f81, 0x7f81817f, 0x817f817f, 0x7f817f7f, 0x81817f81, 0x7f7f817f, 0x817f8181, 0x817f7f7f, +0x81818181, 0x817f7f81, 0x7f81817f, 0x817f817f, 0x817f817f, 0x817f7f7f, 0x7f7f817f, 0x8181817f, +0x7f7f7f7f, 0x81817f81, 0x8181817f, 0x7f81817f, 0x817f7f81, 0x7f7f7f7f, 0x817f7f7f, 0x817f817f, +0x81818181, 0x8181817f, 0x81817f81, 0x81817f7f, 0x7f7f817f, 0x81817f81, 0x7f7f817f, 0x8181817f, +0x7f818181, 0x7f7f817f, 0x8181817f, 0x817f7f7f, 0x81817f7f, 0x7f818181, 0x81818181, 0x8181817f, +0x817f817f, 0x7f7f7f7f, 0x7f7f817f, 0x7f7f8181, 0x7f7f7f81, 0x7f7f817f, 0x7f818181, 0x81818181, +0x81818181, 0x7f817f81, 0x7f7f817f, 0x7f817f7f, 0x81817f7f, 0x7f7f817f, 0x8181817f, 0x7f7f7f7f, +0x7f7f8181, 0x7f7f817f, 0x7f7f7f7f, 0x817f817f, 0x7f7f7f7f, 0x817f7f81, 0x817f8181, 0x81818181, +0x7f818181, 0x7f818181, 0x8181817f, 0x7f7f7f7f, 0x8181817f, 0x7f817f7f, 0x7f7f817f, 0x81818181, +0x7f817f7f, 0x817f8181, 0x817f7f81, 0x7f817f81, 0x817f7f81, 0x7f7f7f7f, 0x817f8181, 0x7f7f8181, +0x81817f81, 0x81818181, 0x7f7f817f, 0x817f7f81, 0x7f81817f, 0x8181817f, 0x81818181, 0x81817f7f, +0x7f81817f, 0x81818181, 0x7f7f8181, 0x7f817f81, 0x7f817f81, 0x7f817f7f, 0x817f817f, 0x7f818181, +0x7f817f7f, 0x7f7f7f7f, 0x817f817f, 0x817f7f7f, 0x7f81817f, 0x817f7f81, 0x81818181, 0x7f7f817f, +0x817f817f, 0x81817f81, 0x817f7f81, 0x81817f7f, 0x7f81817f, 0x7f81817f, 0x7f817f81, 0x81817f81, +0x7f7f817f, 0x8181817f, 0x7f818181, 0x7f7f7f7f, 0x81818181, 0x8181817f, 0x7f817f81, 0x8181817f, +0x7f81817f, 0x7f7f8181, 0x817f817f, 0x817f7f81, 0x7f817f81, 0x81817f7f, 0x7f818181, 0x81818181, +0x81817f81, 0x817f7f81, 0x7f81817f, 0x817f817f, 0x81817f81, 0x7f7f7f81, 0x7f818181, 0x81817f81, +0x817f7f81, 0x7f7f817f, 0x817f8181, 0x7f817f7f, 0x817f817f, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f817f, +0x817f817f, 0x817f817f, 0x817f8181, 0x7f7f7f81, 0x817f7f81, 0x817f8181, 0x7f7f7f81, 0x8181817f, +0x81817f7f, 0x7f7f7f81, 0x81817f7f, 0x817f7f81, 0x817f817f, 0x81817f7f, 0x7f7f7f7f, 0x7f81817f, +0x7f817f7f, 0x81818181, 0x81818181, 0x7f7f7f81, 0x81817f7f, 0x8181817f, 0x7f818181, 0x7f817f7f, +0x81817f81, 0x817f7f7f, 0x7f7f8181, 0x7f817f7f, 0x81817f7f, 0x8181817f, 0x817f8181, 0x817f7f81, +0x7f7f7f7f, 0x81817f7f, 0x81817f81, 0x81818181, 0x81817f7f, 0x817f817f, 0x817f817f, 0x7f7f817f, +0x7f81817f, 0x81818181, 0x7f817f81, 0x7f7f8181, 0x7f7f817f, 0x7f7f7f81, 0x81817f7f, 0x817f8181, +0x7f817f7f, 0x8181817f, 0x817f8181, 0x817f7f7f, 0x7f7f7f81, 0x8181817f, 0x817f7f81, 0x7f817f81, +0x7f7f8181, 0x7f81817f, 0x7f7f7f7f, 0x7f7f8181, 0x7f817f7f, 0x7f7f7f81, 0x7f817f7f, 0x8181817f, +0x7f81817f, 0x817f817f, 0x7f817f81, 0x8181817f, 0x7f818181, 0x81817f81, 0x7f818181, 0x81818181, +0x7f7f7f81, 0x817f7f7f, 0x817f8181, 0x7f817f7f, 0x7f817f81, 0x817f7f81, 0x81818181, 0x7f7f7f81, +0x7f7f7f81, 0x7f7f7f81, 0x7f81817f, 0x817f817f, 0x7f7f8181, 0x7f817f7f, 0x7f817f81, 0x7f818181, +0x7f7f8181, 0x817f8181, 0x7f817f7f, 0x7f7f8181, 0x7f7f7f81, 0x817f8181, 0x7f817f7f, 0x7f817f7f, +0x7f7f817f, 0x7f7f7f81, 0x817f7f7f, 0x7f817f7f, 0x817f817f, 0x7f817f81, 0x7f7f8181, 0x81817f81, +0x7f7f8181, 0x7f818181, 0x817f7f81, 0x7f818181, 0x81817f7f, 0x817f7f7f, 0x7f7f8181, 0x81817f7f, +0x7f817f7f, 0x7f818181, 0x7f81817f, 0x81818181, 0x7f7f817f, 0x7f818181, 0x7f817f7f, 0x81817f81, +0x7f7f7f7f, 0x7f817f7f, 0x7f7f817f, 0x7f7f7f81, 0x7f7f7f7f, 0x8181817f, 0x7f817f81, 0x7f81817f, +0x7f7f7f81, 0x7f7f7f81, 0x817f817f, 0x7f7f7f81, 0x81818181, 0x7f7f7f81, 0x8181817f, 0x817f7f81, +0x7f7f817f, 0x817f817f, 0x817f8181, 0x7f7f8181, 0x81817f7f, 0x7f817f81, 0x8181817f, 0x7f81817f, +0x7f818181, 0x7f818181, 0x7f817f81, 0x7f7f7f7f, 0x7f81817f, 0x817f7f81, 0x7f7f7f81, 0x81818181, +0x7f7f7f81, 0x817f8181, 0x7f81817f, 0x8181817f, 0x7f7f7f81, 0x8181817f, 0x7f7f7f81, 0x7f81817f, +0x81817f81, 0x7f817f81, 0x81817f7f, 0x81818181, 0x817f817f, 0x817f7f7f, 0x817f8181, 0x81818181, +0x7f81817f, 0x81818181, 0x81818181, 0x7f818181, 0x7f7f7f7f, 0x81817f81, 0x8181817f, 0x7f7f817f, +0x817f7f81, 0x817f7f81, 0x817f8181, 0x7f7f8181, 0x817f8181, 0x7f7f7f81, 0x817f7f81, 0x81818181, +0x8181817f, 0x81817f81, 0x7f7f8181, 0x7f7f8181, 0x7f7f8181, 0x817f817f, 0x7f817f7f, 0x7f81817f, +0x81817f7f, 0x81818181, 0x7f81817f, 0x7f817f7f, 0x7f7f7f81, 0x817f7f81, 0x81818181, 0x7f7f7f81, +0x7f7f8181, 0x7f81817f, 0x7f7f7f7f, 0x7f7f8181, 0x81817f81, 0x7f81817f, 0x81817f81, 0x7f7f8181, +0x817f817f, 0x7f817f81, 0x7f7f8181, 0x817f817f, 0x7f817f7f, 0x817f7f81, 0x7f7f7f81, 0x7f7f817f, +0x817f8181, 0x7f7f817f, 0x8181817f, 0x81818181, 0x81817f7f, 0x7f817f7f, 0x7f7f817f, 0x81817f7f, +0x7f7f817f, 0x7f81817f, 0x81818181, 0x81817f81, 0x8181817f, 0x7f817f7f, 0x81818181, 0x7f7f7f81, +0x7f7f7f7f, 0x81817f7f, 0x81817f81, 0x7f7f817f, 0x7f7f8181, 0x8181817f, 0x81818181, 0x7f7f7f81, +0x7f817f7f, 0x7f817f7f, 0x817f7f81, 0x7f7f817f, 0x8181817f, 0x817f7f7f, 0x7f817f81, 0x817f8181, +0x7f817f7f, 0x7f817f7f, 0x7f7f817f, 0x7f7f817f, 0x7f817f7f, 0x7f817f7f, 0x7f818181, 0x817f7f81, +0x7f81817f, 0x7f817f81, 0x7f817f81, 0x817f7f81, 0x7f7f7f7f, 0x81817f7f, 0x7f81817f, 0x81818181, +0x7f7f8181, 0x7f7f7f81, 0x7f817f81, 0x81817f81, 0x81817f81, 0x8181817f, 0x7f818181, 0x81817f81, +0x817f7f81, 0x817f7f7f, 0x7f7f7f81, 0x7f7f7f7f, 0x7f7f8181, 0x7f81817f, 0x7f7f817f, 0x7f7f817f, +0x817f7f81, 0x7f7f7f7f, 0x81817f81, 0x817f817f, 0x817f817f, 0x817f817f, 0x7f81817f, 0x7f7f8181, +0x7f818181, 0x817f8181, 0x7f7f8181, 0x7f817f81, 0x81818181, 0x7f817f81, 0x817f7f7f, 0x7f817f7f, +0x7f7f8181, 0x817f8181, 0x7f818181, 0x81817f81, 0x7f818181, 0x8181817f, 0x8181817f, 0x7f818181, +0x8181817f, 0x7f7f7f7f, 0x7f7f817f, 0x7f818181, 0x7f7f7f81, 0x7f7f817f, 0x81818181, 0x817f7f7f, +0x8181817f, 0x817f817f, 0x7f7f8181, 0x7f7f7f81, 0x81818181, 0x8181817f, 0x817f817f, 0x817f817f, +0x7f818181, 0x7f7f8181, 0x8181817f, 0x817f817f, 0x81817f7f, 0x81818181, 0x7f81817f, 0x817f817f, +0x7f7f7f7f, 0x7f817f81, 0x7f818181, 0x817f7f7f, 0x817f7f81, 0x7f817f81, 0x7f7f817f, 0x7f7f7f7f, +0x81817f81, 0x7f7f7f7f, 0x817f8181, 0x7f7f817f, 0x8181817f, 0x7f81817f, 0x7f818181, 0x7f7f7f7f, +0x7f7f7f7f, 0x7f7f7f81, 0x7f818181, 0x7f818181, 0x8181817f, 0x7f7f8181, 0x817f7f81, 0x7f817f7f, +0x817f7f7f, 0x7f7f7f81, 0x7f7f8181, 0x7f817f81, 0x817f7f81, 0x7f7f7f81, 0x7f817f7f, 0x7f7f7f81, +0x817f817f, 0x817f7f7f, 0x81818181, 0x7f818181, 0x81818181, 0x81818181, 0x7f7f7f81, 0x8181817f, +0x81818181, 0x7f817f7f, 0x817f7f81, 0x7f7f8181, 0x7f818181, 0x8181817f, 0x8181817f, 0x817f7f81, +0x7f817f7f, 0x81817f7f, 0x817f8181, 0x7f817f7f, 0x7f81817f, 0x81817f7f, 0x7f817f7f, 0x817f7f7f, +0x817f7f7f, 0x7f817f81, 0x8181817f, 0x817f7f7f, 0x7f817f81, 0x81817f81, 0x817f817f, 0x7f7f7f81, +0x7f817f81, 0x7f817f7f, 0x817f817f, 0x817f7f81, 0x7f7f7f7f, 0x81817f81, 0x7f818181, 0x81817f81, +0x817f8181, 0x7f817f7f, 0x81817f81, 0x817f7f81, 0x7f7f7f7f, 0x81817f81, 0x81817f81, 0x81817f81, +0x81817f7f, 0x7f81817f, 0x81818181, 0x7f7f8181, 0x817f817f, 0x817f8181, 0x817f7f7f, 0x8181817f, +0x81817f7f, 0x8181817f, 0x7f817f81, 0x81817f7f, 0x81818181, 0x817f7f7f, 0x7f81817f, 0x817f817f, +0x81818181, 0x817f7f81, 0x817f817f, 0x817f7f7f, 0x81817f7f, 0x7f7f8181, 0x817f7f81, 0x8181817f, +0x7f818181, 0x7f817f81, 0x8181817f, 0x7f7f7f81, 0x8181817f, 0x81818181, 0x7f7f8181, 0x7f7f7f7f, +0x7f81817f, 0x817f8181, 0x817f817f, 0x7f81817f, 0x81818181, 0x7f817f7f, 0x7f7f7f7f, 0x7f7f7f7f, +0x7f7f8181, 0x817f8181, 0x817f817f, 0x7f7f7f81, 0x817f7f7f, 0x7f7f817f, 0x7f817f7f, 0x817f8181, +0x81817f81, 0x817f8181, 0x7f81817f, 0x817f8181, 0x817f8181, 0x817f7f81, 0x7f7f817f, 0x7f818181, +0x7f7f817f, 0x7f7f7f7f, 0x817f8181, 0x7f7f7f7f, 0x8181817f, 0x817f8181, 0x81818181, 0x817f7f7f, +0x7f7f817f, 0x817f7f7f, 0x817f817f, 0x7f818181, 0x8181817f, 0x817f817f, 0x7f7f7f7f, 0x7f81817f, +0x7f817f81, 0x81818181, 0x7f817f7f, 0x817f817f, 0x7f7f7f7f, 0x817f7f81, 0x7f7f8181, 0x81817f81, +0x81818181, 0x81817f81, 0x7f7f817f, 0x81818181, 0x817f8181, 0x7f81817f, 0x81817f7f, 0x7f81817f, +0x817f8181, 0x81818181, 0x7f7f7f81, 0x81817f81, 0x817f7f7f, 0x817f8181, 0x7f7f817f, 0x7f7f8181, +0x7f81817f, 0x7f818181, 0x7f81817f, 0x7f7f817f, 0x7f81817f, 0x7f7f7f81, 0x7f81817f, 0x817f8181, +0x81817f81, 0x81818181, 0x817f8181, 0x81818181, 0x7f81817f, 0x7f7f8181, 0x7f81817f, 0x81817f81, +0x7f818181, 0x81818181, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, 0x81817f7f, 0x7f81817f, 0x81818181, +0x81817f81, 0x81817f7f, 0x7f7f817f, 0x817f7f7f, 0x8181817f, 0x81817f81, 0x81817f81, 0x8181817f, +0x817f817f, 0x81817f7f, 0x7f817f7f, 0x817f817f, 0x817f7f7f, 0x8181817f, 0x7f7f7f7f, 0x817f817f, +0x817f8181, 0x7f817f7f, 0x7f817f7f, 0x7f81817f, 0x7f7f817f, 0x7f817f7f, 0x7f7f8181, 0x7f7f7f7f, +0x7f81817f, 0x817f817f, 0x7f7f817f, 0x7f7f8181, 0x7f7f7f7f, 0x817f7f7f, 0x7f81817f, 0x81818181, +0x7f7f817f, 0x7f81817f, 0x7f7f7f7f, 0x7f7f8181, 0x7f7f8181, 0x81818181, 0x817f7f7f, 0x81818181, +0x81817f81, 0x817f817f, 0x7f817f81, 0x7f7f8181, 0x7f7f7f81, 0x81817f81, 0x7f7f7f81, 0x817f7f7f, +0x817f817f, 0x817f8181, 0x7f817f81, 0x817f7f7f, 0x7f817f7f, 0x7f7f7f81, 0x81817f7f, 0x7f7f7f81, +0x7f817f7f, 0x7f81817f, 0x817f817f, 0x7f7f8181, 0x7f81817f, 0x8181817f, 0x7f7f7f7f, 0x7f7f817f, +0x7f7f7f7f, 0x7f817f7f, 0x7f7f817f, 0x7f817f81, 0x817f817f, 0x81817f7f, 0x7f81817f, 0x8181817f, +0x817f7f81, 0x7f7f7f81, 0x81818181, 0x7f817f81, 0x81818181, 0x7f818181, 0x817f7f7f, 0x81818181, +0x81818181, 0x7f7f7f7f, 0x8181817f, 0x7f7f7f81, 0x81818181, 0x7f7f817f, 0x817f7f81, 0x7f81817f, +0x817f817f, 0x7f7f7f81, 0x7f81817f, 0x7f81817f, 0x7f818181, 0x817f817f, 0x81817f7f, 0x7f7f817f, +0x7f818181, 0x7f817f7f, 0x7f7f8181, 0x817f817f, 0x81817f7f, 0x7f7f7f81, 0x817f817f, 0x7f7f817f, +0x81817f7f, 0x7f7f7f7f, 0x7f81817f, 0x817f7f7f, 0x7f7f8181, 0x7f7f8181, 0x7f7f817f, 0x7f81817f, +0x81817f81, 0x817f7f81, 0x7f81817f, 0x81817f7f, 0x817f8181, 0x817f8181, 0x817f7f7f, 0x7f7f7f7f, +0x7f7f7f7f, 0x81817f81, 0x817f817f, 0x81817f81, 0x7f7f7f7f, 0x817f817f, 0x817f7f81, 0x81817f81, +0x817f7f81, 0x817f7f7f, 0x81817f81, 0x817f7f81, 0x7f81817f, 0x81817f7f, 0x7f818181, 0x8181817f, +0x7f7f7f81, 0x81817f81, 0x7f817f81, 0x81818181, 0x817f8181, 0x7f7f8181, 0x81818181, 0x817f7f81, +0x817f8181, 0x81817f7f, 0x8181817f, 0x817f817f, 0x8181817f, 0x7f818181, 0x81817f81, 0x81818181, +0x817f817f, 0x817f7f7f, 0x7f7f7f7f, 0x7f7f7f81, 0x7f817f81, 0x81817f81, 0x7f7f8181, 0x8181817f, +0x817f7f81, 0x7f818181, 0x7f7f817f, 0x7f81817f, 0x81818181, 0x81817f7f, 0x7f817f7f, 0x817f8181, +0x7f7f8181, 0x81817f81, 0x7f7f8181, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x7f817f7f, 0x81817f81, +0x7f7f817f, 0x817f817f, 0x817f8181, 0x7f817f7f, 0x7f817f7f, 0x8181817f, 0x817f8181, 0x817f7f7f, +0x817f7f81, 0x817f8181, 0x8181817f, 0x7f818181, 0x7f818181, 0x7f7f7f81, 0x817f7f81, 0x7f817f81, +0x81817f81, 0x817f8181, 0x817f7f7f, 0x817f7f7f, 0x81818181, 0x7f7f7f81, 0x817f817f, 0x817f7f81, +0x81817f7f, 0x7f818181, 0x7f7f817f, 0x7f818181, 0x7f7f8181, 0x7f817f7f, 0x817f8181, 0x81817f81, +0x81818181, 0x7f817f7f, 0x81817f81, 0x7f817f81, 0x817f7f81, 0x7f7f7f81, 0x81817f7f, 0x7f817f7f, +0x7f7f8181, 0x817f8181, 0x7f818181, 0x8181817f, 0x81817f81, 0x817f7f7f, 0x7f817f81, 0x81818181, +0x817f817f, 0x81818181, 0x817f7f81, 0x817f7f7f, 0x817f817f, 0x7f817f81, 0x7f817f7f, 0x817f7f81, +0x7f818181, 0x817f7f7f, 0x7f81817f, 0x81817f7f, 0x817f8181, 0x817f8181, 0x81818181, 0x817f7f81, +0x7f7f817f, 0x81817f7f, 0x8181817f, 0x7f7f8181, 0x7f81817f, 0x8181817f, 0x817f7f81, 0x7f81817f, +0x817f817f, 0x7f7f817f, 0x817f7f7f, 0x8181817f, 0x7f7f817f, 0x7f81817f, 0x8181817f, 0x7f81817f, +0x81818181, 0x81818181, 0x7f81817f, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x7f818181, 0x817f7f7f, +0x817f7f7f, 0x81817f81, 0x7f817f81, 0x7f817f7f, 0x817f8181, 0x7f81817f, 0x7f7f817f, 0x81817f7f, +0x7f7f7f7f, 0x817f7f81, 0x81817f81, 0x817f817f, 0x7f7f8181, 0x817f7f81, 0x7f7f8181, 0x81818181, +0x7f7f7f81, 0x81817f7f, 0x817f7f81, 0x8181817f, 0x7f7f7f81, 0x817f7f7f, 0x81818181, 0x81817f7f, +0x7f817f7f, 0x817f7f7f, 0x81817f7f, 0x81817f7f, 0x817f7f7f, 0x81817f7f, 0x817f8181, 0x7f81817f, +0x7f7f7f7f, 0x817f817f, 0x817f7f81, 0x8181817f, 0x817f7f7f, 0x7f817f7f, 0x8181817f, 0x81817f81, +0x817f7f81, 0x7f7f7f81, 0x7f7f7f81, 0x7f818181, 0x7f817f81, 0x81818181, 0x817f8181, 0x8181817f, +0x7f817f7f, 0x817f8181, 0x7f818181, 0x817f7f81, 0x7f817f7f, 0x7f7f8181, 0x817f817f, 0x817f8181, +0x7f81817f, 0x7f7f817f, 0x7f7f8181, 0x7f7f817f, 0x8181817f, 0x81817f81, 0x817f8181, 0x7f7f7f81, +0x8181817f, 0x8181817f, 0x8181817f, 0x7f7f7f81, 0x8181817f, 0x7f818181, 0x81818181, 0x7f7f7f81, +0x7f817f7f, 0x81818181, 0x817f8181, 0x817f8181, 0x7f7f7f7f, 0x817f7f81, 0x817f817f, 0x817f7f81, +0x81818181, 0x817f8181, 0x7f7f8181, 0x7f7f8181, 0x817f817f, 0x81817f7f, 0x7f7f8181, 0x817f8181, +0x7f7f817f, 0x7f7f817f, 0x7f817f81, 0x7f817f7f, 0x7f818181, 0x7f81817f, 0x81817f7f, 0x817f8181, +0x817f8181, 0x7f818181, 0x8181817f, 0x7f817f7f, 0x81817f81, 0x7f7f8181, 0x817f7f81, 0x817f8181, +0x7f7f7f7f, 0x817f7f7f, 0x7f817f81, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f7f81, 0x81818181, 0x81817f81, +0x7f817f7f, 0x7f818181, 0x817f7f81, 0x8181817f, 0x81817f7f, 0x8181817f, 0x7f81817f, 0x7f817f81, +0x817f8181, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, 0x7f818181, 0x7f81817f, 0x7f7f8181, 0x7f7f7f81, +0x817f817f, 0x81817f7f, 0x7f81817f, 0x81817f7f, 0x7f818181, 0x817f8181, 0x81818181, 0x8181817f, +0x81817f7f, 0x7f817f7f, 0x817f8181, 0x81817f7f, 0x7f7f7f7f, 0x7f7f7f7f, 0x81817f81, 0x817f817f, +0x817f8181, 0x81817f81, 0x81817f81, 0x7f817f7f, 0x7f7f817f, 0x7f7f7f81, 0x7f7f7f7f, 0x8181817f, +0x81817f81, 0x7f817f7f, 0x81817f81, 0x81817f81, 0x7f817f7f, 0x7f7f817f, 0x81817f81, 0x817f8181, +0x81817f81, 0x8181817f, 0x7f7f7f81, 0x817f8181, 0x81817f7f, 0x81817f81, 0x817f8181, 0x817f8181, +0x817f817f, 0x817f7f7f, 0x81817f7f, 0x7f817f81, 0x7f7f8181, 0x7f7f7f7f, 0x7f818181, 0x81817f81, +0x817f8181, 0x817f7f7f, 0x817f817f, 0x8181817f, 0x81818181, 0x7f817f81, 0x81817f7f, 0x81818181, +0x7f818181, 0x817f817f, 0x7f7f7f81, 0x817f817f, 0x7f7f817f, 0x817f817f, 0x7f7f7f7f, 0x817f7f7f, +0x817f7f7f, 0x7f817f81, 0x7f7f817f, 0x81817f7f, 0x7f817f7f, 0x817f7f81, 0x817f817f, 0x817f817f, +0x817f817f, 0x7f817f81, 0x7f818181, 0x7f7f7f81, 0x7f817f81, 0x81817f81, 0x81818181, 0x7f817f7f, +0x7f7f7f81, 0x817f817f, 0x7f817f7f, 0x817f7f81, 0x7f7f7f7f, 0x817f8181, 0x7f7f7f7f, 0x817f817f, +0x81818181, 0x81818181, 0x8181817f, 0x81817f81, 0x81818181, 0x817f8181, 0x7f817f7f, 0x7f817f81, +0x817f8181, 0x7f818181, 0x817f7f81, 0x817f7f7f, 0x7f7f817f, 0x8181817f, 0x817f7f7f, 0x7f817f7f, +0x7f7f7f7f, 0x7f81817f, 0x7f7f817f, 0x7f81817f, 0x81817f81, 0x817f8181, 0x7f817f81, 0x817f8181, +0x81818181, 0x817f817f, 0x81817f81, 0x817f817f, 0x817f8181, 0x817f7f7f, 0x7f7f7f7f, 0x7f7f7f7f, +0x81817f81, 0x7f81817f, 0x817f7f81, 0x7f818181, 0x81817f7f, 0x7f7f8181, 0x7f7f8181, 0x7f817f7f, +0x8181817f, 0x7f81817f, 0x8181817f, 0x81818181, 0x81817f7f, 0x7f7f817f, 0x81817f81, 0x8181817f, +0x7f81817f, 0x7f817f81, 0x7f7f817f, 0x7f817f7f, 0x7f817f7f, 0x817f8181, 0x81817f81, 0x7f7f7f7f, +0x7f817f7f, 0x81817f7f, 0x7f7f7f7f, 0x817f817f, 0x7f7f7f81, 0x817f7f81, 0x7f818181, 0x7f7f8181, +0x7f817f81, 0x7f817f7f, 0x8181817f, 0x7f7f7f81, 0x7f7f8181, 0x81818181, 0x7f7f8181, 0x7f817f7f, +0x7f7f8181, 0x7f81817f, 0x7f7f8181, 0x81818181, 0x7f817f7f, 0x7f7f8181, 0x7f817f7f, 0x8181817f, +0x7f817f7f, 0x817f7f7f, 0x81817f81, 0x7f817f81, 0x817f817f, 0x7f817f81, 0x7f817f7f, 0x81818181, +0x7f817f81, 0x7f7f8181, 0x7f81817f, 0x81818181, 0x7f7f7f81, 0x7f817f7f, 0x7f7f7f7f, 0x7f817f81, +0x7f81817f, 0x817f7f7f, 0x817f8181, 0x7f817f7f, 0x817f7f81, 0x7f7f7f7f, 0x7f817f7f, 0x7f817f81, +0x817f7f7f, 0x817f8181, 0x7f7f7f7f, 0x817f7f81, 0x81817f7f, 0x81818181, 0x817f817f, 0x81818181, +0x7f7f7f81, 0x7f7f7f81, 0x7f7f7f7f, 0x81817f81, 0x7f81817f, 0x7f7f7f81, 0x7f7f7f81, 0x81817f7f, +0x817f7f81, 0x817f7f81, 0x7f817f7f, 0x7f7f7f7f, 0x8181817f, 0x81817f7f, 0x7f7f7f7f, 0x7f818181, +0x7f818181, 0x81818181, 0x817f8181, 0x81817f7f, 0x7f7f8181, 0x817f817f, 0x7f818181, 0x7f7f8181, +0x7f7f817f, 0x7f7f8181, 0x7f7f7f7f, 0x8181817f, 0x81818181, 0x817f8181, 0x7f7f7f7f, 0x7f81817f, +0x81817f7f, 0x7f7f8181, 0x81817f81, 0x817f7f7f, 0x817f7f7f, 0x7f817f81, 0x817f7f7f, 0x81817f7f, +0x817f817f, 0x7f7f7f81, 0x817f7f7f, 0x817f817f, 0x817f7f7f, 0x817f8181, 0x81818181, 0x7f81817f, +0x7f7f8181, 0x7f81817f, 0x81817f7f, 0x7f818181, 0x81817f81, 0x817f7f81, 0x7f7f8181, 0x7f817f81, +0x7f7f7f81, 0x7f7f7f81, 0x7f7f7f7f, 0x817f8181, 0x8181817f, 0x817f7f81, 0x817f7f81, 0x7f817f81, +0x7f7f817f, 0x7f817f81, 0x7f817f7f, 0x7f817f81, 0x81818181, 0x81818181, 0x7f7f7f7f, 0x81818181, +0x7f817f7f, 0x81818181, 0x7f81817f, 0x7f817f81, 0x81817f7f, 0x817f7f81, 0x7f7f817f, 0x7f817f7f, +0x7f818181, 0x7f7f817f, 0x7f817f7f, 0x7f7f7f7f, 0x817f8181, 0x81817f81, 0x817f7f81, 0x7f7f7f81, +0x81818181, 0x7f817f7f, 0x81818181, 0x7f818181, 0x817f7f81, 0x7f817f7f, 0x81818181, 0x81818181, +0x7f817f81, 0x7f7f7f7f, 0x81817f7f, 0x7f818181, 0x81817f7f, 0x81817f81, 0x8181817f, 0x817f817f, +0x817f817f, 0x817f7f81, 0x7f7f8181, 0x817f7f7f, 0x817f8181, 0x7f7f7f7f, 0x7f7f7f7f, 0x817f7f81, +0x817f7f7f, 0x7f7f8181, 0x7f817f81, 0x7f81817f, 0x817f7f7f, 0x7f7f7f7f, 0x7f7f8181, 0x7f7f817f, +0x817f7f7f, 0x81818181, 0x8181817f, 0x7f817f7f, 0x7f7f8181, 0x81817f81, 0x7f81817f, 0x7f7f8181, +0x817f7f7f, 0x817f7f81, 0x7f81817f, 0x7f7f7f81, 0x7f817f81, 0x8181817f, 0x817f7f81, 0x7f817f81, +0x81817f81, 0x81817f81, 0x81817f7f, 0x7f7f817f, 0x817f7f81, 0x7f81817f, 0x817f817f, 0x7f817f81, +0x817f7f81, 0x7f7f817f, 0x7f7f817f, 0x7f7f7f7f, 0x7f81817f, 0x7f7f817f, 0x8181817f, 0x817f7f7f, +0x7f817f81, 0x7f817f7f, 0x7f81817f, 0x81817f81, 0x7f817f7f, 0x7f81817f, 0x81817f7f, 0x7f7f7f7f, +0x817f817f, 0x7f7f7f7f, 0x7f81817f, 0x817f8181, 0x81818181, 0x7f81817f, 0x7f817f7f, 0x8181817f, +0x817f817f, 0x81818181, 0x81817f81, 0x81817f81, 0x7f7f8181, 0x7f7f7f81, 0x7f7f7f7f, 0x8181817f, +0x7f817f81, 0x817f7f81, 0x81818181, 0x7f7f8181, 0x81817f7f, 0x81818181, 0x7f81817f, 0x7f7f7f7f, +0x81818181, 0x7f81817f, 0x7f7f7f81, 0x8181817f, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f8181, 0x81818181, +0x81818181, 0x7f7f8181, 0x7f817f81, 0x81817f7f, 0x7f7f817f, 0x817f7f81, 0x7f7f817f, 0x817f817f, +0x81817f7f, 0x7f7f817f, 0x7f7f7f7f, 0x81817f81, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, 0x7f7f8181, +0x7f817f7f, 0x7f817f81, 0x817f8181, 0x817f7f7f, 0x7f818181, 0x7f81817f, 0x817f817f, 0x7f7f8181, +0x81817f7f, 0x7f7f7f7f, 0x817f7f81, 0x8181817f, 0x81818181, 0x7f817f81, 0x7f817f81, 0x7f81817f, +0x7f81817f, 0x7f7f817f, 0x817f7f7f, 0x7f817f7f, 0x817f7f81, 0x7f7f817f, 0x817f8181, 0x7f7f7f7f, +0x7f81817f, 0x7f7f817f, 0x817f817f, 0x817f817f, 0x7f817f7f, 0x7f7f7f7f, 0x7f81817f, 0x7f7f8181, +0x817f7f81, 0x7f7f7f81, 0x7f818181, 0x817f817f, 0x7f7f817f, 0x7f817f81, 0x817f7f81, 0x81818181, +0x81817f81, 0x7f7f7f81, 0x7f817f7f, 0x7f818181, 0x817f817f, 0x7f7f8181, 0x7f81817f, 0x7f81817f, +0x817f7f81, 0x81818181, 0x7f817f81, 0x7f7f7f81, 0x7f81817f, 0x7f818181, 0x7f817f7f, 0x81817f81, +0x81818181, 0x7f817f7f, 0x81817f81, 0x81817f81, 0x7f7f817f, 0x817f8181, 0x81818181, 0x7f81817f, +0x81817f81, 0x7f7f817f, 0x7f7f817f, 0x7f817f81, 0x817f7f7f, 0x8181817f, 0x81817f81, 0x81818181, +0x7f817f81, 0x7f7f7f7f, 0x81818181, 0x7f7f8181, 0x81817f81, 0x81817f7f, 0x7f7f817f, 0x7f7f7f7f, +0x7f7f7f81, 0x7f7f8181, 0x7f7f7f7f, 0x7f7f7f81, 0x81817f81, 0x817f8181, 0x7f7f817f, 0x7f7f7f7f, +0x817f817f, 0x8181817f, 0x817f8181, 0x7f7f7f7f, 0x7f7f7f81, 0x81817f7f, 0x7f7f7f7f, 0x7f7f817f, +0x7f81817f, 0x7f7f7f81, 0x817f8181, 0x81817f81, 0x817f7f81, 0x7f7f817f, 0x7f81817f, 0x7f7f7f7f, +0x81818181, 0x81817f81, 0x7f818181, 0x7f817f7f, 0x7f7f8181, 0x81817f81, 0x8181817f, 0x7f81817f, +0x81818181, 0x7f818181, 0x817f7f81, 0x81817f7f, 0x81818181, 0x817f7f7f, 0x7f7f7f81, 0x817f7f7f, +0x817f817f, 0x817f7f81, 0x81818181, 0x81817f81, 0x7f818181, 0x81817f7f, 0x817f817f, 0x7f81817f, +0x817f7f81, 0x7f81817f, 0x8181817f, 0x7f7f7f7f, 0x8181817f, 0x7f7f817f, 0x7f817f7f, 0x817f7f7f, +0x7f818181, 0x817f8181, 0x7f7f817f, 0x817f7f81, 0x7f817f81, 0x81817f81, 0x7f7f8181, 0x7f81817f, +0x7f7f817f, 0x7f817f7f, 0x7f817f7f, 0x817f7f81, 0x817f817f, 0x817f7f81, 0x81818181, 0x7f7f817f, +0x817f8181, 0x7f81817f, 0x7f817f7f, 0x81817f81, 0x7f7f817f, 0x81817f81, 0x7f7f817f, 0x81818181, +0x81817f7f, 0x7f817f7f, 0x817f8181, 0x7f7f8181, 0x817f7f81, 0x81818181, 0x7f7f8181, 0x7f817f7f, +0x7f817f7f, 0x817f817f, 0x817f7f81, 0x81817f81, 0x7f7f8181, 0x817f7f7f, 0x7f7f8181, 0x81817f81, +0x7f817f7f, 0x817f8181, 0x817f817f, 0x81817f81, 0x81817f81, 0x7f7f817f, 0x7f7f8181, 0x7f7f7f81, +0x81817f81, 0x7f81817f, 0x7f7f8181, 0x81818181, 0x817f7f7f, 0x7f81817f, 0x7f7f817f, 0x7f817f7f, +0x7f818181, 0x7f81817f, 0x817f8181, 0x7f7f7f7f, 0x7f81817f, 0x7f818181, 0x817f817f, 0x81818181, +0x7f818181, 0x7f817f81, 0x7f81817f, 0x7f7f8181, 0x817f7f81, 0x7f7f8181, 0x7f81817f, 0x7f7f7f81, +0x7f818181, 0x7f817f7f, 0x7f7f8181, 0x7f7f8181, 0x7f818181, 0x81818181, 0x817f7f81, 0x817f8181, +0x7f7f7f7f, 0x7f818181, 0x7f7f7f7f, 0x7f7f7f81, 0x817f8181, 0x7f7f7f7f, 0x817f7f7f, 0x7f818181, +0x81818181, 0x7f7f817f, 0x81818181, 0x7f818181, 0x7f817f7f, 0x7f817f81, 0x7f7f8181, 0x81818181, +0x7f7f7f81, 0x817f8181, 0x7f7f7f7f, 0x81818181, 0x81817f7f, 0x7f817f7f, 0x7f817f7f, 0x817f817f, +0x7f817f81, 0x817f8181, 0x817f817f, 0x817f817f, 0x7f7f8181, 0x817f7f81, 0x7f7f8181, 0x7f7f7f81, +0x7f7f7f7f, 0x7f7f7f7f, 0x8181817f + +e = +34560 + +k = +6144 + +rv_index = +0 + +iter_max = +8 + +iter_min = +4 + +expected_iter_count = +8 + +ext_scale = +15 + +num_maps = +0 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_SOFT_OUTPUT, RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE, RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN, +RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_sbd_posllr.data b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_sbd_posllr.data new file mode 100644 index 00000000..13ad0908 --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c1_k6144_r0_e34560_sbd_posllr.data @@ -0,0 +1,1225 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_DEC + +input0 = +0xf6bb1100, 0x4e433af1, 0x38f02efa, 0x0ee43d00, 0x5037c35f, 0x5620e729, 0xdd46c4d7, 0x5ae3e2f7, +0xe0d7fa2f, 0x2e4ae20d, 0xa7190ebf, 0xd2382614, 0x98ffabe0, 0xf7ee0c5d, 0xffd8ddd7, 0x5f03ecc7, +0xf9ffdde5, 0xf12446f6, 0x42693a3d, 0xfcc81aae, 0x2b0abc2c, 0x43f2a1b5, 0x044a302c, 0xf8db4fa6, +0xcad2d138, 0x452b5526, 0xf3083fd3, 0x30ea4b22, 0xe2bd66f4, 0xbc37bebe, 0x001e183b, 0xfac72ebf, +0x11965cf5, 0x70226311, 0xbd2016ec, 0xcf23f931, 0x4629f6fb, 0x3104de76, 0x00d7c529, 0xa0c8f8f4, +0x2615c2e0, 0x23daf4b4, 0x56bf05c4, 0x9212102c, 0xa5fe400d, 0xec2a5497, 0x021032bc, 0x3b55340e, +0xd6f1003c, 0x2a3625d0, 0xc81eee31, 0xa26644d7, 0x0ffb041c, 0x22cc20dc, 0x472cdd9b, 0xd8e24ed2, +0x4ee740ab, 0x9bfc50e2, 0xbaaafe55, 0xeecd5c19, 0x57150c31, 0x9d3f13c1, 0x251749e1, 0x076ca964, +0xde0da74b, 0xcafac963, 0x52d006e2, 0xe392d7f4, 0x3a20bcf6, 0xd1ee6fce, 0xe5cf850a, 0x31253fe7, +0xcefed8a9, 0x520f9824, 0x5fdbe3c9, 0x9b123cc5, 0x6ad019f2, 0x171f0368, 0xc2fe2855, 0x0db851c2, +0x095010d9, 0xd110df25, 0x1452a52c, 0xe5d1c2cc, 0x45dbef40, 0x2aef2353, 0x07af2e2a, 0x4f23ff6f, +0x3ddb31cb, 0xa8430c49, 0xcaf7b329, 0xf933c5bc, 0xfe3c52b3, 0x7fd60a27, 0xce0ac9cf, 0xcff0cdd7, +0x5400cd01, 0x7f81a8c1, 0x3c7f8178, 0x737f87f6, 0x5a5e7f81, 0x817f6ff6, 0x813b1381, 0x41d18173, +0x810fb9f9, 0xb734c526, 0x50b3a6d3, 0x282d6e50, 0x81972d1f, 0x7d52bd53, 0x6749268b, 0xc681812d, +0x2c93912e, 0xb1813bdc, 0xed817fd1, 0xa8813376, 0x7f3c7f41, 0x64569f7c, 0xf53d5c67, 0x1e70e6dd, +0x024b6591, 0x817f7f8f, 0x0a722d63, 0x7fed578b, 0x7f239a7f, 0x81029081, 0xa8ca8b81, 0xb8577fde, +0x818ec07f, 0x7fa87cb9, 0xa7084967, 0x81934650, 0x7f8171f7, 0xed3d144a, 0x8107739c, 0x3c6fe27f, +0xcd6ed165, 0xfb3718a4, 0x5f708f4d, 0x14e45eb2, 0xfe5c7f7f, 0x7fc57f6a, 0x813683af, 0x817f7fb3, +0x007fa7ef, 0xdcd681a1, 0xd72a1268, 0x769f7881, 0xb27fa281, 0x89c27fe3, 0x729b2f59, 0x73afb8a0, +0xd07f4eb8, 0x81dd6d55, 0x7fa3d2cf, 0x7f81b51e, 0x7f7f7be7, 0x60819981, 0xf31158ad, 0xde7fe4b4, +0x427f2dd3, 0xd3812481, 0x280de7a4, 0x0a653fc6, 0x8181261f, 0xd6aa8f81, 0x2c7fe484, 0x42df747f, +0x7f5baa2d, 0xc1bd583c, 0x966b5031, 0xbe818725, 0x6b6444a4, 0x610023e7, 0x5f183b2c, 0xa77fb769, +0xb2cb1581, 0x0881cc75, 0x8a8c6fdc, 0x558181e4, 0x107f947f, 0x749e4ca3, 0x8cb23db1, 0xce668132, +0x70b848eb, 0x008a2430, 0x7fd3d77f, 0x682b35eb, 0x2e6aae34, 0xd42fa2cd, 0x3eab816f, 0xfdaf8196, +0x04e43fcc, 0xf1819600, 0x7f6d64f2, 0x81ba8957, 0x81a781a2, 0xb169d67a, 0x7f4e557f, 0xe0818197, +0x7b474ecc, 0x3f7b8154, 0x135684f8, 0x81b0fb81, 0x81e33e93, 0x815c817f, 0x810b818a, 0x4d445b34, +0x95cf2bb0, 0x32c69b7d, 0x2640e181, 0x7fe8b8c8, 0x9bdd64f7, 0x22bfa50b, 0x8181c87f, 0xf7812486, +0x7f977fa0, 0x5826a925, 0xce7fadf6, 0x9e73fb81, 0x538d7f0e, 0xe85dab87, 0xc3706181, 0x4181817f, +0x7f7f15ca, 0x952ca87f, 0xa9748be0, 0xbf812477, 0xd081a553, 0x9a817f7f, 0x1f4cb192, 0x7f8181d5, +0x2ba44ad2, 0x76605566, 0x7f1cd2b9, 0x446eae90, 0x8956b670, 0xaadc8b7f, 0x710b0a90, 0x83816439, +0xc6ffc6d2, 0xf07f00fb, 0x7fa25582, 0x7fd11f3a, 0x7f104c81, 0x81d574fd, 0x81814081, 0x82b78e7f, +0x2f29f42b, 0x277f4bff, 0xf49d4b81, 0x28581681, 0x22523281, 0x607ff910, 0x61ac1ad2, 0x58313ccf, +0x8168a21f, 0x7fe61879, 0x817f21b3, 0x407f2f45, 0x7f5fa666, 0x81bc41fb, 0xfd81fb81, 0x0ed48163, +0x81818181, 0x7f7b92fd, 0x44507d7f, 0xba45c9d3, 0x81852b64, 0x7fa918eb, 0x5f6ff5c6, 0xa4ad8181, +0xc27f8f81, 0xb8af5481, 0x7fc5817d, 0x7a814ea2, 0xf77fcf36, 0x50913981, 0x81a01781, 0x5ea1a4ba, +0x18d252b8, 0xc0416881, 0x7f817f1e, 0x0b4e38cf, 0x7ffb6922, 0xfca481b5, 0x7f1dc1ca, 0x7f560da2, +0x7f7f813a, 0x1900a881, 0x6d4d8116, 0x617f0469, 0xb3118e81, 0x95408119, 0x817ff08a, 0x7fdc7f8b, +0xf682a40a, 0x3381817f, 0x81b70d82, 0x7f7f6f81, 0x8190217f, 0x2f817f4b, 0xe195b209, 0x5ee39575, +0x8150bae6, 0xa4fe8b7f, 0x7f7f654c, 0x7f921971, 0x53817f85, 0x1c7f5ff6, 0x5f35737f, 0xce7fc2c6, +0x6c81817f, 0x7f7c410d, 0x5794308e, 0x15f4a87f, 0x7fb64d7b, 0x1d0a8181, 0xa8b67f37, 0x253b397f, +0x887f92b7, 0x7f81767f, 0x81c18101, 0xaa2861c0, 0xbd691cd9, 0x81d74ef3, 0x81a78153, 0xcb7f819b, +0xf681c5e7, 0x9f37817f, 0x3a7f869b, 0x56a41581, 0x04be8111, 0x4898b48b, 0xbc81465d, 0xb3815281, +0x7f6a1d45, 0x1fa87fce, 0x810e81cc, 0x7f8d410c, 0x8190a193, 0x81817f81, 0x5dadcbb4, 0x81a83699, +0x86a68164, 0xe27f1281, 0x682a7f27, 0x62a03cbd, 0x8181c75d, 0x39653881, 0xc6187e83, 0x617fd27f, +0x81917f90, 0x57796c81, 0xdc7f567f, 0xad712381, 0xdd81ecbe, 0xec17667c, 0x929e5098, 0x857ffbbf, +0x8d5b8153, 0x9769887f, 0x65173381, 0x66b8ad84, 0x043481b9, 0x81c05ca9, 0x449d4c48, 0x8c45bc82, +0xca7f81ee, 0x2d8f33e1, 0x81e22f81, 0xca9eb542, 0x8e947fe5, 0xb642a4a8, 0x7f22c73f, 0x85a6176a, +0x567f7ffb, 0x587f4243, 0xa2819225, 0x58307f9b, 0x2c6b0f81, 0x8130a068, 0x814fa904, 0x400a9f2d, +0x30617fd6, 0x81720981, 0x97b98100, 0x0ca17c7f, 0xf1816081, 0x690f8181, 0x94165481, 0x81817f9d, +0x81444268, 0x97759781, 0x8740267f, 0x757f1a7f, 0x7f7f81ca, 0x957fb87f, 0x816f723a, 0x4a8613f2, +0x3bfef57b, 0x7f1274e8, 0x810ab05b, 0x7fb19ecd, 0xb07fc75f, 0x7fb2965f, 0x60abb881, 0xc229a8e9, +0xb1817f26, 0xe324067f, 0x8181e344, 0x7ea59cac, 0xd21a7f65, 0x8d15989f, 0x09026bea, 0x1d7f933b, +0xae81c181, 0xabf37147, 0x0d08da7f, 0xa7ac4b9e, 0x8281e044, 0x7f8195b1, 0x7ff844de, 0xe9057fed, +0x85fcf1cd, 0xa89e7e2b, 0xf974a481, 0x1f813c6d, 0x237fcbbb, 0x655f8192, 0x8134062b, 0x6f91815d, +0xdf3281e1, 0xe6fd817f, 0x72b3007f, 0x86caf9cf, 0xea812981, 0x6a8ad67c, 0x36aaa01d, 0xbb102886, +0x812faf52, 0x6281de5a, 0x817f8174, 0xebff4735, 0x7894bf7c, 0x169e8194, 0x3bc281f8, 0xe69d2a15, +0xb8e5b68d, 0x367f9b8b, 0xea818181, 0xeb7fbbb7, 0x9368a27f, 0x5af5e19e, 0x5f7f7f7f, 0xcd518181, +0x7f948109, 0x817f4a81, 0xf8a0de7f, 0x667f7fa5, 0x1a8881c1, 0x8646b17f, 0xb0eb7f81, 0xe3fe2ba2, +0x460d18da, 0x3db45835, 0xb0c2847e, 0x9c8681a0, 0x817ff88d, 0x7fec639d, 0x817f7f7f, 0x81810d2f, +0xef28241e, 0x811d9db4, 0x4ac4ada5, 0x7ff29f7f, 0x7fd981ac, 0x7f815e23, 0xcc3dbc8a, 0x7c769b08, +0x6a07ef50, 0x66599c9c, 0x7e007f92, 0x9381aa5a, 0x33471b81, 0x79b27d96, 0x6d5d9d5a, 0xbf40687d, +0x7fbc408e, 0xae598111, 0x18222b1e, 0x0a7b817f, 0x7f4961b4, 0x81818f5e, 0xd5a8ae44, 0xe185bd50, +0xb4a65d81, 0xab81c3d6, 0x5f814b7f, 0x8db72efc, 0xc0be14d5, 0x77336a45, 0x2dca7e7f, 0x81857f7f, +0xf1d80b33, 0x1856b3dd, 0x789e75cd, 0x7f7f7f81, 0x8b7f6951, 0xc94cc5b6, 0x407f3f5b, 0x72c4b87f, +0x8135f77d, 0xab02ec1c, 0x8db481e0, 0x77da84df, 0x897f5c49, 0x8181b24d, 0x7f54715e, 0x233ef77f, +0x8165daa7, 0xd3813aa2, 0x7f7f7f18, 0x96e99677, 0x56814a72, 0xa4acba2e, 0x2683347f, 0x3d5b20ac, +0x887fb981, 0x90a9ec91, 0x008ab581, 0x81588181, 0x5c017f7f, 0x10818181, 0x5d86a42d, 0xd8587f4f, +0xed2035b1, 0x718168fc, 0xc3cccf7f, 0xaa812676, 0xa8d83930, 0xbdb119f6, 0x812b0288, 0xe17f7f7f, +0x989ffd31, 0x9e564b12, 0xabfa3578, 0xacd85381, 0xaf7074e6, 0x93a34c11, 0xc1819896, 0x5a5fa673, +0x6f8181ad, 0x4981167a, 0x817f965e, 0x961b7f29, 0xac539db6, 0x2b645dd8, 0x937fa506, 0xa1b900ba, +0xdd7f7f78, 0x7f866ce7, 0x81a9247f, 0x50d67f53, 0x177a8b82, 0x277ffa7f, 0x7f0942da, 0x444b3181, +0x9d8995aa, 0xbda844b9, 0xae765c85, 0x62818f7f, 0x307f7f00, 0xf37fe681, 0x817f7fab, 0x42e0813d, +0x49f3c7c4, 0x7fd543b8, 0x818183f2, 0x7f4b5500, 0xe2357a7f, 0xdc3a8133, 0xd13913ff, 0xabfd587f, +0x81c3814c, 0x667fab62, 0x9bf82b0c, 0xf84b9583, 0x6a28b2f0, 0x8154623f, 0x7f845e7f, 0x2750f07f, +0x517f2d59, 0x7ffe4db5, 0x60077f7f, 0x81816e62, 0xc40a30dd, 0x67f22b7f, 0xec817f81, 0x7fb8c67f, +0x98c9afd0, 0xb881ecaf, 0x6069c37f, 0x792c2c09, 0x709dc2de, 0x4481bb20, 0xd4b25981, 0xf8ef7792, +0x7381ae17, 0x7fe05250, 0xc3eb637f, 0x6d350069, 0x327f457f, 0x8ce7b8ab, 0xb281c1da, 0x7f817f61, +0xbbcf7fb1, 0xaf818145, 0xb77f5881, 0xb47ff1b2, 0xcec74a81, 0x5f81e7b4, 0x7f1b727f, 0x81b8701b, +0x508106c4, 0x6d815f7f, 0xbc815bea, 0x464b00f2, 0x78818181, 0x7f7f4f81, 0x817e2c53, 0x817e7fa7, +0x7f4da4a9, 0xdd444f23, 0x7f6cba39, 0x8141625a, 0x7f81813f, 0xcd7f817f, 0xa402814a, 0x817f049f, +0x6fb981b4, 0x7867bd7f, 0x9b45bc81, 0xe7739f52, 0x7f816181, 0xab844448, 0x7f06497f, 0xe555c183, +0x817fde68, 0x81bc0624, 0xe33f3a9b, 0x8122c673, 0x6c3d81d5, 0x392815dd, 0x26527f7f, 0xee649844, +0x817f6fef, 0x81ce32d4, 0x102f81fe, 0x81967f81, 0xa42b9c20, 0x81813c81, 0x2a67a463, 0x90592c8a, +0xc5810e67, 0x81d1dd81, 0xbd7f54e1, 0xf52a814e, 0x7f46a7b3, 0x5a89812b, 0x78e43eac, 0x7f776981, +0x817fc5c2, 0x3f818181, 0xc556c337, 0x7f0081a9, 0x817ffe6b, 0x7f095f3d, 0x819b2281, 0xb45881ad, +0xc0817f47, 0x81bd915a, 0xc7616b7f, 0x81e9817a, 0x44977f6c, 0xbc81307f, 0x7f9ac3c9, 0xff061265, +0x3c7b0181, 0x13910e7f, 0xb499ed3c, 0xd850b881, 0x81781837, 0x7f97ff93, 0x81818f23, 0xb9b50d81, +0x49aa4fbd, 0x6b361230, 0x148179d3, 0xbd81dd1f, 0x4cd5d0d0, 0x9ad1738a, 0x2c7f718f, 0x7f9f03b0, +0x789d9ac2, 0x88db95fa, 0x9ae5ac64, 0x727f814c, 0xfa477dac, 0xa670a23f, 0x146afe7f, 0xd4d0451d, +0x6d81819d, 0x9622acc1, 0x818161a2, 0x68263dca, 0x7fce818d, 0x998193ba, 0x9d81a486, 0x7eb97f7f, +0x134470ec, 0x6681bde5, 0x4e2ea77f, 0x81777f7f, 0x7f7fa64f, 0xd3815a7d, 0x81fd4e50, 0x774481e8, +0x7f6c8148, 0xe76b7f81, 0x881bb4e9, 0x7f944f3d, 0xe6da7f20, 0x54c68176, 0x210ebb45, 0xbed6267f, +0xe22f2081, 0x8191ae8a, 0x7c7f257f, 0x667f811c, 0xe2edd355, 0xa4812181, 0x837f5481, 0x6d818120, +0xb7aa7f8f, 0xc17f7081, 0x7fba986a, 0x819a81d5, 0xa9816681, 0x8129c4f5, 0x37ba21a7, 0x29d18a86, +0x4be3ce96, 0x3981b184, 0x447fb50b, 0x261c8155, 0xc05ab18d, 0x777f7816, 0xe48be469, 0x81722d81, +0x7f47a8a1, 0x924a8b72, 0xa2ef4ecd, 0x105c817f, 0x7f90709b, 0xbc81443a, 0xa82f7f65, 0x615d928c, +0x7f81570d, 0x61b0919a, 0x7f33fc81, 0xe8817fca, 0x637ffb00, 0x9c337250, 0x817f8181, 0x08b01e0a, +0x7f4ae897, 0xa0969a3c, 0x7f9d7fe8, 0xe024598e, 0x9e794d7f, 0x72fb988c, 0xf8936747, 0xacbd7f8e, +0x4e7f9781, 0xa4b9f4ea, 0x50ccfbdd, 0x817f4795, 0xbcaf485f, 0x81811a25, 0xa97881c6, 0xecbd7fd7, +0x6e89a832, 0x1e815f96, 0x74d0437f, 0x858101b9, 0xbf1b7f8b, 0xcb7fdc9e, 0x1901bda8, 0x4c40c087, +0x5884f075, 0x7f561faf, 0xc3878172, 0xcd0081c4, 0xd93e8fa6, 0xe8748146, 0x4c7f207f, 0x20da2792, +0xc0d37f7b, 0x909a0133, 0x3b6fb281, 0x69819ba1, 0x81984b27, 0x362ddda3, 0x3981727f, 0xe44c7f81, +0x81b9d0bc, 0xebea7fa7, 0x81a47f2e, 0x81674391, 0x495500fa, 0x0eb881bf, 0x2e22f17e, 0x967f9e81, +0xd2727906, 0xdbe8814d, 0x93cf4d6d, 0x23c3b57f, 0x57be5fe3, 0x2eee7f30, 0x1a501f81, 0x76aaba53, +0x70817fdc, 0x817f2e1e, 0x28381f81, 0x1575817f, 0xc62845e6, 0x8181417f, 0x35d0230c, 0x606fcc28, +0x7f792e44, 0xb7b7ea76, 0xad677f7f, 0x2b1eecee, 0x7f7fe450, 0xefcc6668, 0x264e511c, 0x27818181, +0x81817f7f, 0xce82812c, 0x697f7fec, 0x7e859be9, 0xa1f65543, 0x8170a57f, 0x51231d81, 0x33c03182, +0x81aa7f81, 0x6b7ff34e, 0x51812fcf, 0x7f7f7fe6, 0x6281718d, 0x1eb02c7f, 0x81d64570, 0xa18368e1, +0x4ddb816f, 0x149b81d1, 0x067fa2c6, 0x92decc7f, 0x94001658, 0x814f6d6c, 0x79673f7f, 0x7ff28122, +0x83b97f7f, 0x81248a43, 0xfda17ffe, 0x9e81bc81, 0x81090181, 0x7fdaa350, 0x927f54d4, 0x78fe2e89, +0x7f3d097f, 0x147f2ea7, 0x7f85837f, 0x86a58181, 0xb7ad559c, 0x7796ae60, 0x4e7fef08, 0x2ea371f2, +0x817f7f8c, 0x7fc181c5, 0x8101818e, 0xf1588178, 0x44062eb8, 0xb0832da0, 0x81527f81, 0x7f0e817f, +0x8e06b3e5, 0xa6df2e5f, 0x9d7f7f81, 0xa14ba64d, 0x723d7f91, 0xdb81b64b, 0xb48c117f, 0xe75d0e81, +0x8bd9ba3a, 0x1f817f7f, 0x81847f67, 0x25677f5f, 0x3568b529, 0x7fa2587f, 0x24b02027, 0x47ca91dc, +0x81bba314, 0x7f844f7f, 0xbeb8aa0d, 0x9955fe5b, 0x007f9b81, 0xf0a27f6f, 0x1648e959, 0xc68193a6, +0xbea6ebfd, 0x1654cf81, 0x8142d430, 0x2e7f6693, 0xcc788195, 0x11817f81, 0x3da98171, 0x4fdc5a9c, +0x677fea81, 0x4c6ca0cf, 0x487f8181, 0x7f3b9013, 0x37387f7f, 0xc6b4217f, 0x27afb97f, 0x77b84002, +0xd3a67fef, 0x987f8181, 0x9518817f, 0x5ad4fd7f, 0x7b818146, 0x583c5581, 0xd9643b81, 0x687f92f5, +0x7f8109f2, 0xdfad937f, 0x79a45f7f, 0x007f9c0e, 0x6a6ba6ad, 0xa95bc7e9, 0x9be90653, 0x223381b7, +0x7f7f64ce, 0x81813ea3, 0x037f4f7f, 0x617f7f9a, 0x6481f081, 0x81eb4a81, 0x7f9d9a6e, 0xe80e8a4b, +0x4563f81e, 0x94db8781, 0xbf81bd04, 0x305d8236, 0x2c90d571, 0xa48e4600, 0x81633342, 0x2c537fde, +0xc70f5d7f, 0x65bf6f7f, 0x547f9632, 0x05847f1a, 0x23818147, 0xd6d9459f, 0xc97fb761, 0x8196816d, +0xb5417f5b, 0xb6f91bd9, 0x55b17fbd, 0x3e90c27f, 0x4f7fc37f, 0x7f8c855d, 0xef35817f, 0x295d7b03, +0x3730377f, 0x9c811bac, 0x7fe3ca7f, 0xe87f8124, 0x7f7fe511, 0x2d7f7ca2, 0xba9bc726, 0xf07f9f81, +0x4b476e9f, 0xaa338173, 0x5659708a, 0x817fb67f, 0x81811ded, 0x81817f90, 0xa3077f81, 0xf0d119d6, +0xf4818752, 0xbd98271f, 0x95f610ed, 0x3fc227e0, 0x68139681, 0x3c6c2d81, 0x7f7f96cb, 0x3e8181b1, +0xbbe681c6, 0xb47fd2eb, 0xfdd1b74f, 0x4381fe5f, 0xb081f399, 0x64a10081, 0x79ee3ee5, 0x9559c8a6, +0x814ad4a8, 0x817f7531, 0x81813b81, 0xbfbf9773, 0xe27f81a4, 0x7fc14269, 0x917f8158, 0xaf9d89a7, +0x81b22f81, 0x7f54d4d8, 0x0f39a503, 0xec9d3ea0, 0x35818131, 0x8c7d99a6, 0xc0158181, 0x53817f5e, +0x817f81b0, 0x9a7f9a7d, 0xc081747f, 0x517f704d, 0x7f2db90e, 0xa6dfa67f, 0x36817fce, 0x30ae720e, +0x7fd73d60, 0x5c593bd5, 0xf57f9d7d, 0x448151a3, 0xc82db29a, 0x019b5a75, 0x7f876729, 0xa69efc9d, +0x78a67f81, 0x536c817f, 0x81e470d4, 0x56f270fd, 0x2934478f, 0x9c5a7f68, 0x4ecd81ef, 0x7f5c2b23, +0x7f817f03, 0x827fb17f, 0x7f81c35e, 0x7d7f03f8, 0x7f5da67f, 0x8100671d, 0x06e1a8b0, 0x998d112a, +0x9282ae98, 0xed72813b, 0x7fbc6bda, 0x7fa97f3e, 0x36d1510c, 0x0d81819c, 0x56f8fd97, 0x815081a4, +0x7fc78133, 0x69d34542, 0x7fc1727f, 0x7fd22a97, 0x418a1df0, 0xf519077f, 0x7f6e697f, 0xa9817abc, +0xf43917b5, 0x82e3d76b, 0x86327fe7, 0x7f7fb899, 0x9f81b0a4, 0x475381c2, 0xc62b92e8, 0x7f1b49a3, +0x3679b9d7, 0x817f5364, 0xa87f7fc2, 0x45dcf6ba, 0x3470789d, 0x913e7fa5, 0x06b53681, 0x717f8de7, +0x027f6352, 0x7f7fbe7f, 0x7a81df17, 0x0356fb00, 0x307462bf, 0x6716c56c, 0x8157c7b4, 0x7fcd8181, +0x4b81dd07, 0x3b667f7f, 0x7fa6815c, 0xea37397f, 0x61e94781, 0x4e7f8177, 0x81436b5b, 0x4e818172, +0x5b204b37, 0x115501b4, 0x81491681, 0x661468bf, 0x501f88b5, 0x7f4e63e3, 0x8146e97f, 0x7f09f895, +0x7fd70e81, 0x7f7f7f16, 0x3f327fed, 0x05055631, 0x63b4ec13, 0xc03f16da, 0xa0af33c1, 0x942a7f42, +0xb4ba6515, 0xe99d81de, 0x7f78d8b0, 0x446f2c42, 0x92cf7fde, 0x6b9e7f7f, 0x948181cd, 0x94b9ad81, +0x582e8181, 0xc31748a9, 0x39c6d47f, 0xb2817fff, 0xc5817f81, 0xc6887fa4, 0xe69989e4, 0xa67e7a5c, +0x09477f9a, 0x2d4d817f, 0xb24f81d1, 0x3ff77f7f, 0x6a66813b, 0xf5050547, 0xcf917f6c, 0x81917f7c, +0xb8b23b81, 0xcab60081, 0xe17f7f81, 0x3b47547f, 0x457f4781, 0xe17e9c85, 0xd4b5ba00, 0x777fc3f4, +0x8b013d8c, 0x5381a2dd, 0xf60c7f5b, 0xc3f2c212, 0x5c6c7f35, 0xc77fae7c, 0x7f790cc8, 0xe6848181, +0x7f81584b, 0x813688e2, 0x4cff2987, 0xbeb48126, 0xc4be67bc, 0xb8738181, 0x886a9c7f, 0x519881a3, +0x77fbd5c9, 0x4181c081, 0x20813287, 0x6121bb2d, 0x7f7f227d, 0x1a8181a6, 0x811a5d06, 0xc890a57f, +0x7faf7ece, 0x83a36788, 0xd9fd8e37, 0x7f7b985e, 0x36819a38, 0xbd6096c4, 0x7de5be7f, 0x7bc68150, +0x7f8831bb, 0x81818144, 0x434a63e0, 0xaea4ad6e, 0x407f2785, 0xe8df81ea, 0x4a148aa7, 0xf89f7f65, +0x3c757f62, 0x7f81c80e, 0x81818881, 0x81caa47f, 0x9c570881, 0x1c1a8354, 0x2fa8008d, 0x7f7f81ac, +0xa781d413, 0xa9a45530, 0x327f297f, 0x7f647f5c, 0x81a6324e, 0x81a954cd, 0x7f1c2881, 0xe30b2eb3, +0x81b18181, 0x81157f18, 0x967f7f96, 0x819a7fbc, 0xcccf4fc2, 0xc43a48d5, 0x7fa33d74, 0x3aa97f50, +0x5277817f, 0x69afad86, 0x7f81f278, 0x4fa29716, 0x91818170, 0xdc1a7f5c, 0x278159b1, 0xe87f069f, +0xd28cc56b, 0x157f8181, 0x81339981, 0xcf818114, 0x6eba0b8d, 0xd281527f, 0x8249d583, 0xb37fbc43, +0xc181357f, 0x819f6163, 0xaf194881, 0x7f9872a4, 0xf3c3ab2c, 0x47507f81, 0x523f812e, 0xff81d105, +0x5851cbb0, 0x7fe881c3, 0x60cdbe81, 0xc838ccbe, 0x81817f7f, 0x1d76cb34, 0xe100407f, 0x81d78181, +0x2f81817f, 0x927f3ac9, 0x3cc887bb, 0xda7d6d7f, 0x2c7f1e7f, 0xf58e41ee, 0xfb816281, 0xa211d47f, +0x6eea4bf9, 0x297097b8, 0x9d819e81, 0xf6ef4156, 0x2afb4362, 0x7f16647f, 0x8e10c4c3, 0xb17f6181, +0x3b9d2949, 0x53a6c867, 0x810faffb, 0x8197d27f, 0x41873a37, 0xc48d7f7f, 0x81b6fb69, 0x144b2b78, +0x8181d5c4, 0x787fbc24, 0x4381929f, 0x95927f1b, 0xa3a5e451, 0xb48cc58d, 0x818e81d1, 0x9a81cbaf, +0x713ac87f, 0x7faaa694, 0x9f7f81c3, 0xf65b307d, 0x7fdfa25e, 0xa0bae1fe, 0xf3813f12, 0x19817fc5, +0xc0278a81, 0xe47f8158, 0x817fa281, 0x5bbc8181, 0xd63caaf9, 0x7fbf8181, 0x0054157f, 0x7f7f311c, +0x7f817f56, 0x56c22692, 0x8143b624, 0x0845a315, 0x107a8119, 0xc1cd5c7f, 0x7f62e549, 0x7fbb757f, +0x6b8dec74, 0x6893e220, 0xbb224ba9, 0x31819871, 0xdd2b4f57, 0x3d030250, 0xc1539dc4, 0x1c42df45, +0x7f7f6631, 0xaa81eb8f, 0xa6d4edf5, 0x81ec1981, 0xc92d77f8, 0x1ba35b2c, 0x0e1fa0cd, 0xdd257f7f, +0x817fa041, 0xe9815f81, 0x60f1b338, 0xb97f7fad, 0x097f857f, 0xae6a7f60, 0x145dbced, 0x81b7607f, +0xd4647f3d, 0x7f7f52ec, 0x7fdbbea1, 0xc74feec6, 0x1c197f48, 0xb17d1d61, 0x2e68b081, 0xb717d181, +0xda46377f, 0x105e81b7, 0x7f2e7f58, 0x67c33a7f, 0xb83f666b, 0x68518691, 0xc556f081, 0x7763c400, +0x817fbc99, 0x3a18df21, 0xfbde9374, 0x667fc646, 0x737f7fe9, 0x7f987fbe, 0x27dcb107, 0x25f35d30, +0x420adcd0, 0xab7a7f7f, 0x7f8a28c1, 0xcd7fc20e, 0x6ce69bcc, 0x7f308ce5, 0xb2c96e81, 0x95dbeb1e, +0x7f7f567f, 0x6b7fa0a5, 0x997fed60, 0x9281da7f, 0x3f7f617f, 0x7fc4608e, 0x817f4781, 0xc36b7f81, +0x7fddc17f, 0x6fe27020, 0x8a5a2b27, 0x3a4bfb81, 0xae9e4a81, 0x7f7fda7f, 0x5a7ff006, 0x39b6a0b7, +0xb9721181, 0x725edb2f, 0x8903cf08, 0x81aa7cde, 0x7fe0817f, 0x6c9e7fe8, 0x7f5c8140, 0x819d8181, +0x58777451, 0x7fdb4784, 0xc3186166, 0x817f458f, 0xdce8cdc9, 0x7840bd81, 0x1781954b, 0x60240082, +0xca242d7f, 0x788428b6, 0xf67f8d7f, 0x657f7f8a, 0xbace8106, 0xb8816e93, 0xf9b07ff9, 0x818102fa, +0x32028181, 0xfe7f8e27, 0x818d7f26, 0x7f0781a8, 0x7f7f73f3, 0xfaa78135, 0x7f81810b, 0x72676281, +0x812ed271, 0x7f8a961b, 0x7f81817a, 0x367f4181, 0x03ecb2f0, 0x6c810a7f, 0x577f81b1, 0x81e889b8, +0x7f7f707f, 0x7f90368c, 0x86a36487, 0x42b8c17f, 0x91816160, 0x8150b0a5, 0x81390fa1, 0x7f87d07f, +0x448548d0, 0x32810b93, 0x7f20da7f, 0x5b6a7fed, 0x5c812d7f, 0x7f5a7f69, 0x0c36d681, 0x2ee0544a, +0x817d861a, 0xa47fdaf7, 0x7c8127b1, 0x96216604, 0x4f8138de, 0x03e57e14, 0x73050f7f, 0x42007eeb, +0xe303f07f, 0x2f8a7438, 0x8a7f7fbc, 0x866f8165, 0xf496535f, 0xea2d9bb2, 0x16e67386, 0x6bf837ab, +0x411874b4, 0xa7c15c2a, 0x74bb7f72, 0x8181ab81, 0x81e33481, 0x68b47f7f, 0x957f8181, 0x947fcf27, +0xb47f7b20, 0x997ca926, 0x5a457f0f, 0x87403699, 0x811e3ccd, 0xdb857d7f, 0xc9cec481, 0x2998d8c3, +0x231fa37f, 0x876a2942, 0x812d7ff0, 0x7f90959a, 0x811598a2, 0x7f814697, 0x7f7e1467, 0xad3a8154, +0x81813f26, 0x0851321a, 0xf77f9f7f, 0x76b16294, 0xa56c3e3a, 0x7fd5c569, 0x30dd8185, 0x7f7f819c, +0x7f8a7f7f, 0x81dc4da7, 0xe1ce9eb9, 0x8181a2c8, 0x7fda81be, 0x01797f42, 0x7b7f89a8, 0x81e9de7f, +0x7fa76fc9, 0xdebec9a0, 0x7f381e89, 0x81ba0d7f, 0x459a7f7c, 0x81ac8181, 0xb65e8279, 0xdf8174cb, +0x6ea24abd, 0xb8486638, 0x81637f7f, 0x011d5c81, 0xc27ff63d, 0x81d48160, 0x7c81ba7f, 0x7ffa7f73, +0x7f8881f0, 0xc2d6a15b, 0x7f5d7f7f, 0x06818fd5, 0x8c23ea91, 0xf3d8e75c, 0xab81be93, 0x817f7fa8, +0xe57f816d, 0x6e81f27f, 0x81e566aa, 0x88397868, 0x45f9b7c1, 0x2c4d81e5, 0x52765111, 0x813e3d60, +0x4f45c3e7, 0xef777f91, 0x3e7fc481, 0x3e810d42, 0x81c53f7d, 0x2f7f8481, 0x3a7f8157, 0xf1662881, +0x81f03791, 0xaf818136, 0x476b7f7f, 0x6890cfa6, 0xe4f7e9ae, 0x81f6b9ba, 0xb0623aac, 0xea7f7031, +0x72c30000, 0x407f8197, 0x8176917b, 0x81184acc, 0xdacf7fa6, 0x996cd37b, 0xa68175c8, 0x5e6082e3, +0x067f927f, 0xc47f4481, 0x7fc58113, 0x7f561aff, 0x817f46b2, 0x7f7f8159, 0xcebbc57f, 0x72765b87, +0x817f2f1e, 0x885c7c81, 0x7f7f65a9, 0xeded8987, 0x7f3bab81, 0x812f7f6b, 0xfc67d85a, 0x13297781, +0x607f9621, 0x507fa647, 0x4e39ce7f, 0x81ff5c4b, 0xa246316c, 0x81457f7f, 0x5781812f, 0x1463aa9f, +0xc35d7f14, 0xbcbb5771, 0xd7e45781, 0x636c7365, 0x07102b6e, 0x75e1337f, 0x7f2bb116, 0x257fa0f2, +0x97597f40, 0x7f818810, 0x7f7f7f5c, 0xca7ff660, 0x9c916181, 0x903ea1cc, 0x6c8f7f81, 0xc2b07f25, +0xa43d818c, 0x81f67fb3, 0x67f56081, 0x1ba55531, 0xa77fc6a5, 0xda479281, 0xe7647fdb, 0x7fd2f34d, +0x1b7f4c68, 0x7f8c5781, 0xbcf18110, 0x7fd3b883, 0xde81687f, 0x81358948, 0xeb813d17, 0x25040181, +0x7ff8a39c, 0xbc574351, 0xbe677c7f, 0x487b5068, 0x6c7f816c, 0x7f2dca81, 0xb8c7b17f, 0x7f869781, +0x9c99f0ae, 0x437febc4, 0x289d81fe, 0xab74307f, 0x7c7b299a, 0x62593f52, 0x7f4e513a, 0x7f9b81c0, +0x9e2c0de5, 0x43818175, 0x857064a0, 0xd17f8db2, 0xb0a055de, 0xf04ade7f, 0xe37d7fd6, 0x3956789a, +0xcfbcb53a, 0x7fa39e7f, 0x7ed171a8, 0x49a65c89, 0x3ee166d9, 0x2daca6ca, 0x53d051e7, 0x957fc181, +0x0000e873, 0x815fbf9b, 0xe986ac61, 0x7162fd81, 0xe428090f, 0xe2814730, 0xab8d9d3d, 0x8164e6da, +0x8a6126e4, 0x81817f6e, 0xeb818281, 0x359f7f81, 0xb663c39c, 0x7fb856a8, 0x9d812e30, 0xd07c7f2e, +0x6b58ceb5, 0x8181ca7f, 0x42c081b5, 0xd58a6fd8, 0x7faa9b85, 0x81818181, 0xbb3781f4, 0x7fa98173, +0x17f17f7f, 0x7f819862, 0xd97f0781, 0x812f579a, 0x608196ec, 0x23458155, 0x5bc1a902, 0xaa39817f, +0xcb207d81, 0x227f7f9c, 0x81a54ad1, 0x81027fb4, 0xb6711835, 0x8181a57f, 0x7fed8141, 0x7fa95ec0, +0x7ffc7f8f, 0x572f81e8, 0x7ca87f81, 0x7fbeb67f, 0x637f814e, 0xcf11af7f, 0x307fb081, 0xaa85927f, +0x6f5dd995, 0x81db817f, 0x68275869, 0x724dff18, 0x8176ef58, 0x54814c7e, 0xece02300, 0x5ad71a56, +0x689e7fa7, 0xb67fa77f, 0x107f1359, 0x51a63bc0, 0x22c887ca, 0x0a0e34b7, 0x81587fae, 0xbb79604a, +0xfbb7077f, 0x81967f81, 0x81227187, 0x8129817f, 0x43a84e8a, 0xb981a34b, 0x5d307f81, 0x4d7fe9bd, +0xc973a181, 0x78647f8b, 0x00274105, 0x5d7a09e5, 0x6565a681, 0xfbd0ea6f, 0xf986037f, 0x377fc3e0, +0x9ba57fe8, 0x4f7ff13c, 0xd61f7fa0, 0x817f7fbf, 0x7f3e25c7, 0x81c42fdc, 0xae398181, 0x3125afbf, +0x7fed479c, 0xf4f52ff4, 0x7fc99d81, 0xaa417fb7, 0xa1324f81, 0xff7b817f, 0x167f6f86, 0x2a86c481, +0xa127c15c, 0x6d8e0000, 0xcf207f6d, 0x6b815681, 0xd01f50e7, 0xab81e704, 0xce81a4e4, 0x7f177f7f, +0xaa819181, 0x7f81a044, 0x1c0c7f5e, 0x4f0e56cf, 0x46a581ca, 0xb3d52970, 0x6fb3b2c8, 0x7f8189de, +0xf0058133, 0xd181d0da, 0x7481c46e, 0x427fc7d5, 0xe4528188, 0x4a359b7f, 0xd1ae25bd, 0xb735818f, +0x811a2117, 0xddd2829a, 0xec696aa1, 0x99cf945e, 0x518f3d7c, 0x8e814381, 0xa2819647, 0x85af7f43, +0xc81d7f90, 0xb8361781, 0xb67f9a64, 0xc3d38181, 0x97b07dae, 0x54687f7f, 0x206d5b79, 0x437089ad, +0x7f814881, 0x77b63840, 0x61816434, 0x9dca5e22, 0x818129bf, 0x8b49dd3b, 0xc792813a, 0x3b407f63, +0x7fc48f8e, 0x7f96815a, 0x6b430c8b, 0x974ee1a1, 0x7f8181fa, 0xb47098a2, 0x81168181, 0x816b71f5, +0x0a596d81, 0xa5945681, 0xa14181d0, 0xb4944bfa, 0x81f7816b, 0x7f7f287f, 0x7f81cc48, 0x6d819077, +0x7dff7f58, 0x5a4e7f81, 0xb7ee6df2, 0x81712a85, 0x0a652e7f, 0x86a1277f, 0x3c568181, 0x7f817f65, +0x64fb3e0b, 0x71a9687f, 0xa65c7f72, 0x3fb34043, 0x8132f77f, 0xec8133bf, 0x7f8119a5, 0x081f5bb1, +0x7f81777f, 0xedcca8a1, 0x7ffe7f25, 0xec934f43, 0xdbcc8181, 0x64416881, 0x4a8d93f4, 0x7fb64c7f, +0xf12f9bc2, 0x812a8f90, 0x1b9c32de, 0x30a30d81, 0xf079e16a, 0x633720c2, 0xad44fd53, 0x3c812f3b, +0x56b32873, 0x0000b62f, 0x76cb58a6, 0x31086ead, 0x9c307f65, 0xec664f81, 0x7f6186cd, 0xb7047f39, +0xe87f2ccd, 0xe79f8f3c, 0x88b34352, 0x7f6dec72, 0x8b4aca8c, 0x217f1c55, 0x53d76693, 0x7f984d2d, +0x125abe92, 0x7f37677f, 0x4f819c81, 0x7fb23b2e, 0xd2c5ea58, 0x7fa7817f, 0x57817f7f, 0xd45239ec, +0xc887fa03, 0x7f7f7f42, 0x8181444d, 0x2ba67f1a, 0x7fcc527f, 0x813d7f81, 0x68ad12aa, 0xa87f6763, +0x62577f7c, 0x067f7f5a, 0x3201a788, 0x9872b0a1, 0x814548d0, 0x3a883096, 0xc4701d52, 0xb59381dd, +0xdc3381a8, 0x816e3f7f, 0x4f57813e, 0x4679d77f, 0x9a14b75d, 0x8181da4b, 0x817f9d48, 0x52818181, +0x2281c97f, 0xbfb92d7f, 0x1f6f357f, 0x2ca57f3d, 0xb9b4626f, 0xab4bba25, 0xf27f4b5a, 0x4b478a56, +0x818181c3, 0x6a767f29, 0x81e16dcd, 0x4d818181, 0xd1819ce8, 0xdbce6082, 0x757f41bc, 0x777f7fbc, +0x5fa9d103, 0x0241e8d6, 0x1bb381a1, 0xf08177b7, 0x00df007f, 0x3bcdbb5f, 0x5e7f710a, 0x7f3f7f1e, +0x657f0f2b, 0x2ffae79b, 0x495c8110, 0x7f7f8150, 0xe8449f7f, 0x8434a77f, 0x42d56352, 0xfd717f8d, +0x81c514b3, 0x7f147f50, 0x7f856696, 0x7f844c81, 0x8157714b, 0x50cba2ca, 0x73539ff7, 0xb5ed7fea, +0x5d1f637f, 0xbc66496d, 0x68b633df, 0xea76156c, 0x7b4654a3, 0x81547f8d, 0x7f537f27, 0x5db64581, +0x7f367f81, 0x7f81a08c, 0x81440000, 0x3965f076, 0x40977036, 0xcf81ee31, 0x6e7f19bd, 0x703a7f03, +0xe94b81c6, 0x4330aa7f, 0x060411c3, 0x91f6f37f, 0xbdc9e781, 0x81b87f51, 0x4b6e228f, 0xbc3eb87f, +0xad916bce, 0xdfb0976a, 0x98b86387, 0xb651bdee, 0x8cf46cb9, 0x9b819a8d, 0x9d8181f1, 0x7fe0ab97, +0x714d0264, 0xb87f67a4, 0x7f9d3150, 0xe95c7f81, 0x81a7537c, 0x93f4d02e, 0x433a477f, 0xe5067fab, +0x81de5836, 0x64667f81, 0xc5255988, 0x0a7f824f, 0xf97b7fbf, 0x40c97f82, 0xc1d47a74, 0x7f7f3081, +0xfe49fc23, 0xba819538, 0xb87f5581, 0x5ea47fb2, 0x35b37292, 0xb9848178, 0x276f5235, 0x647f4ebd, +0xee2a491a, 0xfd29bb60, 0x21f53728, 0xae9d817b, 0x2ba97f81, 0x6accd440, 0x7fbec77f, 0x909b817f, +0x227f7f35, 0x767f22d5, 0x0ba3c2d1, 0xd38f817f, 0x467fd862, 0x673e8181, 0x7f3b7f81, 0x70427f81, +0x6f725d24, 0x731d64de, 0x32a9816d, 0xc1817f7a, 0x627f8181, 0x7f947615, 0xc37f72d2, 0x7f7f4261, +0xf147be64, 0xc481afab, 0x7fd9a5dc, 0x5c812690, 0x7c34783b, 0x9087817f, 0x7fd25245, 0xea5fcfb1, +0x81817f81, 0xc84ae6b4, 0x8149c875, 0x814a1a7f, 0xab7f6881, 0x6e7f3d81, 0x3ca58181, 0x75cb4781, +0x8181817f, 0x5ec4f2c5, 0x8165e07d, 0x643d6e7d, 0x78bd9764, 0x40817f24, 0x5f077581, 0x7f2ebb4b, +0x7f7fa781, 0xe657ca7f, 0x0000ff21, 0x7fa9bf6a, 0x81c1c30f, 0x9138393c, 0xa2b38174, 0x7f4c24bc, +0x2b918175, 0x7fea5d7f, 0x50c1a17f, 0x0bfefac7, 0x81fe4ea4, 0xab7f7fbd, 0x559b7fb9, 0xcb81e356, +0xd7963f72, 0x7f7a1b81, 0x8c9cd9d3, 0xbf889d1f, 0x161a88cf, 0xe6a0abb9, 0xbb014f2b, 0x81fa947f, +0x7f41509e, 0xb4480d7f, 0x7f7f8181, 0xeca6b540, 0xd27fc381, 0x3aaeb38f, 0x7f7f7f7f, 0x372a7f05, +0xd6a34b81, 0xee378159, 0xcf8181b8, 0x07487fcf, 0xcbb4ed77, 0x29817fab, 0x7fe5b342, 0x63c3bb81, +0x0724d1d8, 0x50e47f7f, 0xa67f8f93, 0x8181c37f, 0x28817a81, 0x7f81fb7f, 0xaab17f7f, 0x81997f7f, +0xa423816f, 0xf953c16a, 0x4c2362cc, 0x81812bf3, 0x5a817f4a, 0xffcbcce0, 0x9c7f8776, 0x5b5a7f7f, +0x64817f92, 0x5139c47c, 0xbe8573ce, 0xe5457f81, 0x2ca4815b, 0x503e14b8, 0x4ca75681, 0x907f7f7f, +0xa2377d19, 0x7fd7c09a, 0x7fbc8191, 0x657f61a2, 0x814f5c81, 0x7f48607f, 0xa16f3ad5, 0x819d227f, +0x247fc10e, 0x7f817f32, 0xc39981f5, 0xb357261b, 0x347f7027, 0xe3814506, 0xa1032e49, 0x979eb571, +0xb06ac281, 0x81815e7f, 0x34b57fb7, 0x7ffe2681, 0xc181630b, 0xbda9ce3b, 0x81f9287f, 0x4ef12081, +0xb86a54ba, 0xbc4d13c7, 0x81817fd9, 0x62817f1c, 0x7f7f4fa1, 0x847f7fce, 0x81bf68f5, 0xd7625f73, +0x81e79d7f, 0x8137a1ca, 0x423581cb, 0x26580000, 0xd7d6811e, 0xce6e6c81, 0x8168173b, 0x498468b2, +0x8c7c817f, 0x6f81b881, 0xad899a25, 0xd049e481, 0x7fdd817f, 0x63816bbe, 0x048b8d81, 0x8eb764c6, +0x987b9ea5, 0x81fae493, 0x7f72817f, 0xa57fbd7f, 0x7f0f81b3, 0xb37f4a33, 0x2e092988, 0x1b172b25, +0x09107f3e, 0x48f5fc7f, 0x69aaa80e, 0x184fe781, 0x7f9db5e6, 0x8195276c, 0x252a8181, 0x8b81ccfa, +0x3c308181, 0x9d19b086, 0xf0d589f7, 0x207408ca, 0xe34e9ed0, 0xe67fbbf0, 0xc0996525, 0x0dca5881, +0x813d4181, 0x7f729738, 0x0a865e5d, 0x8bbea581, 0x5c4aac3f, 0x8181a63f, 0xa1f0b873, 0xa92481d0, +0x837f817f, 0x047fdd84, 0xc8767f2a, 0x441a81be, 0x818681aa, 0x05490179, 0xba81ba7f, 0xde246581, +0x466d70d7, 0xb6bc3f5b, 0x5c428f2d, 0x8105cb77, 0x0206175a, 0x5f6ac2b1, 0x9a4d2a4d, 0x7fb0917d, +0x1281ce12, 0x81028181, 0x41b0a4ad, 0x29973681, 0x973e992e, 0x81617dba, 0xc1818181, 0xb87fc1af, +0x81137feb, 0x627f5210, 0x8181de1d, 0x7f615f70, 0xabbb817f, 0x817fc57f, 0xdbe5b6f3, 0x9c8a7f72, +0x3c2826df, 0xae03d3b5, 0x7f6cc481, 0xd9c41c7f, 0x83ad7f81, 0x336370a8, 0x8bbfaf2e, 0x3561caa5, +0x548d142c, 0x810366b5, 0xa981d36e, 0xbf3afc7f, 0x7fd78157, 0x61817bf1, 0x74afcdd7, 0x18ac3d61, +0xcc45b6e0, 0x3d81af88, 0xf97934de, 0x947a8f81, 0x8b7fa6b8, 0xe0dd8981, 0x2d812381, 0x967f6409, +0x24bc0b1d, 0xdb7fa943, 0x18197fc3, 0x47ef7f6b, 0x66a6bb7f, 0xd374af81, 0xf48e7f9f, 0x52399754, +0xe1abbf6f, 0x04751d7f, 0x3690c581, 0x557fc3a6, 0xc1815413, 0x9086df7f, 0x81e57481, 0x81538167, +0x4df9c981, 0x9463a57f, 0x81c5063a, 0x7f7eaa81, 0xa6dc9423, 0xabc07ffb, 0xd981817c, 0x816064ae, +0x7f815dab, 0xc0b12da0, 0x1444be7f, 0x3a7781e0, 0x887f527f, 0xc0817f1f, 0x05cba881, 0x6599057f, +0x817ffd53, 0xca907fab, 0x557f0d7f, 0x5d029ce9, 0x57818168, 0xc4093382, 0x52e5b33a, 0x763e09b7, +0x814da277, 0xf9208cc5, 0x817f7fcb, 0x7f5fac7f, 0x6d7f6544, 0x7f0a7fec, 0xe7b4337f, 0x7fae8181, +0xa8c5608c, 0x6f9bb388, 0xb27fc773, 0xf4acaa81, 0x1cad30ac, 0x2d817f8b, 0x2b817f16, 0xcca2a581, +0xcb77dad4, 0x58487a22, 0x7fe57f69, 0x45387012, 0x73504724, 0x61357f81, 0xc77f8181, 0xabc7e11b, +0x8181c16e, 0xa681ab27, 0x8c81c48f, 0xbd1de360, 0x9924337f, 0x7f230156, 0x3dd6a87f, 0x477f9381, +0xbeb38181, 0x47815ccb, 0x8125817f, 0xb8057f7e, 0x7f7fc081, 0x46815873, 0x5f93ba81, 0x7f35767f, +0x7f7f5a94, 0x748c84e5, 0x4c7ffc90, 0x7fcd247e, 0x44898e81, 0x1268ecc9, 0x81d2777f, 0xda42e7c7, +0xee267f81, 0x7f5f269f, 0xe3d60c91, 0x81c97f0b, 0xd7b00000, 0x9b813a83, 0x7f7f8181, 0xb71cfa35, +0xf5888156, 0x853e9cc5, 0x1c1c1014, 0x2e7f8b1b, 0x81bea644, 0x70f4e8d8, 0x8158a181, 0x18caad9f, +0x3b28815e, 0x7f460155, 0x7f7f7aee, 0x444b51e5, 0x8682a381, 0x7f508181, 0x5e86466d, 0x7f81287f, +0x7f7c7f50, 0x8173e4de, 0x4c747a65, 0x69816cd7, 0x6674813b, 0x48274e90, 0x92ccf278, 0x99f181c7, +0x81b20243, 0xbf4b4b81, 0x81d981f4, 0xa1e981dd, 0x6c7f7b6e, 0x98cad579, 0x9a7f1685, 0x81e8812e, +0x5d7fcf52, 0x37819772, 0xdb437fab, 0x1d83782a, 0xd3ff71b3, 0xc7e28181, 0x81817ca6, 0x1808127f, +0xb515639d, 0x812c8108, 0xa2a97f14, 0xdd8102c6, 0x727f7f98, 0x5f197f19, 0x7b22817f, 0x46562e7f, +0x59b89c7f, 0xaba28d81, 0x5e4e7181, 0x909da6b9, 0x817d8128, 0x9c167abf, 0x3ef4817f, 0xf5628d72, +0x97727fbb, 0x88c6a600, 0x81c9927f, 0x817443e0, 0x81ac7f7f, 0x7fec67cc, 0xc353bb96, 0x4caae113, +0x7fa37fd9, 0x525c45dc, 0xdf798ba6, 0x7c811049, 0x812570c7, 0x7f788a6b, 0x797f1fef, 0x2022fd81, +0x38870936, 0x813f7d7f, 0x81810941, 0x6b75817f, 0x5aa11953, 0xcc7f7f81, 0x61778581, 0x5e458181, +0xcfff7f90, 0x9020747f, 0xb4e35ad5, 0xea0e8109, 0x21f97f13, 0x41c07f40, 0xc6ddc4ad, 0x085227c8, +0x11dc9f85, 0x7fda7f55, 0x5a92b34c, 0x11659b4d, 0x000081bc, 0x81816781, 0x81cce139, 0xf165d51c, +0x810a817f, 0x7f7f82d6, 0xb5b3817f, 0x40909bfb, 0x7f04a01f, 0xd0288164, 0x810164f3, 0x59778b81, +0xc87f2997, 0x9d99417f, 0x65528173, 0x727f454a, 0x9f187fd9, 0x37948141, 0x217fcf7f, 0x81818319, +0xc27f817f, 0x47817f81, 0x756e817f, 0xa28c8cd3, 0x84b77f81, 0x44ef8e4f, 0x7f7f7f7f, 0x932e2c45, +0x680d20a8, 0x817f1122, 0x52128a21, 0x2b8136ff, 0x0b7f817f, 0xda8c817f, 0x24c17f7f, 0x3dcb39b7, +0x81d18181, 0x81fe1a81, 0x0590978b, 0x71b20319, 0x7f65bfb4, 0x4fb42290, 0xde1a7fd2, 0xc8bd77d3, +0x917f32a0, 0xb03fd9db, 0x4c1e872b, 0x814f5481, 0x44817f7d, 0x398a887f, 0x4a19e8ac, 0x4d818189, +0xc3818181, 0x7f8198dc, 0xb77f527f, 0x817fe681, 0xa83d818c, 0x420b5e81, 0x6e5adbbf, 0x7abd56c4, +0x717d7f7f, 0xc7811c0c, 0x767fba89, 0x6d1d5106, 0x5a06df17, 0x86c28181, 0xcdd6888e, 0x0e747f8a, +0xde81a9c3, 0x45459db8, 0x81114e22, 0xbd7f8197, 0x3c3d85cf, 0x8c815fe8, 0x544d7f64, 0x81986177, +0x8c20a0b5, 0x7f4b8178, 0x17db6031, 0x7f42df7f, 0xee93cc0f, 0x8ab829b2, 0x29a04e65, 0xfc30035f, +0xc57b7f7f, 0xe51ef77f, 0x7fd95481, 0x76957f7f, 0x819f945e, 0xc91a187f, 0x81657e40, 0x8192816d, +0x7339dce0, 0x8bd1c3c7, 0xa2812e1b, 0x81db3d8e, 0x1781da89, 0xd0810000, 0x7fc8c57a, 0x8fc32003, +0x1c847f81, 0x10dfb271, 0x7ff8cc62, 0x2037d87f, 0x8151501e, 0x17c74c74, 0xb8ab2852, 0x4b7f1c97, +0x6a811596, 0x46acc3e0, 0x3c81f49e, 0xd6d81a4e, 0x2e815066, 0x46218186, 0x41682bb4, 0xa1ac315d, +0xb87fdc73, 0x378181ad, 0xe97f979a, 0xccb07981, 0xb10a7f38, 0x03488893, 0xfef78df3, 0x8145467c, +0x3b67b401, 0xb3cf34bc, 0x81476c00, 0xdd638b81, 0x5e43af81, 0x2bab7f6c, 0x6d7f7f6f, 0x7f657f70, +0xbc816e77, 0x8981c27f, 0x3e40b685, 0xd93d7256, 0xbd86c8c3, 0x817fdcce, 0x43899081, 0x817f1693, +0xc0db9a68, 0x7f57a5a8, 0xd14c817f, 0x8c4ab37a, 0x901081dd, 0x292a452b, 0x7f81e85a, 0x189ece81, +0xc10b9401, 0x81817f69, 0x358b3d7f, 0xa86bb1a9, 0xbd7f7f8a, 0x57fd7fe3, 0x81964506, 0x74295986, +0xab7fbc8d, 0x817f62e6, 0xab816277, 0x5737ab20, 0x9c7f6f50, 0x7f5781af, 0xe344ef7f, 0x98a695ca, +0x81d2a20e, 0xa77f6605, 0xc1d00398, 0x177a90d5, 0x7ff6db7f, 0x0221897f, 0x6c8105c6, 0x578981d7, +0x64ac81c3, 0x1d9381f9, 0x7f706e7f, 0xd5797f81, 0xf9567f81, 0x27ed81d0, 0x496f2d01, 0x48b8973b, +0xb34c1f32, 0x7f42139a, 0xeac95332, 0xf05f7783, 0x037f8145, 0xa98f3dbd, 0xc5ea24a8, 0x81417fc1, +0x81655e97, 0xcbc682df, 0xec10815a, 0x817f7fa5, 0xb8b54dff, 0x00003e84, 0xa7ece56d, 0x819d81c2, +0xd0ee6d81, 0x0e5e4db0, 0xe3818181, 0x7f4a2c13, 0xb13d46ac, 0xc181f081, 0xfe51c8ab, 0xa7747fb6, +0xc2506129, 0x9d98ce4a, 0x81627f81, 0x816d6481, 0x81576b7f, 0x817f814f, 0xc881cc8b, 0x686d098f, +0xae7fcb7f, 0x3b39a87f, 0x81888879, 0x47877f30, 0xd9bd9e9e, 0x061a0ee3, 0x7f3631a3, 0xbf5b227f, +0x7f7f837f, 0xc5d18162, 0xa3405318, 0xc3f27fac, 0x7f43c6f6, 0x927f7fad, 0x817f7fdd, 0x7f813b8b, +0x7fe59a65, 0x81fd87d7, 0xda54547f, 0x814d114a, 0x7e7f813d, 0xf47f077f, 0xa0869981, 0x38db7fea, +0xf61986b0, 0x7f9146cb, 0x81db9bb7, 0xfc3b8177, 0xbd7ff9a1, 0x7f33c5aa, 0x74335381, 0x815ca42b, +0x7f814a81, 0x49977342, 0xab9ed775, 0x9874db81, 0x6381227f, 0x1b81b27f, 0x817f3716, 0xd8e6bf07, +0x81030494, 0x56cedf81, 0x7f607f9a, 0x7f7f4f52, 0x3cb978f2, 0xd57fca81, 0xd7bc5c81, 0x3a2265b7, +0x7f7f98bf, 0xac068158, 0xa8851581, 0x1db9a67f, 0x7f24d44f, 0x57a34093, 0x7529b981, 0x50a3817f, +0x4c3f6381, 0xbd7f81ad, 0x2281a481, 0x1d2e7f81, 0x77b85621, 0x3f818105, 0x307f783f, 0x7f7fe0b2, +0x4d4a297f, 0x8d7f4d7f, 0x88818a7f, 0x129cf8b7, 0x9b817f7f, 0x7fe5ce65, 0xa48e7213, 0xf481a737, +0xb43d4fd4, 0x7fb182c3, 0xa18181ba, 0x077f8127, 0x4f4b468a, 0xe1bea062, 0xc34d0000, 0xc0f381ba, +0x8181817f, 0xbb2d9e7f, 0x7f216b7f, 0x868181ad, 0x7f7fcdf6, 0x810b3d55, 0x627f4c43, 0x81903e81, +0x54ea9a4d, 0x3181ae1e, 0x7f8116d0, 0x74b5c181, 0x607b431b, 0x5552d081, 0x7f818122, 0xe9449f4c, +0x6fb434b5, 0x1f1b9552, 0x54d7b178, 0x8b1b9e33, 0x4143cb33, 0x13b0817f, 0x7f51317f, 0x81247f65, +0xa6fd7b18, 0xb1d6550e, 0xaa6481c1, 0x81421f41, 0x50c0813c, 0x95ac7eec, 0x442a7981, 0xc39289ea, +0x667fcc5d, 0xf681817f, 0xd729c236, 0x5342819b, 0x813f457f, 0x819d2481, 0x817facd0, 0x7ff2dd9a, +0x81cafb81, 0x26c72930, 0x177ba325, 0x81015e07, 0xc5667f7f, 0x81fa497f, 0x7cb3505e, 0x85e3815d, +0x67596b53, 0x857f6696, 0xc97f7f66, 0x7a813978, 0x5f7f7f81, 0x043d9d81, 0x8c81a6a6, 0x0e8197f1, +0xd17664d1, 0x9f81814d, 0x7f4c4f3e, 0x5e7f756a, 0x8181815a, 0xe090817f, 0x81377f81, 0xaba9813d, +0x8985c047, 0x2106c97f, 0xc846f2b2, 0xb19a83ce, 0x64107f16, 0x7f792fde, 0x87516244, 0x9c81673e, +0x2f5c8187, 0x555a8181, 0x95d9f26a, 0x70408e9b, 0x7f81b57f, 0x8f0de19b, 0xd37f7a6e, 0x9ba9fa7f, +0x7f0b6568, 0x1a4f7f14, 0x3b4f5b71, 0x814e377f, 0x8125a7ec, 0x1d817f43, 0x811128dd, 0x66816435, +0x9de99e3e, 0x81b27f48, 0x7fc3783f, 0x69983781, 0x457a817f, 0xd4296caa, 0x0000b659, 0x723a92a9, +0x81860888, 0x8c81c44c, 0x7489b15c, 0x817310ed, 0x74a48e9c, 0xc6531a73, 0x7f7f64d0, 0x502d7fb9, +0x437f9f7f, 0xb1bfa444, 0x81df09f0, 0x0946ab9b, 0x7f21ca81, 0x9d818166, 0x81e8d281, 0x917f6791, +0x36ac811b, 0x9d817347, 0x344c02a7, 0xab36817f, 0x81818152, 0x32013a89, 0xd7ab3008, 0x6f608181, +0x817f7f7f, 0x153e0881, 0x81dfcaba, 0x6c5e577f, 0x2f7f6524, 0x817aa07c, 0xdd7f0175, 0x8b4b7fec, +0xbc81ead5, 0xae8b205d, 0xc9a9743e, 0xf33f14cf, 0x49ca7f58, 0x507f6976, 0x8cbe66d4, 0x9778a724, +0x6d8147e3, 0x81334092, 0x7f4c0101, 0x82d5ea73, 0xb59c6657, 0x1ab842f8, 0xbc1d8581, 0x4004d917, +0xe5eaaee2, 0x7fa9813b, 0x817fae58, 0xd77f1281, 0x10732f61, 0x836d137f, 0x7fbccb3b, 0x1a3a81d7, +0x81047fef, 0x790aa87f, 0xc19eaac6, 0x60667f07, 0xb7f1a97f, 0x39f39e7c, 0x057f7f7f, 0xaa7f81c8, +0x79bef7a0, 0xf3d3134d, 0x720f7fbd, 0xb4a5b69e, 0x716c817f, 0x9a817fdf, 0x347f817f, 0x4b995cb4, +0x876a4f7e, 0x8d92687f, 0x7f9e8181, 0x8172817f, 0x8ea21cc3, 0x61bd96b0, 0x81fba378, 0x6db77f7f, +0xdf307181, 0xda0f81c4, 0xa8e36d65, 0x70a7a559, 0x0be0f27f, 0xb56f4971, 0x35813a6d, 0x188130b5, +0x517ff881, 0xec54b6ff, 0x8183817f, 0x20ec2381, 0x3e7f9613, 0x878af66d, 0x81657f81, 0x7fb80000, +0x7cb38d5b, 0x817f7f45, 0x7f6352c1, 0x3881727e, 0x637f5ec2, 0x81437fc6, 0x7f7f90d2, 0xb1558699, +0x2b81904a, 0x7f84ad54, 0xedab2281, 0xbe9cbd81, 0x7f5001d1, 0xea2a206c, 0x813cf1ab, 0x81658221, +0x81a6817f, 0x8167929a, 0x726dd87f, 0x817f817f, 0x93b45a87, 0x7fa179e2, 0x7f4c7f81, 0x8157817f, +0xa2fd7e81, 0x44284a58, 0x3b98c224, 0x45008124, 0x16da978d, 0xb772ba81, 0x7a4f3481, 0xe97fb9c8, +0x7f49529e, 0xaa02580d, 0xa254adb3, 0xac0a817f, 0xdc532c32, 0x7fe6887f, 0x7f313d7d, 0x61d17f11, +0x81e1bf02, 0x7f62c97f, 0x348d5d50, 0x12ec7f7f, 0x8f7f2884, 0x7f3f371e, 0x021c8e81, 0x65a07fc7, +0x5e14b632, 0x817ff481, 0x6681de78, 0xba4f971c, 0x65c07f81, 0xac81d181, 0x7f11b17f, 0xdcc1011d, +0x7f7f23c1, 0xcc819b6a, 0x81a73d06, 0x5e943954, 0x09c8b87f, 0x43405fe8, 0x817f8d58, 0x550f4d1d, +0x82c27254, 0xd0408532, 0x4460477f, 0x8b7f9cec, 0x7fc9817f, 0x8181165f, 0x28d8817a, 0x0527752f, +0x8168813f, 0x7f48cc81, 0x7481c91e, 0x6881c481, 0x4b0a7f55, 0xcc81e624, 0xe57fac7f, 0x6f0f81a0, +0x417b5a81, 0x9381dd8c, 0xa881c4e5, 0x8c505264, 0x695ab14c, 0xecc99781, 0x3981c57f, 0x0c204553, +0x4a77d59c, 0xa2301c81, 0x5ba47f00, 0x451eb2f0, 0xb0459baf, 0x934bcc63, 0x5652c0a1, 0xb381b193, +0x68626b5b, 0x361741d5, 0x7fe281b3, 0x70ad7f7f, 0x8161f792, 0xbfda5b21, 0xed97ad84, 0x7f59cb7f, +0x8ed1ecf7, 0x75d784ac, 0x0f81b07f, 0x3ed5bb6b, 0x819122a9, 0x2257a06b, 0x81687fb1, 0xc008057f, +0x97812981, 0xcb2081cd, 0xd9bb6681, 0x407f7f7f, 0xc27f814e, 0x23a2527f, 0x767203ad, 0xed17ee7f, +0xd4b89558, 0x8195bc81, 0x817f7fe0, 0xe4b7997f, 0x081ca76d, 0xb87f817f, 0x006350bb, 0x812739de, +0x557f811e, 0x8d4387af, 0x98815781, 0x81818148, 0xb1bc8105, 0x04cf273e, 0x3f2b289b, 0xae96437e, +0x819281b9, 0x95cf7f82, 0x7bf77f81, 0xab7f0441, 0x81652854, 0x34de0d81, 0xac816081, 0x43b12e45, +0x91b75bdd, 0x7da27794, 0x9bbb727f, 0x301d812e, 0x7fa88120, 0x27a15075, 0x27c82cc1, 0xef1a813a, +0x8114c4d2, 0x817b7f2b, 0xff877f47, 0x7fd47fcd, 0xd08ceb35, 0xe07f4386, 0x67b3a47d, 0xad7fd64d, +0x817f7044, 0x537feabb, 0x8d3f36ec, 0x6c3883ce, 0x43b6909d, 0x81bcd181, 0xfa7fdae9, 0x3c7f5440, +0x3946347f, 0x7f22ac7f, 0x815dea81, 0xb6c6c8a1, 0xd57f81c9, 0x526fd581, 0xce3c3115, 0x3d41c54f, +0x5c5ac2b0, 0x7c948586, 0xa47f7f96, 0x8b7fc351, 0x309aadf9, 0xa0cfef81, 0xb081acd7, 0xd7413cb2, +0x7f4a6cc9, 0x5c3bb39f, 0x7fc7e581, 0x68cc8114, 0x0926a5fa, 0x82c37581, 0x8181e642, 0x57ab2ec6, +0x497f0000, 0x5fa07f9c, 0xa17fc405, 0xeaa0de7f, 0xa14a7fb3, 0xc4c12a81, 0xa3527d15, 0xa854d4d4, +0xf781a3db, 0xa07f26ae, 0x812c7f74, 0x1f7f2fc4, 0x3a9381d4, 0x17b6005d, 0xd414ee60, 0x18b49d68, +0x9cc77f9e, 0x30aa7f91, 0x7fb9207f, 0x64158b40, 0x81423c7f, 0xea2fa9cc, 0x7fa83381, 0xca647f8e, +0xd6b9657f, 0x81b26a7a, 0x4fb7b2f1, 0x1b7f9eda, 0xfb8181ab, 0x7f2557d7, 0x71e8701c, 0xa2817f81, +0x81817f9e, 0x04139d39, 0xc9f6533f, 0xa1f74db1, 0x814042db, 0xca6e8114, 0x7fb3f64a, 0x03f981a0, +0x7f9e0755, 0x7f7ba781, 0xd554d19a, 0xd81a5661, 0x7fb33f3a, 0x8f81c8bf, 0x012ff975, 0xb1a576ee, +0x81be6681, 0x818db381, 0xbd7f8151, 0x81b281ae, 0xddd2fbdc, 0x81bfa81c, 0xce818181, 0x774c9c28, +0x7fd05d59, 0x811a54ce, 0x2f7f727f, 0xbe816b81, 0x7fb03af2, 0xc44bfc7f, 0xcf6ba181, 0x14be8161, +0x6715a197, 0x76386d26, 0x07835f81, 0x817c818d, 0x81693281, 0x818118fa, 0x777c9e62, 0xccad6446, +0x769c5b8b, 0xdbd781b7, 0xe4b063f9, 0x2b568182, 0x8c7f8187, 0x7f7f677f, 0x6b7fa675, 0x6be85f61, +0xc36f7d32, 0x7f81005f, 0xd0d0997f, 0x3a818181, 0x7fa7b6b7, 0xc39ef17f, 0xcc7f437f, 0x3d51729d, +0x3eb72475, 0x1b069d81, 0x7fe8abee, 0x604c7df9, 0x7fb47f81, 0x3a263b75, 0x75a488e6, 0x7f68d47f, +0x00004e97, 0x7ffb0973, 0x7f53a009, 0x7f014834, 0x6f81022c, 0xf0b1405f, 0x06c84ca5, 0x4bcf7fc9, +0x4388ec9a, 0xf528fa74, 0x8e3b816f, 0x7f7f4c72, 0x817f5224, 0xbd998181, 0x81b181c6, 0xbd817f88, +0x1763f57f, 0x7a796e44, 0xe98e81a9, 0x5360a2fa, 0x97b39081, 0x707f1b65, 0x81c67f7f, 0x4e461c7f, +0x817f231b, 0xc2d08181, 0x366081c6, 0xd2e67f90, 0x13f1b87f, 0xd781819f, 0x7f347fa2, 0xb1147fc5, +0xe6b1fe4e, 0x3a1a96b2, 0x177f8191, 0x3e3d96fe, 0xd0d261a0, 0xb25e7fc9, 0x472b81ce, 0xbad6817f, +0x707b45ca, 0x81817f81, 0xc48fb7f8, 0x90373f82, 0xa681e0b5, 0x71419190, 0x6978f77f, 0xf4463125, +0x48f1937f, 0x4a45c611, 0x7f97e348, 0xf07f1e21, 0x81fbfc60, 0x8783a281, 0x00931cdd, 0x81e70f91, +0xe24e7f7f, 0x7f61817f, 0x81967c47, 0x6d9bc90e, 0xec58817f, 0x8157357f, 0x7f7fc674, 0x7f7e3c81, +0xaa8b7fbc, 0x7f3ba469, 0x7a22f57f, 0x7f7f7f27, 0x81c89e7f, 0x8145d9c8, 0x012f737f, 0xe91f15cc, +0x9b9cb749, 0x818142cd, 0x290a7f4a, 0x89cfc4b1, 0x8337ebe2, 0xd8224205, 0x8181a49e, 0x58b04ea1, +0x7fea95a7, 0x861681fb, 0xcea37f5f, 0x81309c8f, 0x6ded8145, 0xbd5be86e, 0xc2ce9402, 0x557f57be, +0x4ef830e9, 0x7f81f97f, 0xf804367a, 0x5de6817f, 0x156de096, 0x2c06df0c, 0x98b170d7, 0x817f81a9, +0x24597fd4, 0x81810000, 0x0e24a66a, 0x819e5681, 0x6f73e481, 0x054b532a, 0xc0814026, 0xb3a36181, +0x6da37f9a, 0x7f83574f, 0xe0a381c6, 0x2075bf7f, 0x28438181, 0xb1cd7f74, 0x7ffb497f, 0xd17f337f, +0x46085c73, 0xd57f5681, 0x704b7ff3, 0x449e3f7f, 0xeb6e09d6, 0xb37f4b02, 0x817f1cb9, 0x672edf7f, +0xd0c2c25b, 0x5ed0885b, 0x4676876b, 0x2bd8d970, 0x9a7fbbc6, 0x81095581, 0x81b27f6b, 0x753c7f7f, +0x7f812c7f, 0x7f477fbe, 0xdaa67f7e, 0x7f099859, 0x7b0fea70, 0x816b8186, 0xf0677f5c, 0x16455d9b, +0xa98170a9, 0xa181529b, 0x7f817f67, 0x8f72c351, 0x2a7f7f7f, 0x2f5f347f, 0x695c7f79, 0x282fbbe0, +0xb78581f0, 0x817f297f, 0x49818c81, 0x817fc78c, 0x3bd68181, 0x86d7deda, 0x7fa0d208, 0x1c7f3481, +0xf5be9a57, 0xb9ded881, 0x8ab88c81, 0x647ae23a, 0xb5571142, 0xe09d7fc1, 0xa5d9817f, 0xaca07f7f, +0x7f4d5286, 0xa57fde7f, 0x94d0917f, 0xb160442b, 0xacf47b2b, 0xd17f7f81, 0x2e37f976, 0xbe7f5d66, +0xc2815f28, 0x5c81487f, 0xc2026481, 0x81a194dd, 0xba7f2c91, 0xb7357f37, 0x81b25698, 0xd2818154, +0x45b78130, 0xc6bead58, 0x6f5a5656, 0xb181a6ae, 0x07c9a153, 0x75d681d3, 0x8c92817f, 0x81503520, +0xa66f812f, 0xa59b7a81, 0x17a09164, 0x17087fb2, 0xd0bb319b, 0x6d8153ae, 0x8af1a67f, 0x651c46b0, +0x4b637f1b, 0x00002981, 0x2c7f437f, 0xd9deb6ab, 0x9fba847f, 0x6f289b21, 0x015f811c, 0x8176f498, +0xeac003ae, 0x68818192, 0xb37fa6a6, 0xd07f4aa5, 0x83819472, 0xa5da41a0, 0xbc7fbe7f, 0xa67fccd9, +0x7f0c86c5, 0x116e7f15, 0x7f67b45a, 0x907fcf81, 0x81812c66, 0x05747283, 0xac962b7f, 0x2a7f81ea, +0x81c2b57f, 0xb1b4b868, 0x7f811ed1, 0x7f66c937, 0xc881e6ab, 0x740323fd, 0xcf765745, 0x3592d370, +0x5c81f3e5, 0x527f3efc, 0x818781d9, 0xf6819b53, 0x1c547f51, 0x4e29b37f, 0x36f1537f, 0xcb819758, +0x538f5b7f, 0xa5172c22, 0x52427f7f, 0x6b7f8128, 0x7186134a, 0xf67f1b62, 0x6081e015, 0x829abe81, +0xaa8a65fc, 0xd27fb70f, 0x7f0a0881, 0x3d7fc998, 0x443e78b0, 0x9443c354, 0x81266724, 0x9b81812b, +0x813181a0, 0x7fddca68, 0x7f260f81, 0x7a818181, 0x812681ca, 0xf48152e0, 0x817f8388, 0x81306c7f, +0x7f71d020, 0x81bd8481, 0x5234d5c2, 0x31be8136, 0x3403d432, 0x11286844, 0x81a57f1e, 0xe3b57fd4, +0x7aa88f52, 0xabc64c52, 0x977f8181, 0x7fe68ca6, 0x5dd6eac3, 0xd26a81a9, 0x017f8183, 0x902cc266, +0x7fba7f52, 0x617f81cf, 0x7f657ff0, 0x045d7f63, 0x3c04bb4e, 0xaa4e7f81, 0x847fe385, 0x67817f7f, +0x42f33d28, 0x61a5aa5a, 0x817f7f1e, 0xa3a715d7, 0x5da91c3a, 0x7f9f7f81, 0xc48281ce, 0x813081fa, +0x7f817e53, 0x6d227dbf, 0x6e7b0000, 0xf49b4091, 0xf6b3d8e9, 0x89fb29ae, 0x44289881, 0x3cc78181, +0xe58103b9, 0x535bffd3, 0x817fa81b, 0x469f82f0, 0x57837fe1, 0xe14af07f, 0x54bf19b2, 0x7482595d, +0x6781bbd2, 0x4b7b7fb7, 0xa4c45060, 0x380155c8, 0xb72aa381, 0x4530847f, 0xcb509581, 0x34818181, +0xb03cd87f, 0x90818122, 0x812a8181, 0x817f677f, 0x2c2a597f, 0x35f0c581, 0x81b11261, 0x4d678120, +0x816d7f5b, 0x448166d8, 0x0c567f61, 0x81189e21, 0xcf932f08, 0x4c5a917f, 0x687f9b7f, 0x257f81d2, +0xcc7f6220, 0x8c947f81, 0x24642d1f, 0x728155d3, 0x14ca75d2, 0x817d7f38, 0x7f818181, 0x7fb31f87, +0x13a581b3, 0xd9b5215f, 0xa8afdbf5, 0x147fe67f, 0x81caab97, 0xa96daa7f, 0xf88b897f, 0xe7937fc2, +0x5a8b7f4b, 0x7d59277f, 0x8d687f9d, 0x7f924b81, 0xd41c82c0, 0xa5119c7f, 0x4fcfeba1, 0x3e077f81, +0xaf65994c, 0x83d481a7, 0x81257fb0, 0xa481387f, 0x36d8a463, 0x8181811a, 0xc75643a7, 0x474c6c1a, +0x7fc0e8cf, 0x8cdc896e, 0x55830655, 0x7f81337f, 0x7f65db1d, 0xa881cd81, 0x7f812ec8, 0x7f76cdd4, +0x634a443d, 0xea7f3ed9, 0x90a8d093, 0xb05e819c, 0x7f7f5c4e, 0x811e5b81, 0x73a1b989, 0x8181717f, +0x927d7f09, 0xdeaf7fca, 0x528e302a, 0x81654681, 0x7fe92e81, 0x87d58110, 0x1ca86e20, 0x7291a4cc, +0xc76c7f81, 0x577f5136, 0x00005b6e, 0x817fe37f, 0xf7ac7f7f, 0xf258ee81, 0x81814c63, 0x2a97f57f, +0x5f7ca0a7, 0x7f2fa37f, 0x94360c7f, 0x7f23117d, 0x81343a7f, 0x55b0d9de, 0xbae0189e, 0xc77f228a, +0x817f7861, 0xa57f7fac, 0x878ede24, 0xb57fad5a, 0x5c907f93, 0x814e9b81, 0x7febf59c, 0x7724e1a9, +0x569d9281, 0x1b644699, 0x4d4e26bf, 0x397f7f6e, 0x8ac93027, 0xe6885974, 0x51a2c1a4, 0x7f7f5b7f, +0xaad8ac97, 0x122c8186, 0x97810a5b, 0x818581e2, 0xc77f7f35, 0x67611c60, 0x3f8178ea, 0xa47fa65e, +0x888f8132, 0x81817cd4, 0x1081b4bd, 0x817f7fb8, 0x449a3fcc, 0x7f9832de, 0x51e96f58, 0xcbde52c0, +0x6b515783, 0xbda3b297, 0xd64b6381, 0x7f693c81, 0x6d311abb, 0x73468e81, 0xde81817f, 0x4f811581, +0x90d6817f, 0x152c9d49, 0xad7f17d7, 0x7f834681, 0xa47fe6dc, 0x06817fb3, 0x7f817f69, 0xd18190b1, +0xca7f8183, 0x1881f381, 0xac50814a, 0x7f8dca7a, 0x71358131, 0x28783add, 0xd31b153b, 0x71919b97, +0x81877f07, 0x707fca4f, 0x57c6814b, 0xc0a17fbf, 0xff5bb344, 0x4a7fa99d, 0x7434c104, 0xaa7ff0a5, +0xb6ec6841, 0xb4917e1b, 0x5a81ab94, 0x81a3aab1, 0x257fce30, 0xe26b7fc7, 0x1d7a7fe5, 0xfc7ca6c0, +0x81437ff7, 0x7dc59e14, 0x7f0cf9e9, 0x90e37f7f, 0x7f6f12bc, 0x52d36381, 0xc15f88ac, 0x3536277f, +0x8d3e2a7f, 0xa18d7a7f, 0x48907f23, 0x7f810000, 0xfd813357, 0x6ed4520b, 0x843907e2, 0x66814378, +0x7f7f6bd8, 0x0e3381ad, 0x03847f90, 0x05814769, 0x8181c181, 0x1c7f7fac, 0xe83cbfb5, 0xb15c735c, +0x7f8f7f7f, 0x7f846b81, 0x81d2eb7e, 0xf7447f41, 0xa2b57f81, 0x2ba7f381, 0x817f75bc, 0x78553181, +0xc7b07f7f, 0x7f557a41, 0x46a83ddb, 0x9b447d13, 0xccfe7b81, 0x7f3ae281, 0x75464bfb, 0x7f4e5560, +0x7fad14ce, 0x98fd2a7f, 0x7f637f81, 0x4a22cf5c, 0xca81813e, 0x4ffc7347, 0x9d777fa5, 0x810585ff, +0x71e57f81, 0x3f7f8181, 0xfce8819c, 0xbb315486, 0xf1e5a974, 0xf1e061dc, 0xcd7f8181, 0x8181817f, +0x547fb97f, 0xa663c341, 0x9e5aab7f, 0x35768dad, 0x817f3b48, 0x46bc815f, 0x707f89b3, 0x860713c8, +0x4a7c6532, 0x541084e8, 0x5f77bb7f, 0x299dbebe, 0xd3f27f8c, 0x25096727, 0x81194cba, 0x7ceff409, +0x9c8c267f, 0x7f7b1fe0, 0x635c0fdd, 0x308132ac, 0x57330951, 0x7c257f81, 0x6266dd49, 0x7f89b481, +0x817f7fae, 0x7f81aebd, 0xc7fb7f7f, 0x4537e27f, 0x1d288b7f, 0x7f4a9b7f, 0x8150969f, 0xf533f121, +0x7f782217, 0xe428816e, 0x10ca6981, 0x6981b39f, 0xe35fdb7f, 0xc6814220, 0x79ba8158, 0x811e815d, +0x772d8184, 0xc784810c, 0xa91abf8f, 0x3c7f6f7f, 0x6d9981bf, 0x9b42dbe9, 0x42948181, 0x5681817f, +0xdba39bf2, 0xf77f7f37, 0x18eebbdb, 0x39c6b7b7, 0xdc2fb5b9, 0x99d00a7f, 0x02dbe492, 0x4481606e, +0x7f6f81a1, 0x7f2f557f, 0x2aee7f6d, 0xed2ad2ae, 0x6ce4ec0c, 0xdb4cae3a, 0x81646c83, 0x7fa3c1be, +0x7f383730, 0x81aeba9d, 0xc0528bdc, 0xd17f3982, 0x7fa47c7d, 0x81814e81, 0x813d81bc, 0x20816349, +0x857fdfe9, 0xe46aa581, 0xe8de53b5, 0x34471a81, 0x3b4f7feb, 0x8174c981, 0x2fc7c735, 0xc818ec81, +0xb28ed3df, 0x627f1100, 0x7f40067f, 0x7fd37fa8, 0x637c5b81, 0x113dd969, 0x81376881, 0x5d817f67, +0x7f7fda7f, 0x7f7f7f70, 0x77577fb4, 0x87bb3d7f, 0x5b817f93, 0x839b517f, 0x7f45818e, 0xe47f3381, +0x93c2b281, 0x944207f3, 0x36dc77b3, 0x50b86e4a, 0x8181e27f, 0x0b81a79b, 0xf9975950, 0x977b9e47, +0xea7f8181, 0x5547b549, 0x3da47f0d, 0x474b8e72, 0x7f7f9ce1, 0x7f2e2e42, 0x6f7f45c9, 0x08d56ede, +0x8524810b, 0x657fd15f, 0x7f8d0c14, 0x6c7fff7b, 0xd530f39d, 0x7fb4b023, 0x651d8a1b, 0x13817024, +0x7f850774, 0x1f219db3, 0xd8817f52, 0xd95c0feb, 0x9ef0e67f, 0xd9dd4c7f, 0x64b0819a, 0xbc60e997, +0x412fd0db, 0x817fa57f, 0xa3a47f81, 0x4d81815b, 0x81874e3a, 0x5f1e7c81, 0x7fc63b81, 0xbf817581, +0xb5b0a5ab, 0xde2c8145, 0x81789629, 0x097481b8, 0xad5bdca3, 0x212d81e5, 0x4d817f3c, 0x70813761, +0x5d6d0f40, 0x81a190eb, 0xc942c14d, 0x81b462d8, 0x81890000, 0x6586fbcf, 0x81cb3322, 0xa5c0b931, +0x7f78e98d, 0x2492625b, 0xaf81b981, 0x4e59817f, 0x677fb9d3, 0x2c3b817f, 0x920e5981, 0x7f2c7f5d, +0xc379d081, 0xdfbddf43, 0x85818172, 0x6781e442, 0xda9a7f14, 0xcac70881, 0x98818113, 0x0f817f6f, +0xeba852c8, 0x7fa6407a, 0x7fdc8188, 0x819fd349, 0xee738137, 0xe881417f, 0xb6777fea, 0xc6947f73, +0x8181240f, 0xa035c264, 0x7f419e81, 0x193f815f, 0xaedc812e, 0x7fb97f24, 0x924a81e6, 0x63f58183, +0x3e4ec808, 0x7f4fbe86, 0x5faba653, 0xf0c4f4d7, 0x69cf647f, 0xfb55c681, 0x7fe1e381, 0x7f817f6b, +0xc0410481, 0xc65d71b6, 0xba1f8135, 0x997f04cf, 0x49db5b1c, 0xbe817443, 0x817b812c, 0x36e18163, +0xd1893a47, 0x231df6f2, 0x816d045e, 0x3a568139, 0x81628181, 0x6588ffd2, 0x23603081, 0xf5b4817d, +0x8112b5bc, 0x7f9cfbae, 0xc02b8d40, 0xc7f20532, 0xce81487f, 0x0b1a6614, 0x7f17392f, 0x7f8f818c, +0x7f7f8388, 0x5492467a, 0xbf5e3081, 0x777f26bb, 0xae9c4a7f, 0xae81ba0e, 0x6c6281ae, 0x81238143, +0x7f997ffe, 0x798b949c, 0xc3d67f36, 0x38817ff9, 0x2f814ab6, 0x594bad4b, 0x7f6f7fd6, 0xab234181, +0xaf818581, 0x77865381, 0x9d7f8173, 0x27ac8181, 0xbdbd8daf, 0x7c7f7f7f, 0xc6817fd9, 0x34412181, +0x1fe7eb78, 0x7f38f49b, 0x81040b8f, 0x34ec8581, 0x0000907f, 0xd5818181, 0x7865c6d5, 0xada81975, +0x0447e955, 0x7f3d09b3, 0x2b8156e8, 0x976d81dd, 0x29cee57f, 0xa34f537f, 0xe9a981f6, 0x0c7fe841, +0x7f7eda7f, 0xd77f1881, 0x3f929481, 0x77e3c852, 0x0d817fc1, 0x7f4795ab, 0xd1337f4f, 0xdd4b077f, +0x97a4b3d8, 0x607ff155, 0x942b16c3, 0x113a723c, 0x52ce7882, 0x5e81f5e9, 0x81755669, 0x7fea4d5f, +0x5d6d7f81, 0x7f90e710, 0xe3817f6d, 0xe810299c, 0x4e814937, 0x3c269d81, 0x33a891aa, 0x81dd7ff3, +0xa9d1db0d, 0xfb2181eb, 0x7f817fa4, 0x75ec817c, 0xf17f8195, 0x81e18181, 0x44321c58, 0x35a62913, +0xcd5b0875, 0x397f8233, 0x797f7f81, 0x7f829cbc, 0x9a697f81, 0x07b7fea9, 0x7d2b0b7f, 0x8f984dce, +0x0c13b97f, 0xe9203f3c, 0xea473769, 0x7b7fdc81, 0x7f7885c1, 0xde7f5b39, 0x32bd81f1, 0x67b4917f, +0x443069cb, 0x6e972b83, 0xbc66829f, 0x7492817f, 0x7f4d7481, 0x7f25896f, 0x9ea0da7f, 0xceba7f81, +0x7f45d87f, 0x93dbf04c, 0x50814197, 0xb7167f81, 0x6d7fad19, 0x0f81bea5, 0xc57f8177, 0xca4c7f7f, +0x7d84e3f6, 0x7bc443d4, 0xd7b0ae81, 0xc8731f44, 0x06e481ac, 0x85b08181, 0x53cfe4be, 0x73761a3d, +0x707f9981, 0xa87ff27f, 0x68b5b87f, 0xb561c081, 0x7f660001, 0x7b81521e, 0x6093614b, 0xbc778748, +0x64400f81, 0x81d13281, 0x60475381, 0xbe768181, 0x21aad606, 0x81910000, 0x20b6a511, 0x5953d899, +0xdd7f7fba, 0x8181fabe, 0x81b8e77f, 0x817ff17f, 0x6757257f, 0x9c2cc933, 0x51813593, 0x7f7fadef, +0x8ec47f43, 0x348139d7, 0x7f5c82b7, 0x664419bc, 0x8d7b15a5, 0x8181a581, 0xd54d7f63, 0x7f0381b7, +0x81a17f68, 0x814a5e0d, 0x17813dc4, 0xbd94872d, 0xa25a8101, 0xa6810e54, 0xd8568fc4, 0x7f7f3a7f, +0x818478ba, 0x517f81c7, 0x70443b81, 0x7ce481b6, 0x7f817f7f, 0x81b27fe5, 0x4bc67f32, 0x81bbe681, +0xa1c8fe81, 0x857f1c0d, 0x7f81793a, 0x7fbfcf74, 0xadd47f5c, 0x7f8138b4, 0x6f63cd25, 0xd70e497f, +0x81aa7fc7, 0xbaa84f5a, 0x8197ccd3, 0x7f556b7e, 0xca815dff, 0xb9ba7f7f, 0x81747f7f, 0x7fdf967f, +0x65a98181, 0x81bc8181, 0x8e11bab7, 0xbbbf7ffd, 0x12b8457f, 0x59fe8481, 0xf25cdd81, 0x958f840c, +0x5e6a5f81, 0xe9008181, 0x7a68cb38, 0x76818914, 0xa3857fc7, 0xce98717f, 0x8126504d, 0x7fa48183, +0xfa60e14a, 0x7fb77f22, 0x5f3a7f28, 0x24910bde, 0x2fd07f4f, 0x64a6810b, 0x818181a4, 0x7a065fb8, +0x7fe69795, 0x45815e3b, 0x7fa98741, 0x52c581b8, 0x74bb2a22, 0x817a8166, 0xc2cb6581, 0x493c7f7f, +0x883e816b, 0x3c7f7f7f, 0x7f3381c1, 0xab767f6d, 0x81b4588c, 0xb2e24cb9, 0xd9bccba1, 0x70e0162c, +0xea1c810a, 0x907fdd49, 0x424aece8, 0xd90423b0, 0x22df1b28, 0x6b005d25, 0x113e61b3, 0xa2103bbe, +0x20d6c538, 0x17e6c841, 0xdc08e2c4, 0x2002b1ef, 0xc51a77e3, 0x8111f752, 0xea698110, 0x1849b6e8, +0xe6b006d2, 0xdc40b507, 0xc738b9ec, 0x42290a3f, 0x522355bc, 0x590736d3, 0xd9a340e5, 0xe30a19b5, +0x3de9eff9, 0x4437cc7f, 0xecfd0a7b, 0x34e4fcc7, 0x4afe1556, 0x1ecb301f, 0x0f511d1d, 0xf8ec40db, +0x4c07caf9, 0x59f73ebd, 0x45064328, 0x03c27f19, 0xd1661dcb, 0xe4e3ae17, 0x000fcd5b, 0xdd121ec8, +0x3adb1413, 0x112b9888, 0x1d3624fd, 0xa8a347f6, 0xfeb632d5, 0x1d28fc11, 0x21ac2b5f, 0x371fe9f7, +0x3d5ef435, 0xe84000a1, 0xaebebcc9, 0x34555ff0, 0x39a9ca2a, 0x580fc0e6, 0xcf7a5699, 0xf5e65242, +0x07ae3170, 0x06b223ea, 0x638e421b, 0xd8dc0cab, 0x66c44fb0, 0x954117d7, 0x9691a163, 0x415bc8a1, +0x0868d844, 0x9593f551, 0x364bec3b, 0x57460bfb, 0xdc310e3e, 0xe4dbef28, 0xc6443317, 0x81a02211, +0x450c2323, 0xefca9dcf, 0xfebf3fbf, 0xafef377c, 0x3f81ac26, 0x13b4e548, 0xf60a05a9, 0x03d4d840, +0x1970ecd3, 0x95019cfd, 0x8128341d, 0xc4d8d440, 0x16b80a75, 0xc2332b1d, 0x47e90e1b, 0x2e4eadcc, +0x0aa5f04b, 0x03302c42, 0x1a1ed437, 0x3b623efa, 0xc1cec834, 0x4442d130, 0xdce837f0, 0xc3f90934, +0x5ea50731, 0x257f1504, 0x2da5defd, 0x22d3dd34, 0xc429fa2b, 0xf2cc0a0b, 0x3e2e0000, 0x020f1c20, +0x2e962a1f, 0x06edf5f3, 0xceb5dfe4, 0x3b213d57, 0x08c24237, 0xf32ad940, 0x321768b0, 0xb133ebcd, +0xfca78110, 0xc03001db, 0xfca42f7f, 0x7071645f, 0x30bf8185, 0xd1f6d479, 0x02076839, 0x34cd900d, +0x500cda81, 0xe542b08e, 0x282afac6, 0x30a835ce, 0xbc710153, 0x574b009c, 0xe8b24309, 0x02d2cbc8, +0xaae1ea5a, 0x190eff7f, 0xbc656b07, 0xfdc8094e, 0xc0ee4452, 0x2a819969, 0xb79a0a0e, 0xe334a040, +0x12323acf, 0xd9617f3c, 0x3709af42, 0x8b67b338, 0xff00e313, 0x5046e8de, 0x45ec1107, 0x4e2618c9, +0xb40a62dd, 0x0da39629, 0x0a62a937, 0xc9d92e1d, 0xcf4daf81, 0x3f26a9a3, 0xd01e3f2b, 0xac294211, +0xd7c221e7, 0x15c928f2, 0xd79412cb, 0x110fb1fc, 0xddd3fa22, 0x4e7fa8ad, 0xd4a0ebf8, 0x5b144a01, +0x4cf90bdc, 0xb43366e7, 0x1cb8503e, 0xff152c3a, 0xae04fd5f, 0xd896041e, 0x1c2ca245, 0xd7adf0e3, +0x36ee2de1, 0xf0350023, 0x2e529db9, 0xba18d1bc, 0xbb2aba00, 0x52c7c4fb, 0x09df6147, 0x1102f72e, +0x13054b2f, 0xf9f9fced, 0x292a14c9, 0x60095adf, 0x8b43fec1, 0x3ca5a3c9, 0xe73665c7, 0x2f12b049, +0xb0fadecf, 0xd0e46081, 0x42532cf0, 0xe420e3fe, 0xaf5b0501, 0x27f0b481, 0x871c0ec6, 0xcd1c8cb7, +0xe4135816, 0x0a2aab7f, 0xbf732d1d, 0x613f1aee, 0xac09b927, 0x7d663532, 0x0000dee4, 0xfdeccc4a, +0x34b86af1, 0xf0bb16dd, 0x5829085f, 0x51ffdcc2, 0xd50c9f39, 0x64dfc723, 0x2cd52b2b, 0x083e0b2b, +0x224ae1b5, 0x0fd63733, 0xf081cef7, 0x7f73e3c4, 0x36cca9d4, 0xcc2600ae, 0x0757af1e, 0x21cc2ef7, +0xf175c93f, 0x59221d2d, 0xe6c7f468, 0xe3ea1fcb, 0xcd3d539a, 0x271e345c, 0x3555341c, 0xd2532b7f, +0x7fc8c57f, 0x4025dbe5, 0xf6926f3d, 0x264d39db, 0x2ed7c504, 0xde2a554b, 0x25aeb624, 0xd031a1dc, +0x47dbd918, 0xa0004941, 0x0202e11c, 0xd407fda2, 0xd9bd0fdb, 0x30f94d53, 0xd32b1cc6, 0xa6cb1947, +0x16fa1b49, 0xc70d1f56, 0xa7daec0d, 0x813d4649, 0x0f520200, 0x0d174704, 0x337f521a, 0x52c2cf34, +0x6b280539, 0x81da4508, 0x0ef2a91d, 0x07121df9, 0x031e410e, 0xf0180925, 0x055022b0, 0x2156f018, +0xa2e812c6, 0xfbe05cbb, 0x232e4462, 0xed378bca, 0xcd31afa7, 0x063a2e09, 0xe836ff1b, 0xefdeaf26, +0xfeef274a, 0xc9a1159d, 0x66db4731, 0x05a50ace, 0x1b4c58fd, 0x6b0ba273, 0x251319b8, 0xc7523041, +0xa20364b7, 0x34812fc4, 0x055f5d29, 0x5858fefa, 0xf809192d, 0x2f063981, 0xce056e8e, 0x1f7e68cc, +0xbbc07f8b, 0xcc1c1240, 0xf45c1452, 0xeb1deac8, 0xe3f9d3fe, 0x12bc2a12, 0x515bd8d4, 0xb3b361d6, +0x3304f1dc, 0x63060dbb, 0x41c8c73d, 0xce985d1d, 0xd2b2544b, 0x6d241fd4, 0xb6b8b1c5, 0x21830000, +0x15efc4c3, 0x2c430fc1, 0x14400bef, 0x2c33083e, 0xbe04fb07, 0xad42d311, 0x4ed2b731, 0x1627d2d2, +0x07aa0f34, 0xfe38df20, 0xe35bcd1b, 0xe13bfbcf, 0xcc3df1c9, 0x84355a1c, 0x55e602d2, 0x47c52a81, +0x16d85858, 0x0d11d8d2, 0x54f4b081, 0x303523ad, 0x2dbc16e3, 0xb9d6f0d7, 0x13f76190, 0xeec0133a, +0x81eccebc, 0x46202d2b, 0x86b52303, 0xa1f928ba, 0xe421b651, 0xb13cf63f, 0x3136c8e6, 0x4a8128b9, +0x2d7f231a, 0x262cf3e3, 0x9c35d3f9, 0xe9e6753a, 0xefeff441, 0xd4bcd301, 0x42700bd5, 0x29ab001b, +0x28bb6207, 0x01f537d1, 0xbc57fd38, 0x4999290e, 0x3e7f04fe, 0x36adb7fa, 0xdb0314ce, 0xa1d6524e, +0xeee5c994, 0x1d1ce32e, 0xee142708, 0x3bf9e663, 0x06521283, 0xe8c6e6a9, 0x53356ee3, 0xe2cd5d4e, +0x8e1bdacc, 0x382c18e2, 0xc5e2fe00, 0xcee2eaf8, 0xb7532ece, 0xfbea01bf, 0x3ecf7f2d, 0x06b2ffaf, +0x5acbfe35, 0x26c8eefc, 0x8e27d35b, 0x84291501, 0x32f82ffa, 0x9f3a10cc, 0xabcff198, 0x139a3da9, +0x3bd92af7, 0x264febe8, 0xbf48f81b, 0xa1df1aa2, 0x2c0a10ca, 0x04c00073, 0x5681e760, 0x411054e5, +0x88d8c5a0, 0xe3f6b6fd, 0x2a3030ea, 0x0f0ec8b9, 0xa827cbe8, 0x2a2c7f49, 0x51ef20d2, 0xa48c4a81, +0x2b8db1d9, 0xd150813e, 0xd7aab201, 0x36061afd, 0xeed80a8d, 0xcd9932d0, 0xedd3edbe, 0x38f4f017, +0x9853f907, 0xc964f464, 0xc68d40f0, 0x3f5bcc44, 0x975004ec, 0x102d467f, 0xf3f9451d, 0xcde706bb, +0xc87fa602, 0x3f5a88b0, 0x8119c002, 0x3cc14ac8, 0x607abeb7, 0xf95924cd, 0xfb1afa2c, 0xc4967f36, +0xf425dacb, 0xa32d09aa, 0x35106590, 0xad4e5b4d, 0xa1fda650, 0x0326e143, 0xfbf054e0, 0x589fe1bc, +0xc83249b4, 0xb0b81e1b, 0x7081fec8, 0x27530027, 0x84dc89ca, 0x76551d3a, 0x0bbcc3d9, 0x8108bade, +0xb534f6cc, 0x87833920, 0x07ec1b0e, 0x5fd30b29, 0xf124dd07, 0xc981b03d, 0x71b9417e, 0x476c76b6, +0x7fcb3662, 0xaec632e9, 0x2e6705dd, 0xaec5acf9, 0xc84cd6de, 0x5312d428, 0x12fc4738, 0xd31d5b1e, +0x61edd156, 0x60fdfddb, 0xff676a0c, 0x2a3baad6, 0x0d1f6eab, 0x0001dcce, 0x180a31ca, 0x1ba681ec, +0x90871091, 0xe2ef01f5, 0x35186658, 0x95cac76a, 0x1ae32a09, 0xebedc8df, 0x0da7c007, 0x274fc73f, +0xd62a1ff5, 0xd6d7403a, 0xc4921d32, 0xb5f6cadc, 0xdd34f021, 0xe9334bfe, 0x5c9256c1, 0xbecbfd00, +0x139acdb2, 0xe3913eab, 0xceed1c3b, 0xdd2c5aa6, 0xfb475d17, 0x4633f4e7, 0x373b4b4f, 0x72ee99f3, +0xb65542d3, 0x48eb1d39, 0xa6d32fae, 0xc20aabe4, 0x105f4c0d, 0x4848bb65, 0x3bd329d2, 0x470c4709, +0x43fb2438, 0x0620c0d7, 0x4f962821, 0xfd2f992c, 0x51f8aea5, 0x2521cfe9, 0xbf6e2f9d, 0x005e3653 + +hard_output0 = +0xb5bc6ac8, 0xf5373664, 0x1310345c, 0xd5bae4e7, 0x1fc9e83e, 0xebfdfded, 0x84bd86ab, 0xb7aabe00, +0x60b44fea, 0xb9067464, 0x30325378, 0xa9195955, 0xf70c6e5c, 0x90922632, 0xc90b1cdb, 0xf2f5fb69, +0x73056b63, 0x1a33bf3f, 0x17755b5c, 0xc58bff6d, 0x2f4390b2, 0x2869d508, 0xe7c7dfe8, 0x38552963, +0x21da5367, 0x07282b9b, 0xa4767105, 0x1e294251, 0xe350a940, 0xb8a6aa27, 0xed12d778, 0xf10d9ece, +0xab93527f, 0xcf2da7e7, 0x68f6d0b1, 0x811f4bca, 0x577b06b2, 0x3234f13e, 0x30bab7df, 0x8dc47655, +0xbb843bed, 0x86da3aba, 0x30950c97, 0xdd096d7a, 0xa871fd6c, 0x8bee4e6b, 0x8fea30d0, 0x6c05b4d2, +0xf3e144d3, 0xd24ebb1f, 0x065635e5, 0x8d3f2cf9, 0x536c6c6a, 0xfbb0a5d0, 0x3d707b42, 0xc44d5982, +0xa5f4ad8f, 0xf32c0970, 0x1bccf1a6, 0x05916020, 0xa64fb176, 0x5ede6a35, 0xaf4966da, 0x9df5e0e7, +0x75042abc, 0x9ef10481, 0x11ddcbc8, 0xa0f5518c, 0xd5c23418, 0x2393d558, 0xfbe7dfeb, 0xed1c64c2, +0x86a36508, 0xde2dfb1e, 0xb8d0fef9, 0x24505232, 0xc894e71c, 0xbcc752a0, 0x40b74e83, 0x90d23c8c, +0x728e4a61, 0x108f0b08, 0x66f522ee, 0xc258d851, 0x35a31c44, 0x11311b5b, 0xfd3d5be9, 0x5ae448ff, +0x4f64994b, 0x5b8247a9, 0x4021114d, 0x2f0b6e82, 0x5eaa9828, 0x50ac71c0, 0xfb86ee52, 0x0dc1ac9b, +0xbbd47645, 0x8f357115, 0x978ceea0, 0xd557db99, 0x99b30388, 0xfc9a8a1c, 0x0f75be1a, 0x50143e22, +0x8840989b, 0x738ec50e, 0xe6b2783d, 0xf67899c8, 0x27ebed69, 0x6c415a16, 0x3a6cc2dc, 0xcd4e4e5d, +0x6cb12b2e, 0xdb88d7c0, 0x79cd1582, 0xbc422413, 0xe72ad2f4, 0x8eaac30f, 0x0bd86747, 0x6d87f69d, +0x15d62038, 0x4b375630, 0x0d51b859, 0x16db2cb2, 0xf210603a, 0x0abeb833, 0x55c694d0, 0xe57ca43b, +0x0ba94428, 0x1398a406, 0xe47d3889, 0x5a20203d, 0x250d7a1a, 0xd930ffec, 0x03992e79, 0xf2759376, +0x024ec121, 0x91fc3a2c, 0xb7e11cc5, 0x4ff7d459, 0xb8700134, 0xd6e61758, 0x4eba0a32, 0xb747e3ec, +0x7073fad7, 0xded80f99, 0x331e2f1b, 0xfa1f1bed, 0x056424a2, 0x1d1d95e0, 0x550b9ec8, 0x51ee2a38, +0x19525153, 0xd70c4cd5, 0x0d6cd7ad, 0xe44d1cf2, 0x30dfecda, 0xdacd7fe8, 0x7321d795, 0xddf48ef4, +0xe271e6a4, 0x9c1feecb, 0x951fcd7b, 0x8acc5a03, 0x3fb83527, 0xe306de74, 0x7b9cd6ee, 0x8e140885, +0xd4c91e8d, 0xe8c39733, 0x0f02f87f, 0xfb06b1b9, 0x0dc9349c, 0xf76bae8e, 0x4f642a07, 0x3d48a9aa, +0xe3ea323a, 0xa1cd5c8a, 0x40aa0e70, 0x132042d3, 0xa9732f6c, 0xd15a00c4, 0x43d3b046, 0x9a51ebd4, +0xc46ee0ed, 0xe2a2148b, 0xf5c478f0, 0x1fb01cf3, 0xf4f321ec, 0xd973811f, 0x11ad11b9, 0x5c67adda + +soft_output0 = +0x81818181, 0x7f7f7f7f, 0x7f7f7f81, 0x7f817f81, 0x7f7f8181, 0x7f7f7f7f, 0x817f817f, 0x81817f81, +0x7f817f81, 0x7f817f81, 0x7f81817f, 0x7f7f7f7f, 0x7f7f8181, 0x7f81817f, 0x817f8181, 0x81817f81, +0x7f7f7f81, 0x817f7f81, 0x7f7f8181, 0x7f817f81, 0x7f81817f, 0x817f7f7f, 0x817f8181, 0x7f7f8181, +0x81817f7f, 0x81817f7f, 0x81817f7f, 0x817f7f7f, 0x7f7f7f7f, 0x817f7f81, 0x7f7f817f, 0x7f817f7f, +0x7f818181, 0x8181817f, 0x81818181, 0x7f817f7f, 0x7f7f817f, 0x81817f81, 0x7f7f8181, 0x817f817f, +0x7f817f81, 0x7f7f8181, 0x817f7f81, 0x81818181, 0x817f817f, 0x81817f7f, 0x7f7f8181, 0x7f7f8181, +0x8181817f, 0x817f7f81, 0x81818181, 0x817f8181, 0x7f7f817f, 0x7f7f7f81, 0x7f81817f, 0x7f81817f, +0x7f81817f, 0x8181817f, 0x7f7f7f81, 0x817f7f81, 0x7f817f7f, 0x7f7f7f7f, 0x8181817f, 0x7f81817f, +0x817f7f81, 0x7f7f7f7f, 0x7f7f7f7f, 0x7f7f8181, 0x7f7f8181, 0x7f817f7f, 0x7f81817f, 0x7f7f817f, +0x81817f81, 0x81817f81, 0x81817f81, 0x817f817f, 0x7f817f7f, 0x7f81817f, 0x81818181, 0x8181817f, +0x7f817f7f, 0x817f7f7f, 0x7f7f817f, 0x817f7f81, 0x7f7f7f81, 0x81818181, 0x817f8181, 0x7f7f7f81, +0x7f817f81, 0x7f817f81, 0x81817f81, 0x7f7f7f7f, 0x7f7f8181, 0x81817f7f, 0x7f7f8181, 0x7f7f817f, +0x7f7f817f, 0x81817f7f, 0x8181817f, 0x817f8181, 0x81817f81, 0x817f7f7f, 0x817f7f81, 0x7f81817f, +0x7f7f7f7f, 0x7f817f81, 0x7f7f817f, 0x7f817f7f, 0x81818181, 0x817f7f81, 0x7f817f81, 0x7f817f81, +0x817f7f81, 0x7f818181, 0x7f81817f, 0x817f817f, 0x7f7f817f, 0x81818181, 0x7f818181, 0x8181817f, +0x817f7f81, 0x7f81817f, 0x7f7f7f7f, 0x81817f81, 0x817f8181, 0x7f818181, 0x817f8181, 0x81817f7f, +0x7f7f8181, 0x81817f81, 0x7f7f817f, 0x817f817f, 0x81818181, 0x7f81817f, 0x817f817f, 0x81818181, +0x817f8181, 0x817f8181, 0x817f8181, 0x7f817f81, 0x7f7f7f81, 0x7f7f8181, 0x7f817f7f, 0x7f7f817f, +0x7f817f81, 0x7f81817f, 0x817f7f7f, 0x7f7f8181, 0x7f817f7f, 0x7f7f8181, 0x7f7f7f81, 0x817f7f7f, +0x7f817f7f, 0x817f817f, 0x7f817f81, 0x7f817f81, 0x7f7f8181, 0x81817f81, 0x8181817f, 0x81817f81, +0x817f7f81, 0x7f818181, 0x7f7f8181, 0x817f7f81, 0x7f817f81, 0x817f7f7f, 0x7f7f8181, 0x81817f7f, +0x7f7f7f7f, 0x817f8181, 0x81817f81, 0x7f7f7f7f, 0x817f817f, 0x7f7f7f81, 0x7f7f7f81, 0x81817f7f, +0x8181817f, 0x817f8181, 0x81817f81, 0x7f7f7f81, 0x7f7f7f81, 0x81818181, 0x81817f81, 0x7f7f8181, +0x8181817f, 0x7f817f81, 0x7f817f81, 0x7f817f7f, 0x8181817f, 0x8181817f, 0x81817f81, 0x817f8181, +0x81818181, 0x7f817f7f, 0x81817f81, 0x7f817f81, 0x7f817f81, 0x7f817f7f, 0x8181817f, 0x817f8181, +0x817f7f81, 0x817f7f81, 0x7f817f81, 0x817f8181, 0x7f7f7f81, 0x7f81817f, 0x81818181, 0x8181817f, +0x7f7f8181, 0x7f7f7f81, 0x81817f7f, 0x7f818181, 0x817f7f7f, 0x7f817f7f, 0x7f7f7f81, 0x7f817f7f, +0x8181817f, 0x7f81817f, 0x817f7f7f, 0x817f817f, 0x7f7f7f7f, 0x7f817f81, 0x8181817f, 0x7f81817f, +0x7f81817f, 0x81817f7f, 0x817f7f81, 0x7f817f81, 0x7f81817f, 0x7f7f7f7f, 0x81817f81, 0x81817f7f, +0x81817f7f, 0x81817f7f, 0x7f818181, 0x8181817f, 0x7f7f8181, 0x7f818181, 0x7f7f8181, 0x7f7f8181, +0x817f7f7f, 0x81818181, 0x7f818181, 0x7f7f8181, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, 0x7f7f817f, +0x7f7f8181, 0x7f817f7f, 0x7f817f7f, 0x817f8181, 0x7f7f817f, 0x81817f81, 0x817f8181, 0x7f7f817f, +0x81817f7f, 0x7f7f7f81, 0x7f81817f, 0x7f817f7f, 0x7f81817f, 0x7f7f817f, 0x817f817f, 0x81817f7f, +0x817f817f, 0x817f7f81, 0x81818181, 0x81817f7f, 0x817f8181, 0x817f8181, 0x817f7f7f, 0x7f7f7f7f, +0x7f81817f, 0x7f818181, 0x817f817f, 0x7f818181, 0x7f7f7f7f, 0x7f7f7f7f, 0x8181817f, 0x7f7f7f81, +0x7f7f7f7f, 0x7f7f8181, 0x7f817f81, 0x7f818181, 0x81817f81, 0x7f7f7f81, 0x817f7f81, 0x8181817f, +0x817f817f, 0x7f7f7f81, 0x7f818181, 0x81817f81, 0x817f7f81, 0x7f7f7f7f, 0x817f817f, 0x81817f7f, +0x817f7f7f, 0x81818181, 0x8181817f, 0x81817f7f, 0x7f7f7f7f, 0x817f8181, 0x7f817f81, 0x817f8181, +0x817f7f7f, 0x7f7f7f81, 0x817f8181, 0x81818181, 0x7f81817f, 0x817f7f7f, 0x7f81817f, 0x817f817f, +0x7f817f7f, 0x7f818181, 0x7f818181, 0x7f817f7f, 0x81817f7f, 0x7f817f7f, 0x7f7f817f, 0x81817f81, +0x81817f81, 0x817f7f7f, 0x81817f7f, 0x81817f81, 0x7f818181, 0x7f7f7f7f, 0x7f818181, 0x7f7f817f, +0x7f817f81, 0x81817f81, 0x7f7f7f81, 0x817f8181, 0x81818181, 0x7f81817f, 0x817f7f7f, 0x7f817f7f, +0x817f7f81, 0x7f7f817f, 0x7f7f8181, 0x7f7f8181, 0x81818181, 0x7f81817f, 0x7f81817f, 0x817f8181, +0x7f818181, 0x817f7f81, 0x817f7f7f, 0x7f817f81, 0x7f817f7f, 0x817f7f7f, 0x817f7f81, 0x7f817f81, +0x8181817f, 0x7f81817f, 0x81818181, 0x7f7f7f81, 0x81817f81, 0x7f818181, 0x7f7f7f7f, 0x7f81817f, +0x8181817f, 0x8181817f, 0x7f7f7f7f, 0x7f818181, 0x817f7f81, 0x7f7f817f, 0x817f7f81, 0x81818181, +0x817f817f, 0x7f818181, 0x81817f81, 0x81818181, 0x817f7f81, 0x7f817f81, 0x7f7f8181, 0x817f817f, +0x817f7f7f, 0x817f8181, 0x81817f81, 0x81818181, 0x817f817f, 0x7f817f7f, 0x7f817f7f, 0x7f817f81, +0x817f7f7f, 0x81817f81, 0x817f7f81, 0x7f7f817f, 0x7f817f7f, 0x7f81817f, 0x8181817f, 0x81817f81, +0x81817f7f, 0x7f817f81, 0x7f81817f, 0x81818181, 0x7f7f817f, 0x817f7f7f, 0x7f817f81, 0x8181817f, +0x817f8181, 0x7f81817f, 0x81817f7f, 0x7f817f81, 0x81818181, 0x7f81817f, 0x7f81817f, 0x7f81817f, +0x7f7f817f, 0x7f817f7f, 0x7f81817f, 0x817f7f81, 0x7f81817f, 0x81818181, 0x7f7f8181, 0x7f818181, +0x8181817f, 0x81818181, 0x8181817f, 0x817f817f, 0x7f81817f, 0x81817f7f, 0x7f81817f, 0x7f817f7f, +0x81817f81, 0x7f81817f, 0x7f7f817f, 0x817f8181, 0x81818181, 0x7f817f81, 0x81817f81, 0x7f817f7f, +0x817f7f7f, 0x7f7f817f, 0x7f818181, 0x817f8181, 0x7f7f7f81, 0x81817f7f, 0x81817f7f, 0x7f7f7f7f, +0x7f818181, 0x7f7f8181, 0x7f81817f, 0x7f7f817f, 0x8181817f, 0x7f81817f, 0x8181817f, 0x81817f7f, +0x817f7f81, 0x817f817f, 0x7f7f7f7f, 0x817f8181, 0x7f7f8181, 0x817f817f, 0x7f817f7f, 0x81818181, +0x817f7f7f, 0x7f817f81, 0x7f7f7f7f, 0x817f8181, 0x7f7f7f81, 0x7f7f8181, 0x81817f81, 0x81817f7f, +0x7f818181, 0x817f7f81, 0x7f818181, 0x817f817f, 0x7f7f7f81, 0x817f817f, 0x7f817f81, 0x7f7f7f81, +0x7f818181, 0x7f7f817f, 0x81818181, 0x817f7f81, 0x81817f7f, 0x8181817f, 0x817f8181, 0x8181817f, +0x7f818181, 0x7f817f81, 0x8181817f, 0x7f817f7f, 0x8181817f, 0x7f817f7f, 0x7f7f7f81, 0x817f7f7f, +0x817f8181, 0x7f7f7f7f, 0x817f8181, 0x81817f81, 0x81817f7f, 0x7f7f8181, 0x81817f81, 0x7f7f7f7f, +0x7f818181, 0x7f818181, 0x7f7f7f7f, 0x8181817f, 0x7f817f7f, 0x81817f81, 0x81817f81, 0x7f7f817f, +0x7f7f7f7f, 0x817f817f, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f817f, 0x7f817f81, 0x81818181, 0x817f8181, +0x7f7f817f, 0x81817f7f, 0x7f7f7f7f, 0x7f81817f, 0x81817f7f, 0x7f817f7f, 0x81817f81, 0x7f7f7f81, +0x817f8181, 0x81818181, 0x7f81817f, 0x7f817f7f, 0x7f7f7f81, 0x7f818181, 0x817f7f7f, 0x7f7f7f7f, +0x817f7f81, 0x7f818181, 0x7f817f7f, 0x7f7f8181, 0x7f7f8181, 0x81817f81, 0x81817f81, 0x7f7f7f81, +0x8181817f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x817f7f81, 0x7f7f8181, 0x7f7f7f81, 0x817f7f7f, +0x7f817f81, 0x81818181, 0x81817f81, 0x7f81817f, 0x7f7f7f81, 0x817f817f, 0x8181817f, 0x7f818181, +0x817f817f, 0x81817f7f, 0x7f7f7f7f, 0x7f817f7f, 0x8181817f, 0x7f7f817f, 0x7f7f7f7f, 0x7f81817f, +0x7f817f7f, 0x7f7f8181, 0x7f7f8181, 0x7f7f8181, 0x81817f7f, 0x7f7f8181, 0x7f7f8181, 0x8181817f, +0x817f7f81, 0x7f81817f, 0x7f817f7f, 0x8181817f, 0x7f7f817f, 0x7f7f8181, 0x7f7f8181, 0x7f817f7f, +0x817f817f, 0x7f81817f, 0x817f7f81, 0x8181817f, 0x81817f7f, 0x81818181, 0x817f817f, 0x7f7f817f, +0x817f8181, 0x7f817f7f, 0x7f817f7f, 0x817f817f, 0x81817f81, 0x7f7f7f81, 0x7f81817f, 0x7f7f7f81, +0x8181817f, 0x817f7f7f, 0x7f7f817f, 0x7f817f7f, 0x7f7f7f81, 0x7f817f7f, 0x7f7f8181, 0x817f817f, +0x7f7f8181, 0x7f7f7f81, 0x817f8181, 0x7f7f8181, 0x81817f81, 0x8181817f, 0x7f7f817f, 0x7f817f81, +0x7f817f81, 0x7f817f81, 0x7f817f81, 0x817f8181, 0x7f7f7f7f, 0x7f817f81, 0x817f7f81, 0x7f7f7f81, +0x7f7f817f, 0x7f817f7f, 0x7f7f7f7f, 0x817f7f81, 0x81817f7f, 0x81817f7f, 0x81818181, 0x7f81817f, +0x7f7f7f7f, 0x7f7f7f81, 0x7f817f81, 0x8181817f, 0x817f817f, 0x817f817f, 0x817f817f, 0x7f817f7f, +0x81818181, 0x7f817f7f, 0x7f7f817f, 0x81818181, 0x8181817f, 0x81817f7f, 0x81818181, 0x817f817f, +0x8181817f, 0x7f7f7f7f, 0x7f7f8181, 0x7f818181, 0x7f7f7f81, 0x817f817f, 0x7f7f817f, 0x81817f7f, +0x8181817f, 0x817f817f, 0x7f7f817f, 0x7f81817f, 0x7f7f7f81, 0x8181817f, 0x817f7f7f, 0x817f817f, +0x817f7f7f, 0x7f7f7f7f, 0x81817f81, 0x817f7f81, 0x7f817f7f, 0x81817f7f, 0x81817f81, 0x7f817f81, +0x817f7f7f, 0x817f817f, 0x7f817f7f, 0x817f7f81, 0x817f817f, 0x81817f81, 0x81818181, 0x7f817f7f, +0x8181817f, 0x817f7f7f, 0x81817f7f, 0x7f817f7f, 0x817f7f7f, 0x817f7f81, 0x7f7f7f81, 0x81817f7f, +0x817f7f81, 0x7f817f81, 0x7f7f8181, 0x7f81817f, 0x81817f7f, 0x817f7f7f, 0x81818181, 0x7f7f817f, +0x817f817f, 0x81817f81, 0x817f7f81, 0x81817f7f, 0x7f7f7f81, 0x7f817f7f, 0x81817f7f, 0x817f817f, +0x7f817f7f, 0x7f817f7f, 0x81817f7f, 0x817f7f7f, 0x81818181, 0x8181817f, 0x7f7f8181, 0x7f817f7f, +0x7f817f81, 0x7f7f8181, 0x81817f81, 0x817f7f81, 0x7f818181, 0x7f817f81, 0x7f7f7f81, 0x8181817f, +0x81817f7f, 0x817f7f7f, 0x7f818181, 0x7f818181, 0x7f81817f, 0x7f7f8181, 0x817f7f81, 0x7f817f81, +0x81817f7f, 0x81818181, 0x7f81817f, 0x7f7f8181, 0x817f817f, 0x817f7f81, 0x81817f81, 0x817f7f81, +0x81818181, 0x817f7f7f, 0x7f818181, 0x81818181, 0x7f7f817f, 0x817f817f, 0x7f7f817f, 0x7f817f7f, +0x7f817f81, 0x817f7f7f, 0x81817f81, 0x7f817f81, 0x7f817f7f, 0x817f8181, 0x817f817f, 0x7f818181, +0x817f7f81, 0x817f7f7f, 0x81817f81, 0x81818181, 0x7f81817f, 0x7f817f81, 0x7f817f81, 0x817f8181, +0x7f818181, 0x81817f81, 0x7f817f7f, 0x7f7f7f81, 0x7f7f817f, 0x817f7f81, 0x7f7f7f81, 0x81818181, +0x7f7f8181, 0x7f7f817f, 0x817f7f81, 0x81817f7f, 0x7f81817f, 0x7f81817f, 0x817f8181, 0x7f81817f, +0x8181817f, 0x81818181, 0x7f7f817f, 0x817f8181, 0x7f817f7f, 0x7f7f817f, 0x817f7f7f, 0x7f7f7f7f, +0x8181817f, 0x7f7f7f7f, 0x81817f81, 0x817f7f7f, 0x7f7f8181, 0x817f7f81, 0x7f7f817f, 0x7f81817f, +0x81817f81, 0x7f7f8181, 0x7f7f7f81, 0x7f817f81, 0x81817f7f, 0x7f817f81, 0x7f817f81, 0x7f7f817f, +0x81818181, 0x7f818181, 0x817f7f81, 0x817f7f7f, 0x7f81817f, 0x7f7f8181, 0x7f7f8181, 0x81817f7f, +0x7f817f81, 0x817f817f, 0x7f817f7f, 0x81818181, 0x7f817f7f, 0x81817f7f, 0x7f817f81, 0x81817f81, +0x7f7f7f81, 0x81818181, 0x81817f81, 0x817f7f81, 0x7f7f7f7f, 0x817f7f81, 0x7f818181, 0x7f7f8181, +0x7f7f7f81, 0x7f817f7f, 0x7f818181, 0x7f7f817f, 0x817f7f7f, 0x817f8181, 0x817f8181, 0x817f7f81, +0x7f7f8181, 0x7f817f7f, 0x817f8181, 0x81817f81, 0x817f8181, 0x7f81817f, 0x7f7f817f, 0x81817f7f, +0x81817f7f, 0x7f817f7f, 0x81817f7f, 0x81817f7f, 0x7f817f81, 0x7f7f8181, 0x7f7f817f, 0x7f7f7f7f, +0x81818181, 0x7f7f817f, 0x7f7f7f81, 0x7f817f81, 0x81818181, 0x81817f81, 0x817f7f7f, 0x7f7f817f, +0x817f7f7f, 0x7f7f7f81, 0x81817f81, 0x817f8181, 0x7f7f817f, 0x7f7f817f, 0x7f7f8181, 0x817f7f81, +0x8181817f, 0x8181817f, 0x7f7f7f81, 0x7f7f817f, 0x81817f81, 0x7f7f7f7f, 0x817f8181, 0x817f817f, +0x7f7f7f81, 0x7f817f7f, 0x7f81817f, 0x7f7f817f, 0x7f81817f, 0x817f7f81, 0x7f7f7f7f, 0x7f7f7f81, +0x81817f7f, 0x8181817f, 0x7f81817f, 0x7f7f7f81, 0x8181817f, 0x81817f7f, 0x7f818181, 0x7f817f81, +0x8181817f, 0x7f81817f, 0x817f7f7f, 0x817f7f7f, 0x8181817f, 0x817f817f, 0x81817f81, 0x7f7f8181, +0x7f7f7f7f, 0x7f81817f, 0x817f817f, 0x81817f81, 0x81817f81, 0x81817f7f, 0x81817f81, 0x81817f81, +0x7f7f817f, 0x7f7f7f7f, 0x817f7f7f, 0x7f818181, 0x7f817f7f, 0x8181817f, 0x7f7f8181, 0x817f7f7f, +0x7f7f7f81, 0x7f817f81, 0x7f7f7f7f, 0x7f817f81, 0x7f81817f, 0x7f817f7f, 0x81817f7f, 0x7f817f7f, +0x81817f81, 0x7f7f7f81, 0x7f81817f, 0x81817f81, 0x7f817f81, 0x817f8181, 0x7f817f81, 0x7f817f7f, +0x81818181, 0x817f817f, 0x81817f81, 0x7f7f817f, 0x7f7f8181, 0x81818181, 0x8181817f, 0x7f817f81, +0x81817f7f, 0x7f7f7f7f, 0x7f7f817f, 0x7f81817f, 0x81818181, 0x7f81817f, 0x817f7f7f, 0x7f817f81, +0x817f817f, 0x817f7f81, 0x81818181, 0x81817f7f, 0x817f7f81, 0x817f8181, 0x81817f7f, 0x81817f7f, +0x7f7f817f, 0x817f817f, 0x7f817f81, 0x7f817f81, 0x81818181, 0x817f817f, 0x7f7f7f81, 0x81817f7f, +0x817f817f, 0x7f7f7f7f, 0x7f7f8181, 0x817f7f7f, 0x817f7f81, 0x817f7f81, 0x817f8181, 0x8181817f, +0x81818181, 0x81817f7f, 0x81817f7f, 0x817f8181, 0x817f8181, 0x817f817f, 0x7f7f7f7f, 0x7f817f81, +0x8181817f, 0x7f818181, 0x7f818181, 0x8181817f, 0x7f7f8181, 0x817f8181, 0x7f817f81, 0x81817f7f, +0x7f7f7f81, 0x81817f81, 0x7f818181, 0x81817f7f, 0x7f7f817f, 0x7f818181, 0x81818181, 0x7f818181, +0x7f7f7f81, 0x81817f7f, 0x7f81817f, 0x7f7f8181, 0x7f81817f, 0x7f817f7f, 0x7f817f7f, 0x7f817f7f, +0x7f7f817f, 0x7f817f81, 0x7f81817f, 0x7f7f7f81, 0x81818181, 0x7f7f8181, 0x817f8181, 0x7f817f81, +0x7f817f81, 0x81817f81, 0x7f817f81, 0x817f7f81, 0x7f7f7f81, 0x817f7f7f, 0x7f81817f, 0x7f817f7f, +0x7f7f7f81, 0x817f8181, 0x7f81817f, 0x817f817f, 0x81817f81, 0x7f7f817f, 0x7f817f7f, 0x817f817f, +0x7f7f7f7f, 0x817f7f81, 0x817f8181, 0x7f7f7f7f, 0x7f817f81, 0x7f7f7f7f, 0x7f7f817f, 0x8181817f, +0x817f7f81, 0x7f818181, 0x7f817f7f, 0x8181817f, 0x7f7f817f, 0x7f817f7f, 0x817f817f, 0x817f7f7f, +0x817f817f, 0x81817f7f, 0x81818181, 0x7f7f7f81, 0x81817f7f, 0x817f7f81, 0x817f817f, 0x81817f81, +0x7f817f7f, 0x817f7f81, 0x7f7f7f81, 0x8181817f, 0x7f7f817f, 0x817f7f81, 0x817f7f81, 0x7f818181, +0x817f7f7f, 0x8181817f, 0x817f8181, 0x7f7f817f, 0x7f818181, 0x817f8181, 0x7f7f7f81, 0x817f8181, +0x8181817f, 0x7f817f81, 0x81817f7f, 0x817f8181, 0x817f817f, 0x817f7f81, 0x81818181, 0x81818181, +0x8181817f, 0x817f8181, 0x7f7f7f81, 0x7f7f8181, 0x817f817f, 0x7f818181, 0x817f8181, 0x7f7f817f, +0x7f7f7f81, 0x7f818181, 0x7f818181, 0x7f81817f, 0x81817f81, 0x7f818181, 0x7f7f8181, 0x7f817f7f, +0x8181817f, 0x7f817f7f, 0x817f7f81, 0x7f817f81, 0x8181817f, 0x81818181, 0x817f7f7f, 0x817f817f, +0x7f7f7f7f, 0x817f8181, 0x81818181, 0x817f7f7f, 0x817f7f81, 0x7f817f7f, 0x817f7f81, 0x81817f81, +0x7f818181, 0x7f817f81, 0x7f818181, 0x7f7f7f81, 0x81817f81, 0x7f817f81, 0x7f817f7f, 0x7f81817f, +0x81817f81, 0x817f7f7f, 0x817f817f, 0x7f7f817f, 0x7f7f8181, 0x817f7f7f, 0x7f7f817f, 0x7f817f7f, +0x7f81817f, 0x7f81817f, 0x817f7f7f, 0x7f817f7f, 0x817f7f81, 0x8181817f, 0x817f7f81, 0x7f818181, +0x7f818181, 0x817f7f81, 0x7f817f81, 0x7f7f7f7f, 0x81817f81, 0x81818181, 0x7f818181, 0x7f81817f, +0x81817f81, 0x817f7f7f, 0x7f7f817f, 0x7f7f817f, 0x81817f81, 0x7f817f81, 0x7f7f8181, 0x81817f7f, +0x817f817f, 0x81817f7f, 0x7f7f7f81, 0x7f817f7f, 0x7f7f7f7f, 0x817f7f81, 0x7f817f81, 0x817f7f7f, +0x7f7f8181, 0x7f7f817f, 0x8181817f, 0x7f818181, 0x7f817f81, 0x81818181, 0x817f817f, 0x7f7f7f7f, +0x81818181, 0x7f7f8181, 0x7f7f8181, 0x7f81817f, 0x8181817f, 0x8181817f, 0x81817f7f, 0x817f817f, +0x817f7f7f, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x817f8181, 0x81818181, 0x7f7f7f7f, 0x7f7f817f, +0x817f7f7f, 0x7f7f817f, 0x817f8181, 0x7f817f81, 0x7f818181, 0x7f81817f, 0x7f81817f, 0x81818181, +0x7f81817f, 0x817f7f7f, 0x7f7f8181, 0x81818181, 0x7f817f81, 0x81818181, 0x7f817f81, 0x81818181, +0x817f8181, 0x7f7f7f81, 0x7f818181, 0x7f7f817f, 0x817f7f7f, 0x7f7f7f7f, 0x817f7f81, 0x817f7f81, +0x7f81817f, 0x817f7f7f, 0x81817f81, 0x817f7f81, 0x81818181, 0x7f7f7f7f, 0x81817f7f, 0x81817f81, +0x81817f81, 0x81817f81, 0x7f817f81, 0x817f817f, 0x817f817f, 0x817f817f, 0x7f7f817f, 0x81817f81, +0x7f7f817f, 0x8181817f, 0x817f7f81, 0x81818181, 0x817f7f7f, 0x817f8181, 0x7f7f817f, 0x817f7f7f, +0x8181817f, 0x81817f7f, 0x7f817f7f, 0x817f7f81, 0x81817f81, 0x817f7f7f, 0x81818181, 0x817f7f7f, +0x81817f81, 0x7f81817f, 0x7f81817f, 0x7f818181, 0x817f7f81, 0x817f817f, 0x7f81817f, 0x81818181, +0x81817f81, 0x81818181, 0x817f7f81, 0x81817f7f, 0x7f81817f, 0x8181817f, 0x8181817f, 0x81818181, +0x81818181, 0x81818181, 0x7f7f8181, 0x817f7f7f, 0x7f7f7f7f, 0x81817f81, 0x817f7f81, 0x7f818181, +0x817f7f7f, 0x7f817f81, 0x7f7f7f7f, 0x7f7f8181, 0x7f81817f, 0x7f817f81, 0x817f7f7f, 0x81817f81, +0x7f7f7f7f, 0x817f7f81, 0x817f7f7f, 0x7f817f81, 0x81817f7f, 0x81817f81, 0x817f7f81, 0x81817f81, +0x817f817f, 0x7f817f81, 0x81817f7f, 0x7f7f7f81, 0x7f7f8181, 0x7f7f817f, 0x8181817f, 0x7f81817f, +0x817f8181, 0x7f7f7f81, 0x817f817f, 0x7f7f817f, 0x7f7f817f, 0x7f817f7f, 0x7f818181, 0x81817f7f, +0x81817f7f, 0x8181817f, 0x817f817f, 0x7f7f817f, 0x7f817f81, 0x7f817f7f, 0x7f818181, 0x7f81817f, +0x817f7f7f, 0x7f817f7f, 0x81817f81, 0x8181817f, 0x8181817f, 0x7f817f81, 0x8181817f, 0x8181817f, +0x81817f7f, 0x817f7f7f, 0x7f7f7f7f, 0x7f817f7f, 0x81817f81, 0x7f7f7f81, 0x817f7f7f, 0x7f817f81, +0x7f818181, 0x7f817f81, 0x81818181, 0x817f817f, 0x817f817f, 0x7f818181, 0x817f8181, 0x7f7f817f, +0x7f7f7f81, 0x817f7f7f, 0x7f81817f, 0x7f81817f, 0x7f81817f, 0x81817f81, 0x817f817f, 0x7f7f817f, +0x7f817f81, 0x817f817f, 0x7f817f7f, 0x7f81817f, 0x8181817f, 0x817f7f7f, 0x8181817f, 0x7f7f8181, +0x7f818181, 0x7f817f81, 0x7f7f817f, 0x817f8181, 0x817f817f, 0x817f7f81, 0x7f7f7f81, 0x81817f81, +0x817f8181, 0x81818181, 0x81818181, 0x7f7f817f, 0x7f7f7f81, 0x8181817f, 0x7f7f7f81, 0x7f7f7f81, +0x7f818181, 0x817f7f81, 0x7f817f7f, 0x81818181, 0x7f817f7f, 0x7f7f817f, 0x7f7f8181, 0x8181817f, +0x7f817f7f, 0x817f7f81, 0x7f7f7f81, 0x7f81817f, 0x81817f81, 0x7f817f81, 0x817f8181, 0x817f817f, +0x817f7f7f, 0x81817f7f, 0x7f818181, 0x81818181, 0x817f7f81, 0x81817f7f, 0x817f8181, 0x817f8181, +0x81817f7f, 0x8181817f, 0x7f7f817f, 0x7f7f817f, 0x7f7f817f, 0x7f817f81, 0x7f7f7f7f, 0x817f8181, +0x81817f7f, 0x7f818181, 0x7f7f7f81, 0x817f8181, 0x7f817f7f, 0x7f817f7f, 0x7f817f81, 0x817f8181, +0x817f817f, 0x7f7f7f7f, 0x81818181, 0x81818181, 0x81818181, 0x7f817f7f, 0x81818181, 0x7f81817f, +0x7f817f81, 0x8181817f, 0x817f7f7f, 0x817f817f, 0x81817f81, 0x7f817f81, 0x817f7f81, 0x81817f81, +0x7f7f8181, 0x7f817f7f, 0x817f7f7f, 0x7f81817f, 0x81818181, 0x7f7f817f, 0x7f7f7f81, 0x817f7f81, +0x817f7f7f, 0x7f817f81, 0x7f817f7f, 0x817f817f, 0x8181817f, 0x7f7f8181, 0x7f818181, 0x8181817f, +0x7f817f7f, 0x8181817f, 0x7f817f81, 0x7f81817f, 0x7f7f7f81, 0x7f7f7f81, 0x817f817f, 0x81818181, +0x7f817f81, 0x7f81817f, 0x7f7f8181, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f7f81, 0x7f818181, 0x7f7f7f7f, +0x7f7f8181, 0x817f8181, 0x7f7f7f7f, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, 0x7f7f7f7f, 0x7f7f8181, +0x817f7f7f, 0x81817f81, 0x7f818181, 0x81817f7f, 0x7f81817f, 0x7f817f81, 0x7f817f81, 0x7f817f81, +0x7f7f817f, 0x7f7f7f7f, 0x817f7f81, 0x7f7f8181, 0x8181817f, 0x817f7f7f, 0x7f817f7f, 0x817f8181, +0x7f817f7f, 0x817f8181, 0x7f818181, 0x81817f7f, 0x7f817f7f, 0x8181817f, 0x7f81817f, 0x7f7f7f7f, +0x7f81817f, 0x7f817f81, 0x817f817f, 0x817f7f7f, 0x7f7f7f7f, 0x7f818181, 0x817f7f7f, 0x7f81817f, +0x81817f81, 0x7f7f7f81, 0x7f818181, 0x7f817f81, 0x81817f81, 0x7f7f7f7f, 0x817f8181, 0x817f8181, +0x7f817f7f, 0x7f817f7f, 0x817f817f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x8181817f, 0x81817f81, +0x817f8181, 0x817f8181, 0x7f7f8181, 0x7f7f8181, 0x8181817f, 0x7f817f7f, 0x8181817f, 0x7f81817f, +0x7f7f7f81, 0x817f817f, 0x81817f7f, 0x817f7f81, 0x81817f7f, 0x817f7f81, 0x81817f81, 0x7f817f81, +0x817f8181, 0x7f818181, 0x7f817f81, 0x81817f81, 0x81817f81, 0x817f8181, 0x7f818181, 0x81818181, +0x817f7f81, 0x7f817f7f, 0x817f7f81, 0x817f817f, 0x8181817f, 0x81818181, 0x817f7f81, 0x7f818181, +0x81817f81, 0x7f817f7f, 0x817f817f, 0x817f7f81, 0x81817f81, 0x7f817f81, 0x7f7f817f, 0x8181817f, +0x817f7f7f, 0x7f7f817f, 0x7f817f81, 0x817f7f7f, 0x7f7f817f, 0x81818181, 0x817f7f81, 0x7f7f8181, +0x7f7f8181, 0x8181817f, 0x7f7f8181, 0x817f8181, 0x817f7f7f, 0x817f7f7f, 0x7f7f7f7f, 0x81817f81, +0x817f7f81, 0x7f7f8181, 0x7f817f81, 0x817f7f81, 0x7f7f817f, 0x817f8181, 0x7f7f7f81, 0x81817f7f, +0x817f817f, 0x7f7f7f7f, 0x817f7f7f, 0x7f81817f, 0x817f817f, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, +0x7f7f8181, 0x817f817f, 0x7f7f7f81, 0x7f7f817f, 0x817f8181, 0x7f81817f, 0x7f81817f, 0x817f7f7f, +0x7f817f7f, 0x817f817f, 0x817f7f81, 0x817f7f81, 0x81817f81, 0x7f817f81, 0x817f817f, 0x7f81817f, +0x7f7f8181, 0x7f818181, 0x7f818181, 0x817f817f, 0x7f81817f, 0x817f8181, 0x8181817f, 0x7f817f7f, +0x817f8181, 0x817f7f81, 0x81817f81, 0x7f7f7f81, 0x817f7f81, 0x7f7f7f7f, 0x81818181, 0x7f7f8181, +0x7f817f7f, 0x7f7f8181, 0x81817f7f, 0x817f817f, 0x81818181, 0x817f7f7f, 0x8181817f, 0x81817f81, +0x817f817f, 0x817f817f, 0x7f7f817f, 0x7f7f7f81, 0x7f818181, 0x7f817f7f, 0x7f7f7f81, 0x817f817f, +0x7f818181, 0x7f7f8181, 0x7f817f7f, 0x81817f81, 0x7f7f8181, 0x817f7f81, 0x817f7f7f, 0x817f8181, +0x7f7f8181, 0x7f817f7f, 0x817f7f81, 0x817f817f, 0x817f7f81, 0x81817f81, 0x817f8181, 0x817f7f7f, +0x7f7f7f81, 0x817f8181, 0x7f817f81, 0x7f817f81, 0x7f817f7f, 0x7f81817f, 0x7f7f7f7f, 0x81817f7f, +0x7f7f7f7f, 0x817f7f81, 0x7f7f8181, 0x7f7f7f81, 0x817f7f7f, 0x7f817f7f, 0x7f7f817f, 0x81817f7f, +0x8181817f, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x817f8181, 0x7f7f7f81, 0x7f817f81, 0x7f817f7f, +0x7f818181, 0x817f7f7f, 0x81817f81, 0x817f8181, 0x7f818181, 0x81817f7f, 0x817f8181, 0x7f7f8181, +0x81817f7f, 0x7f7f7f7f, 0x81818181, 0x7f818181, 0x817f817f, 0x7f7f7f81, 0x817f7f81, 0x817f8181, +0x7f81817f, 0x81817f7f, 0x817f8181, 0x817f817f, 0x7f818181, 0x7f7f7f7f, 0x7f818181, 0x7f7f817f, +0x7f817f81, 0x817f8181, 0x7f7f7f7f, 0x7f818181, 0x7f7f7f81, 0x7f7f817f, 0x7f817f81, 0x817f8181, +0x81818181, 0x7f818181, 0x7f7f7f7f, 0x7f818181, 0x7f81817f, 0x7f7f817f, 0x81817f81, 0x7f7f8181, +0x81818181, 0x817f7f81, 0x7f817f81, 0x7f7f8181, 0x817f7f81, 0x81817f81, 0x7f817f7f, 0x817f817f, +0x7f81817f, 0x7f818181, 0x817f8181, 0x81817f7f, 0x817f8181, 0x7f7f7f81, 0x7f818181, 0x7f7f817f, +0x817f8181, 0x817f8181, 0x7f817f81, 0x7f81817f, 0x8181817f, 0x817f7f81, 0x817f817f, 0x81818181, +0x7f817f7f, 0x817f817f, 0x817f817f, 0x7f817f7f, 0x7f817f7f, 0x817f817f, 0x7f81817f, 0x81818181, +0x7f7f7f7f, 0x7f7f7f7f, 0x817f7f81, 0x7f81817f, 0x7f7f7f81, 0x817f7f81, 0x817f8181, 0x7f818181, +0x7f7f7f7f, 0x7f7f817f, 0x7f7f7f81, 0x7f818181, 0x817f7f81, 0x81817f7f, 0x81817f7f, 0x81817f81, +0x7f7f7f81, 0x8181817f, 0x7f7f8181, 0x7f81817f, 0x7f7f7f81, 0x81817f7f, 0x7f7f7f7f, 0x7f7f817f, +0x7f817f7f, 0x81818181, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f7f7f, 0x817f817f, 0x81818181, 0x81817f81, +0x7f81817f, 0x7f7f8181, 0x7f81817f, 0x81818181, 0x81818181, 0x7f7f7f7f, 0x817f7f81, 0x7f81817f, +0x7f81817f, 0x7f7f8181, 0x7f818181, 0x7f818181, 0x7f817f7f, 0x8181817f, 0x81817f81, 0x817f7f81, +0x7f7f7f81, 0x7f81817f, 0x817f7f81, 0x817f8181, 0x7f81817f, 0x817f8181, 0x81817f7f, 0x7f817f81, +0x81818181, 0x7f817f81, 0x7f818181, 0x7f7f7f81, 0x7f7f8181, 0x8181817f, 0x81818181, 0x81818181, +0x81818181, 0x817f8181, 0x81817f81, 0x817f7f7f, 0x81817f81, 0x817f7f7f, 0x7f7f8181, 0x7f81817f, +0x7f7f7f81, 0x7f7f7f7f, 0x7f817f81, 0x81817f81, 0x817f817f, 0x7f7f7f81, 0x817f7f81, 0x817f7f7f, +0x7f817f7f, 0x81818181, 0x81818181, 0x7f7f817f, 0x7f817f81, 0x81817f81, 0x817f817f, 0x817f7f81, +0x81818181, 0x7f817f7f, 0x81817f7f, 0x81817f81, 0x817f817f, 0x817f7f7f, 0x7f7f817f, 0x7f7f817f, +0x7f7f8181, 0x817f7f81, 0x7f81817f, 0x7f817f7f, 0x817f7f7f, 0x817f7f7f, 0x817f7f7f, 0x7f7f7f7f, +0x7f7f7f7f, 0x7f7f7f81, 0x7f7f8181, 0x817f8181, 0x817f817f, 0x7f7f7f81, 0x81818181, 0x81817f81, +0x7f7f7f7f, 0x817f817f, 0x81818181, 0x7f7f817f, 0x7f818181, 0x7f7f8181, 0x817f8181, 0x817f7f7f, +0x7f7f8181, 0x7f817f7f, 0x817f8181, 0x817f8181, 0x81817f81, 0x7f817f81, 0x817f7f81, 0x7f7f7f81, +0x7f818181, 0x7f817f7f, 0x81817f81, 0x7f817f7f, 0x817f7f7f, 0x817f7f7f, 0x817f817f, 0x7f7f7f7f, +0x817f8181, 0x7f7f7f7f, 0x7f81817f, 0x7f817f81, 0x81818181, 0x817f817f, 0x817f7f7f, 0x81817f7f, +0x7f81817f, 0x817f817f, 0x7f818181, 0x8181817f, 0x817f817f, 0x81818181, 0x817f7f7f, 0x817f7f81, +0x7f7f8181, 0x817f817f, 0x817f7f81, 0x7f81817f, 0x7f818181, 0x7f7f817f, 0x817f7f7f, 0x7f7f7f7f, +0x7f7f7f7f, 0x81817f81, 0x7f81817f, 0x7f7f7f7f, 0x7f7f7f81, 0x8181817f, 0x81818181, 0x81817f7f, +0x7f81817f, 0x7f7f8181, 0x817f7f7f, 0x7f7f7f7f, 0x817f7f7f, 0x7f81817f, 0x817f817f, 0x81817f81, +0x7f7f7f81, 0x817f7f81, 0x817f7f81, 0x81818181, 0x7f7f8181, 0x81817f81, 0x7f817f7f, 0x7f818181, +0x7f818181, 0x81818181, 0x7f7f817f, 0x7f7f7f81, 0x81817f81, 0x817f7f7f, 0x817f8181, 0x7f7f817f, +0x81818181, 0x81817f81, 0x7f7f8181, 0x7f7f7f7f, 0x7f818181, 0x7f7f817f, 0x817f7f81, 0x817f7f7f, +0x81817f7f, 0x7f81817f, 0x7f817f7f, 0x7f81817f, 0x7f7f817f, 0x7f7f8181, 0x8181817f, 0x817f7f81, +0x7f7f7f81, 0x7f817f81, 0x8181817f, 0x7f7f7f7f, 0x7f817f81, 0x7f7f7f81, 0x817f7f7f, 0x81817f81, +0x7f817f7f, 0x817f7f81, 0x817f817f, 0x81817f7f, 0x7f818181, 0x7f81817f, 0x7f817f81, 0x7f817f81, +0x7f81817f, 0x7f7f7f7f, 0x7f817f81, 0x7f818181, 0x7f7f7f7f, 0x7f7f7f7f, 0x7f7f7f7f, 0x7f7f817f, +0x7f7f7f81, 0x7f818181, 0x7f818181, 0x7f818181, 0x7f7f8181, 0x817f7f81, 0x7f817f7f, 0x81818181, +0x7f81817f, 0x817f817f, 0x7f7f7f7f, 0x817f7f81, 0x7f7f817f, 0x7f7f817f, 0x7f7f8181, 0x7f7f7f81, +0x81817f81, 0x7f81817f, 0x7f7f7f81, 0x81817f81, 0x7f818181, 0x81817f81, 0x7f7f7f7f, 0x7f81817f, +0x81817f81, 0x7f817f7f, 0x7f81817f, 0x81817f81, 0x81818181, 0x7f81817f, 0x7f7f817f, 0x81817f7f, +0x7f7f817f, 0x8181817f, 0x81817f7f, 0x7f81817f, 0x81817f81, 0x8181817f, 0x7f817f81, 0x81817f7f, +0x817f7f7f, 0x7f7f8181, 0x817f8181, 0x7f81817f, 0x7f7f817f, 0x817f817f, 0x817f7f81, 0x7f7f817f, +0x7f7f8181, 0x7f818181, 0x7f7f8181, 0x81817f7f, 0x8181817f, 0x817f7f7f, 0x7f7f8181, 0x7f817f81, +0x817f817f, 0x7f817f7f, 0x81817f81, 0x7f7f817f, 0x81817f81, 0x7f7f817f, 0x81818181, 0x81817f81, +0x81817f7f, 0x7f7f7f7f, 0x7f7f7f7f, 0x817f817f, 0x7f7f8181, 0x7f7f7f7f, 0x7f817f7f, 0x7f7f817f, +0x7f81817f, 0x817f7f81, 0x817f7f81, 0x7f7f7f7f, 0x7f7f7f81, 0x81817f7f, 0x817f7f81, 0x7f7f817f, +0x7f817f7f, 0x81818181, 0x7f81817f, 0x7f817f7f, 0x81818181, 0x8181817f, 0x7f7f7f81, 0x7f818181, +0x8181817f, 0x7f7f7f7f, 0x817f817f, 0x7f818181, 0x81817f81, 0x7f818181, 0x7f81817f, 0x817f7f81, +0x81818181, 0x7f7f7f81, 0x81817f81, 0x7f7f8181, 0x7f81817f, 0x81817f7f, 0x8181817f, 0x7f817f7f, +0x7f7f8181, 0x7f7f817f, 0x7f817f7f, 0x817f817f, 0x7f818181, 0x7f7f7f7f, 0x7f7f7f7f, 0x7f818181, +0x81817f81, 0x7f818181, 0x81817f81, 0x7f818181, 0x7f7f7f81, 0x817f817f, 0x81817f7f, 0x7f7f7f81, +0x817f817f, 0x7f817f81, 0x7f817f81, 0x7f7f8181, 0x7f7f7f7f, 0x7f817f81, 0x8181817f, 0x817f8181, +0x7f7f7f7f, 0x8181817f, 0x817f8181, 0x81818181, 0x7f818181, 0x81818181, 0x7f7f7f81, 0x817f817f, +0x81817f81, 0x8181817f, 0x8181817f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x8181817f, 0x817f7f7f, +0x7f817f7f, 0x7f7f8181, 0x7f7f7f81, 0x7f817f81, 0x81817f81, 0x81817f81, 0x81817f7f, 0x8181817f, +0x817f817f, 0x7f817f7f, 0x817f7f7f, 0x7f7f7f81, 0x817f8181, 0x7f7f7f81, 0x7f817f81, 0x81817f7f, +0x817f817f, 0x817f8181, 0x7f7f7f81, 0x7f7f7f81, 0x81817f7f, 0x7f818181, 0x7f7f8181, 0x817f817f, +0x81818181, 0x7f818181, 0x817f7f81, 0x817f8181, 0x817f817f, 0x7f817f7f, 0x7f7f7f7f, 0x8181817f, +0x7f7f7f7f, 0x7f817f7f, 0x817f817f, 0x817f817f, 0x7f817f81, 0x81818181, 0x7f817f7f, 0x7f7f8181, +0x7f7f8181, 0x81817f7f, 0x81817f7f, 0x7f7f7f81, 0x817f7f7f, 0x7f7f817f, 0x81817f7f, 0x7f7f817f, +0x817f7f81, 0x8181817f, 0x81817f81, 0x7f7f8181, 0x7f7f8181, 0x7f7f7f81, 0x81817f7f, 0x7f7f817f, +0x81817f81, 0x7f817f81, 0x81817f7f, 0x7f7f7f7f, 0x7f81817f, 0x7f7f8181, 0x7f817f7f, 0x7f7f8181, +0x817f817f, 0x817f7f7f, 0x7f817f7f, 0x817f8181, 0x817f7f7f, 0x7f81817f, 0x7f7f817f, 0x817f817f, +0x7f7f8181, 0x7f7f7f7f, 0x817f7f81, 0x7f7f8181, 0x7f7f817f, 0x81817f81, 0x7f817f81, 0x7f817f7f, +0x817f7f81, 0x7f7f817f, 0x8181817f, 0x817f7f7f, 0x81817f81, 0x7f817f7f, 0x817f7f7f, 0x7f7f7f7f, +0x7f817f81, 0x7f7f7f7f, 0x81817f7f, 0x7f7f7f7f, 0x7f818181, 0x817f7f81, 0x817f817f, 0x8181817f, +0x817f7f7f, 0x817f8181, 0x8181817f, 0x7f817f7f, 0x7f817f7f, 0x817f7f81, 0x81817f7f, 0x7f7f8181, +0x81818181, 0x81818181, 0x7f81817f, 0x7f817f81, 0x7f817f81, 0x817f7f81, 0x8181817f, 0x7f81817f, +0x7f7f817f, 0x7f818181, 0x81817f7f, 0x7f817f81, 0x817f8181, 0x7f7f7f81, 0x7f7f8181, 0x817f7f7f, +0x7f818181, 0x7f7f8181, 0x817f8181, 0x7f817f81, 0x7f818181, 0x81817f81, 0x817f7f7f, 0x7f818181, +0x7f817f81, 0x817f7f81, 0x817f817f, 0x7f7f7f7f, 0x817f817f, 0x81818181, 0x7f817f7f, 0x7f817f81, +0x817f7f81, 0x7f7f817f, 0x81817f81, 0x7f81817f, 0x7f7f8181, 0x817f7f81, 0x817f817f, 0x81817f81, +0x817f8181, 0x7f7f7f7f, 0x817f7f7f, 0x7f81817f, 0x7f818181, 0x7f818181, 0x7f7f7f7f, 0x817f7f81, +0x817f7f81, 0x7f7f7f81, 0x8181817f, 0x7f7f7f7f, 0x817f817f, 0x7f817f7f, 0x817f817f, 0x7f7f7f7f, +0x8181817f, 0x7f7f7f7f, 0x7f7f8181, 0x7f817f7f, 0x7f817f7f, 0x7f817f7f, 0x817f8181, 0x7f81817f, +0x7f817f7f, 0x81817f81, 0x8181817f, 0x7f7f7f7f, 0x7f7f8181, 0x817f7f81, 0x7f817f7f, 0x817f7f7f, +0x81817f81, 0x817f7f7f, 0x81817f81, 0x7f81817f, 0x7f818181, 0x7f818181, 0x7f818181, 0x7f81817f, +0x7f7f8181, 0x817f7f81, 0x817f8181, 0x7f817f81, 0x7f817f7f, 0x81818181, 0x7f817f81, 0x81818181, +0x81817f7f, 0x81818181, 0x817f7f81, 0x81817f81, 0x81817f81, 0x7f7f7f7f, 0x81817f7f, 0x7f81817f, +0x7f7f8181, 0x81818181, 0x7f7f7f7f, 0x7f7f8181, 0x81817f7f, 0x81817f81, 0x7f7f7f81, 0x8181817f, +0x8181817f, 0x7f7f817f, 0x81817f81, 0x817f817f, 0x817f7f7f, 0x817f817f, 0x817f817f, 0x7f81817f, +0x8181817f, 0x817f7f7f, 0x7f818181, 0x817f7f7f, 0x817f8181, 0x7f7f8181, 0x817f8181, 0x817f817f, +0x817f7f81, 0x7f7f7f7f, 0x7f7f7f81, 0x8181817f, 0x7f818181, 0x7f7f8181, 0x7f81817f, 0x817f8181, +0x7f81817f, 0x7f7f8181, 0x8181817f, 0x817f7f7f, 0x817f817f, 0x8181817f, 0x8181817f, 0x81817f81, +0x7f7f817f, 0x7f817f7f, 0x817f817f, 0x7f81817f, 0x7f817f7f, 0x81817f7f, 0x7f7f8181, 0x817f7f7f, +0x7f7f7f81, 0x817f8181, 0x7f817f81, 0x817f7f81, 0x817f7f81, 0x81818181, 0x81818181, 0x81818181, +0x7f7f8181, 0x7f81817f, 0x7f817f7f, 0x7f817f7f, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x7f817f81, +0x7f817f81, 0x817f8181, 0x7f81817f, 0x81817f81, 0x7f7f7f81, 0x817f7f81, 0x817f7f7f, 0x817f8181, +0x7f7f8181, 0x7f817f7f, 0x7f818181, 0x7f7f817f, 0x7f81817f, 0x8181817f, 0x8181817f, 0x7f81817f, +0x7f817f81, 0x7f7f817f, 0x817f817f, 0x7f7f7f81, 0x817f8181, 0x7f81817f, 0x817f7f81, 0x817f817f, +0x817f817f, 0x7f817f81, 0x7f817f7f, 0x7f81817f, 0x817f817f, 0x7f7f8181, 0x7f81817f, 0x817f817f, +0x8181817f, 0x81818181, 0x81817f81, 0x817f8181, 0x81818181, 0x81817f7f, 0x817f8181, 0x7f7f8181, +0x7f7f8181, 0x7f7f817f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x81817f7f, 0x7f7f7f7f, 0x817f817f, +0x817f7f7f, 0x7f7f817f, 0x7f817f81, 0x7f7f817f, 0x7f818181, 0x8181817f, 0x7f7f8181, 0x7f817f7f, +0x7f81817f, 0x7f7f8181, 0x7f817f81, 0x817f7f7f, 0x7f7f817f, 0x7f818181, 0x7f7f817f, 0x81818181, +0x7f81817f, 0x817f8181, 0x81818181, 0x81817f81, 0x817f817f, 0x81817f7f, 0x7f7f8181, 0x7f7f7f81, +0x81817f7f, 0x7f817f81, 0x7f817f7f, 0x7f7f7f7f, 0x7f7f8181, 0x817f7f7f, 0x7f81817f, 0x7f818181, +0x7f818181, 0x817f7f7f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x817f7f81, 0x7f7f7f81, 0x817f7f81, +0x7f817f7f, 0x7f7f7f7f, 0x7f818181, 0x7f817f81, 0x8181817f, 0x817f7f7f, 0x7f817f81, 0x81818181, +0x7f7f817f, 0x81817f81, 0x8181817f, 0x7f7f7f81, 0x81817f7f, 0x7f817f81, 0x81818181, 0x817f817f, +0x7f7f7f81, 0x7f81817f, 0x7f7f817f, 0x817f7f81, 0x7f7f7f7f, 0x817f8181, 0x81817f81, 0x8181817f, +0x7f7f7f7f, 0x81817f81, 0x7f7f7f7f, 0x7f7f7f7f, 0x81817f7f, 0x7f817f81, 0x7f81817f, 0x81817f81, +0x817f817f, 0x81817f81, 0x81817f7f, 0x817f7f7f, 0x7f7f7f7f, 0x7f7f817f, 0x7f818181, 0x817f817f, +0x7f7f7f7f, 0x7f817f7f, 0x8181817f, 0x817f7f7f, 0x7f7f817f, 0x7f81817f, 0x7f7f7f81, 0x817f7f7f, +0x7f817f81, 0x7f7f8181, 0x8181817f, 0x7f818181, 0x7f817f7f, 0x817f817f, 0x7f7f7f81, 0x817f817f, +0x7f7f817f, 0x7f7f8181, 0x7f7f7f7f, 0x7f7f7f7f, 0x817f7f7f, 0x817f7f81, 0x7f817f7f, 0x7f81817f, +0x7f7f817f, 0x7f817f7f, 0x7f818181, 0x7f7f817f, 0x817f7f81, 0x7f818181, 0x817f8181, 0x7f817f81, +0x7f817f7f, 0x817f817f, 0x7f7f7f7f, 0x81817f81, 0x7f817f81, 0x7f817f7f, 0x81818181, 0x817f7f7f, +0x817f7f81, 0x817f7f7f, 0x817f7f7f, 0x8181817f, 0x7f7f7f81, 0x7f7f8181, 0x817f7f81, 0x817f7f81, +0x7f817f7f, 0x817f7f7f, 0x817f817f, 0x8181817f, 0x7f818181, 0x7f7f817f, 0x7f817f81, 0x7f7f817f, +0x7f817f7f, 0x7f817f81, 0x8181817f, 0x81818181, 0x7f7f7f7f, 0x7f81817f, 0x81817f7f, 0x7f817f7f, +0x817f8181, 0x7f818181, 0x7f7f7f7f, 0x817f7f81, 0x7f7f8181, 0x7f818181, 0x81818181, 0x7f7f7f7f, +0x8181817f, 0x81817f81, 0x817f7f81, 0x81818181, 0x81817f7f, 0x8181817f, 0x7f817f7f, 0x7f7f7f7f, +0x817f7f81, 0x817f7f81, 0x817f7f7f, 0x7f7f7f81, 0x7f7f817f, 0x81817f7f, 0x7f7f8181, 0x7f7f817f, +0x81818181, 0x7f7f7f81, 0x7f7f817f, 0x7f7f8181, 0x8181817f, 0x7f818181, 0x817f817f, 0x7f7f7f81, +0x817f7f7f, 0x7f817f7f, 0x817f7f81, 0x8181817f, 0x7f81817f, 0x817f817f, 0x8181817f, 0x7f7f7f7f, +0x817f7f7f, 0x7f818181, 0x817f7f81, 0x7f7f8181, 0x817f7f7f, 0x8181817f, 0x817f8181, 0x8181817f, +0x81817f81, 0x817f8181, 0x817f7f81, 0x7f7f8181, 0x817f8181, 0x7f817f81, 0x7f7f7f81, 0x7f818181, +0x7f817f7f, 0x7f7f8181, 0x7f7f817f, 0x81817f81, 0x81817f7f, 0x81817f7f, 0x81817f7f, 0x817f817f, +0x7f81817f, 0x7f7f7f7f, 0x817f8181, 0x8181817f, 0x7f817f81, 0x7f817f7f, 0x817f7f7f, 0x7f7f8181, +0x7f7f8181, 0x7f7f7f81, 0x7f818181, 0x81817f7f, 0x8181817f, 0x817f7f81, 0x7f7f7f7f, 0x7f81817f, +0x817f8181, 0x7f81817f, 0x8181817f, 0x81818181, 0x7f817f81, 0x81817f81, 0x7f7f7f81, 0x817f7f7f, +0x81818181, 0x7f7f7f7f, 0x81818181, 0x81818181, 0x7f7f8181, 0x7f817f81, 0x81818181, 0x817f8181, +0x81817f81, 0x7f81817f, 0x8181817f, 0x7f7f7f81, 0x8181817f, 0x81818181, 0x81817f81, 0x81817f81, +0x7f7f8181, 0x7f7f7f81, 0x7f7f7f7f, 0x8181817f, 0x7f7f8181, 0x7f818181, 0x81818181, 0x7f7f7f7f, +0x81817f7f, 0x817f7f81, 0x7f7f817f, 0x817f8181, 0x817f8181, 0x7f7f8181, 0x817f7f7f, 0x7f7f7f7f, +0x7f7f7f81, 0x817f7f81, 0x7f81817f, 0x817f7f81, 0x817f817f, 0x7f7f7f7f, 0x81817f7f, 0x81817f81, +0x81818181, 0x7f817f7f, 0x81817f81, 0x7f7f7f81, 0x817f7f7f, 0x81818181, 0x7f7f817f, 0x7f818181, +0x7f7f7f7f, 0x81818181, 0x7f7f7f81, 0x81818181, 0x8181817f, 0x7f817f81, 0x7f7f7f7f, 0x7f817f81, +0x7f7f7f7f, 0x7f7f7f81, 0x81817f7f, 0x8181817f, 0x7f818181, 0x7f817f7f, 0x7f7f8181, 0x7f7f817f, +0x7f7f7f81, 0x8181817f, 0x81817f7f, 0x7f81817f, 0x7f817f81, 0x817f7f81, 0x7f7f817f, 0x81817f81, +0x817f7f81, 0x7f7f8181, 0x7f7f7f7f, 0x7f817f7f, 0x7f7f817f, 0x817f7f81, 0x817f8181, 0x7f7f7f7f, +0x7f818181, 0x81817f7f, 0x7f817f7f, 0x817f817f, 0x7f7f8181, 0x7f7f817f, 0x817f7f81, 0x7f7f8181, +0x817f7f81, 0x81817f81, 0x817f8181, 0x817f817f, 0x81818181, 0x7f817f81, 0x81817f81, 0x7f7f8181, +0x7f7f7f81, 0x817f7f81, 0x7f7f7f81, 0x7f7f7f7f, 0x81818181, 0x7f7f7f7f, 0x7f817f7f, 0x7f7f7f7f, +0x81818181, 0x8181817f, 0x7f7f7f7f, 0x7f7f817f, 0x817f8181, 0x817f817f, 0x817f7f7f, 0x817f8181, +0x7f7f8181, 0x7f7f8181, 0x817f817f, 0x7f817f81, 0x817f817f, 0x7f7f7f81, 0x817f7f7f, 0x7f7f817f, +0x7f7f7f7f, 0x817f8181, 0x817f8181, 0x7f7f817f, 0x81817f81, 0x7f7f8181, 0x7f81817f, 0x81817f7f, +0x817f7f7f, 0x817f817f, 0x7f7f8181, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x7f817f81, 0x7f81817f, +0x81817f7f, 0x817f7f7f, 0x7f817f81, 0x81818181, 0x7f7f817f, 0x817f7f81, 0x7f7f817f, 0x7f817f81, +0x81817f81, 0x81818181, 0x7f7f8181, 0x817f7f7f, 0x7f7f7f81, 0x81817f7f, 0x81818181, 0x817f8181, +0x7f7f817f, 0x7f817f7f, 0x7f817f81, 0x817f7f81, 0x7f7f817f, 0x7f818181, 0x7f817f81, 0x7f817f81, +0x7f818181, 0x81817f7f, 0x817f7f81, 0x8181817f, 0x7f7f7f81, 0x81817f7f, 0x7f817f81, 0x8181817f, +0x7f7f817f, 0x817f817f, 0x81817f81, 0x81817f81, 0x81817f7f, 0x7f818181, 0x817f7f81, 0x817f7f81, +0x7f7f7f7f, 0x7f81817f, 0x7f817f81, 0x7f7f817f, 0x7f818181, 0x8181817f, 0x81817f81, 0x7f817f81, +0x7f81817f, 0x817f7f81, 0x7f818181, 0x817f817f, 0x817f7f7f, 0x81817f81, 0x7f817f81, 0x7f817f7f, +0x81817f7f, 0x7f7f8181, 0x81817f81, 0x81817f7f, 0x8181817f, 0x7f7f817f, 0x8181817f, 0x7f818181, +0x7f817f7f, 0x817f7f7f, 0x81817f81, 0x7f81817f, 0x81818181, 0x7f7f7f81, 0x7f7f817f, 0x7f7f7f81, +0x817f7f81, 0x817f7f7f, 0x7f817f7f, 0x7f7f8181, 0x7f817f81, 0x7f817f81, 0x7f81817f, 0x81818181, +0x7f7f7f81, 0x7f7f7f81, 0x81818181, 0x817f8181, 0x7f7f8181, 0x817f8181, 0x817f7f81, 0x8181817f, +0x81818181, 0x8181817f, 0x8181817f, 0x817f7f81, 0x7f7f8181, 0x817f7f81, 0x7f817f7f, 0x817f7f7f, +0x81818181, 0x81818181, 0x81817f7f, 0x7f81817f, 0x7f817f81, 0x81817f81, 0x7f7f7f7f, 0x817f817f, +0x7f7f7f7f, 0x7f817f7f, 0x817f8181, 0x7f7f7f81, 0x817f7f7f, 0x7f817f81, 0x7f7f817f, 0x8181817f, +0x817f7f81, 0x7f81817f, 0x817f817f, 0x7f817f7f, 0x81817f81, 0x7f7f817f, 0x817f8181, 0x817f7f7f, +0x81818181, 0x817f7f81, 0x7f81817f, 0x817f817f, 0x817f817f, 0x817f7f7f, 0x7f7f817f, 0x8181817f, +0x7f7f7f7f, 0x81817f81, 0x8181817f, 0x7f81817f, 0x817f7f81, 0x7f7f7f7f, 0x817f7f7f, 0x817f817f, +0x81818181, 0x8181817f, 0x81817f81, 0x81817f7f, 0x7f7f817f, 0x81817f81, 0x7f7f817f, 0x8181817f, +0x7f818181, 0x7f7f817f, 0x8181817f, 0x817f7f7f, 0x81817f7f, 0x7f818181, 0x81818181, 0x8181817f, +0x817f817f, 0x7f7f7f7f, 0x7f7f817f, 0x7f7f8181, 0x7f7f7f81, 0x7f7f817f, 0x7f818181, 0x81818181, +0x81818181, 0x7f817f81, 0x7f7f817f, 0x7f817f7f, 0x81817f7f, 0x7f7f817f, 0x8181817f, 0x7f7f7f7f, +0x7f7f8181, 0x7f7f817f, 0x7f7f7f7f, 0x817f817f, 0x7f7f7f7f, 0x817f7f81, 0x817f8181, 0x81818181, +0x7f818181, 0x7f818181, 0x8181817f, 0x7f7f7f7f, 0x8181817f, 0x7f817f7f, 0x7f7f817f, 0x81818181, +0x7f817f7f, 0x817f8181, 0x817f7f81, 0x7f817f81, 0x817f7f81, 0x7f7f7f7f, 0x817f8181, 0x7f7f8181, +0x81817f81, 0x81818181, 0x7f7f817f, 0x817f7f81, 0x7f81817f, 0x8181817f, 0x81818181, 0x81817f7f, +0x7f81817f, 0x81818181, 0x7f7f8181, 0x7f817f81, 0x7f817f81, 0x7f817f7f, 0x817f817f, 0x7f818181, +0x7f817f7f, 0x7f7f7f7f, 0x817f817f, 0x817f7f7f, 0x7f81817f, 0x817f7f81, 0x81818181, 0x7f7f817f, +0x817f817f, 0x81817f81, 0x817f7f81, 0x81817f7f, 0x7f81817f, 0x7f81817f, 0x7f817f81, 0x81817f81, +0x7f7f817f, 0x8181817f, 0x7f818181, 0x7f7f7f7f, 0x81818181, 0x8181817f, 0x7f817f81, 0x8181817f, +0x7f81817f, 0x7f7f8181, 0x817f817f, 0x817f7f81, 0x7f817f81, 0x81817f7f, 0x7f818181, 0x81818181, +0x81817f81, 0x817f7f81, 0x7f81817f, 0x817f817f, 0x81817f81, 0x7f7f7f81, 0x7f818181, 0x81817f81, +0x817f7f81, 0x7f7f817f, 0x817f8181, 0x7f817f7f, 0x817f817f, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f817f, +0x817f817f, 0x817f817f, 0x817f8181, 0x7f7f7f81, 0x817f7f81, 0x817f8181, 0x7f7f7f81, 0x8181817f, +0x81817f7f, 0x7f7f7f81, 0x81817f7f, 0x817f7f81, 0x817f817f, 0x81817f7f, 0x7f7f7f7f, 0x7f81817f, +0x7f817f7f, 0x81818181, 0x81818181, 0x7f7f7f81, 0x81817f7f, 0x8181817f, 0x7f818181, 0x7f817f7f, +0x81817f81, 0x817f7f7f, 0x7f7f8181, 0x7f817f7f, 0x81817f7f, 0x8181817f, 0x817f8181, 0x817f7f81, +0x7f7f7f7f, 0x81817f7f, 0x81817f81, 0x81818181, 0x81817f7f, 0x817f817f, 0x817f817f, 0x7f7f817f, +0x7f81817f, 0x81818181, 0x7f817f81, 0x7f7f8181, 0x7f7f817f, 0x7f7f7f81, 0x81817f7f, 0x817f8181, +0x7f817f7f, 0x8181817f, 0x817f8181, 0x817f7f7f, 0x7f7f7f81, 0x8181817f, 0x817f7f81, 0x7f817f81, +0x7f7f8181, 0x7f81817f, 0x7f7f7f7f, 0x7f7f8181, 0x7f817f7f, 0x7f7f7f81, 0x7f817f7f, 0x8181817f, +0x7f81817f, 0x817f817f, 0x7f817f81, 0x8181817f, 0x7f818181, 0x81817f81, 0x7f818181, 0x81818181, +0x7f7f7f81, 0x817f7f7f, 0x817f8181, 0x7f817f7f, 0x7f817f81, 0x817f7f81, 0x81818181, 0x7f7f7f81, +0x7f7f7f81, 0x7f7f7f81, 0x7f81817f, 0x817f817f, 0x7f7f8181, 0x7f817f7f, 0x7f817f81, 0x7f818181, +0x7f7f8181, 0x817f8181, 0x7f817f7f, 0x7f7f8181, 0x7f7f7f81, 0x817f8181, 0x7f817f7f, 0x7f817f7f, +0x7f7f817f, 0x7f7f7f81, 0x817f7f7f, 0x7f817f7f, 0x817f817f, 0x7f817f81, 0x7f7f8181, 0x81817f81, +0x7f7f8181, 0x7f818181, 0x817f7f81, 0x7f818181, 0x81817f7f, 0x817f7f7f, 0x7f7f8181, 0x81817f7f, +0x7f817f7f, 0x7f818181, 0x7f81817f, 0x81818181, 0x7f7f817f, 0x7f818181, 0x7f817f7f, 0x81817f81, +0x7f7f7f7f, 0x7f817f7f, 0x7f7f817f, 0x7f7f7f81, 0x7f7f7f7f, 0x8181817f, 0x7f817f81, 0x7f81817f, +0x7f7f7f81, 0x7f7f7f81, 0x817f817f, 0x7f7f7f81, 0x81818181, 0x7f7f7f81, 0x8181817f, 0x817f7f81, +0x7f7f817f, 0x817f817f, 0x817f8181, 0x7f7f8181, 0x81817f7f, 0x7f817f81, 0x8181817f, 0x7f81817f, +0x7f818181, 0x7f818181, 0x7f817f81, 0x7f7f7f7f, 0x7f81817f, 0x817f7f81, 0x7f7f7f81, 0x81818181, +0x7f7f7f81, 0x817f8181, 0x7f81817f, 0x8181817f, 0x7f7f7f81, 0x8181817f, 0x7f7f7f81, 0x7f81817f, +0x81817f81, 0x7f817f81, 0x81817f7f, 0x81818181, 0x817f817f, 0x817f7f7f, 0x817f8181, 0x81818181, +0x7f81817f, 0x81818181, 0x81818181, 0x7f818181, 0x7f7f7f7f, 0x81817f81, 0x8181817f, 0x7f7f817f, +0x817f7f81, 0x817f7f81, 0x817f8181, 0x7f7f8181, 0x817f8181, 0x7f7f7f81, 0x817f7f81, 0x81818181, +0x8181817f, 0x81817f81, 0x7f7f8181, 0x7f7f8181, 0x7f7f8181, 0x817f817f, 0x7f817f7f, 0x7f81817f, +0x81817f7f, 0x81818181, 0x7f81817f, 0x7f817f7f, 0x7f7f7f81, 0x817f7f81, 0x81818181, 0x7f7f7f81, +0x7f7f8181, 0x7f81817f, 0x7f7f7f7f, 0x7f7f8181, 0x81817f81, 0x7f81817f, 0x81817f81, 0x7f7f8181, +0x817f817f, 0x7f817f81, 0x7f7f8181, 0x817f817f, 0x7f817f7f, 0x817f7f81, 0x7f7f7f81, 0x7f7f817f, +0x817f8181, 0x7f7f817f, 0x8181817f, 0x81818181, 0x81817f7f, 0x7f817f7f, 0x7f7f817f, 0x81817f7f, +0x7f7f817f, 0x7f81817f, 0x81818181, 0x81817f81, 0x8181817f, 0x7f817f7f, 0x81818181, 0x7f7f7f81, +0x7f7f7f7f, 0x81817f7f, 0x81817f81, 0x7f7f817f, 0x7f7f8181, 0x8181817f, 0x81818181, 0x7f7f7f81, +0x7f817f7f, 0x7f817f7f, 0x817f7f81, 0x7f7f817f, 0x8181817f, 0x817f7f7f, 0x7f817f81, 0x817f8181, +0x7f817f7f, 0x7f817f7f, 0x7f7f817f, 0x7f7f817f, 0x7f817f7f, 0x7f817f7f, 0x7f818181, 0x817f7f81, +0x7f81817f, 0x7f817f81, 0x7f817f81, 0x817f7f81, 0x7f7f7f7f, 0x81817f7f, 0x7f81817f, 0x81818181, +0x7f7f8181, 0x7f7f7f81, 0x7f817f81, 0x81817f81, 0x81817f81, 0x8181817f, 0x7f818181, 0x81817f81, +0x817f7f81, 0x817f7f7f, 0x7f7f7f81, 0x7f7f7f7f, 0x7f7f8181, 0x7f81817f, 0x7f7f817f, 0x7f7f817f, +0x817f7f81, 0x7f7f7f7f, 0x81817f81, 0x817f817f, 0x817f817f, 0x817f817f, 0x7f81817f, 0x7f7f8181, +0x7f818181, 0x817f8181, 0x7f7f8181, 0x7f817f81, 0x81818181, 0x7f817f81, 0x817f7f7f, 0x7f817f7f, +0x7f7f8181, 0x817f8181, 0x7f818181, 0x81817f81, 0x7f818181, 0x8181817f, 0x8181817f, 0x7f818181, +0x8181817f, 0x7f7f7f7f, 0x7f7f817f, 0x7f818181, 0x7f7f7f81, 0x7f7f817f, 0x81818181, 0x817f7f7f, +0x8181817f, 0x817f817f, 0x7f7f8181, 0x7f7f7f81, 0x81818181, 0x8181817f, 0x817f817f, 0x817f817f, +0x7f818181, 0x7f7f8181, 0x8181817f, 0x817f817f, 0x81817f7f, 0x81818181, 0x7f81817f, 0x817f817f, +0x7f7f7f7f, 0x7f817f81, 0x7f818181, 0x817f7f7f, 0x817f7f81, 0x7f817f81, 0x7f7f817f, 0x7f7f7f7f, +0x81817f81, 0x7f7f7f7f, 0x817f8181, 0x7f7f817f, 0x8181817f, 0x7f81817f, 0x7f818181, 0x7f7f7f7f, +0x7f7f7f7f, 0x7f7f7f81, 0x7f818181, 0x7f818181, 0x8181817f, 0x7f7f8181, 0x817f7f81, 0x7f817f7f, +0x817f7f7f, 0x7f7f7f81, 0x7f7f8181, 0x7f817f81, 0x817f7f81, 0x7f7f7f81, 0x7f817f7f, 0x7f7f7f81, +0x817f817f, 0x817f7f7f, 0x81818181, 0x7f818181, 0x81818181, 0x81818181, 0x7f7f7f81, 0x8181817f, +0x81818181, 0x7f817f7f, 0x817f7f81, 0x7f7f8181, 0x7f818181, 0x8181817f, 0x8181817f, 0x817f7f81, +0x7f817f7f, 0x81817f7f, 0x817f8181, 0x7f817f7f, 0x7f81817f, 0x81817f7f, 0x7f817f7f, 0x817f7f7f, +0x817f7f7f, 0x7f817f81, 0x8181817f, 0x817f7f7f, 0x7f817f81, 0x81817f81, 0x817f817f, 0x7f7f7f81, +0x7f817f81, 0x7f817f7f, 0x817f817f, 0x817f7f81, 0x7f7f7f7f, 0x81817f81, 0x7f818181, 0x81817f81, +0x817f8181, 0x7f817f7f, 0x81817f81, 0x817f7f81, 0x7f7f7f7f, 0x81817f81, 0x81817f81, 0x81817f81, +0x81817f7f, 0x7f81817f, 0x81818181, 0x7f7f8181, 0x817f817f, 0x817f8181, 0x817f7f7f, 0x8181817f, +0x81817f7f, 0x8181817f, 0x7f817f81, 0x81817f7f, 0x81818181, 0x817f7f7f, 0x7f81817f, 0x817f817f, +0x81818181, 0x817f7f81, 0x817f817f, 0x817f7f7f, 0x81817f7f, 0x7f7f8181, 0x817f7f81, 0x8181817f, +0x7f818181, 0x7f817f81, 0x8181817f, 0x7f7f7f81, 0x8181817f, 0x81818181, 0x7f7f8181, 0x7f7f7f7f, +0x7f81817f, 0x817f8181, 0x817f817f, 0x7f81817f, 0x81818181, 0x7f817f7f, 0x7f7f7f7f, 0x7f7f7f7f, +0x7f7f8181, 0x817f8181, 0x817f817f, 0x7f7f7f81, 0x817f7f7f, 0x7f7f817f, 0x7f817f7f, 0x817f8181, +0x81817f81, 0x817f8181, 0x7f81817f, 0x817f8181, 0x817f8181, 0x817f7f81, 0x7f7f817f, 0x7f818181, +0x7f7f817f, 0x7f7f7f7f, 0x817f8181, 0x7f7f7f7f, 0x8181817f, 0x817f8181, 0x81818181, 0x817f7f7f, +0x7f7f817f, 0x817f7f7f, 0x817f817f, 0x7f818181, 0x8181817f, 0x817f817f, 0x7f7f7f7f, 0x7f81817f, +0x7f817f81, 0x81818181, 0x7f817f7f, 0x817f817f, 0x7f7f7f7f, 0x817f7f81, 0x7f7f8181, 0x81817f81, +0x81818181, 0x81817f81, 0x7f7f817f, 0x81818181, 0x817f8181, 0x7f81817f, 0x81817f7f, 0x7f81817f, +0x817f8181, 0x81818181, 0x7f7f7f81, 0x81817f81, 0x817f7f7f, 0x817f8181, 0x7f7f817f, 0x7f7f8181, +0x7f81817f, 0x7f818181, 0x7f81817f, 0x7f7f817f, 0x7f81817f, 0x7f7f7f81, 0x7f81817f, 0x817f8181, +0x81817f81, 0x81818181, 0x817f8181, 0x81818181, 0x7f81817f, 0x7f7f8181, 0x7f81817f, 0x81817f81, +0x7f818181, 0x81818181, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, 0x81817f7f, 0x7f81817f, 0x81818181, +0x81817f81, 0x81817f7f, 0x7f7f817f, 0x817f7f7f, 0x8181817f, 0x81817f81, 0x81817f81, 0x8181817f, +0x817f817f, 0x81817f7f, 0x7f817f7f, 0x817f817f, 0x817f7f7f, 0x8181817f, 0x7f7f7f7f, 0x817f817f, +0x817f8181, 0x7f817f7f, 0x7f817f7f, 0x7f81817f, 0x7f7f817f, 0x7f817f7f, 0x7f7f8181, 0x7f7f7f7f, +0x7f81817f, 0x817f817f, 0x7f7f817f, 0x7f7f8181, 0x7f7f7f7f, 0x817f7f7f, 0x7f81817f, 0x81818181, +0x7f7f817f, 0x7f81817f, 0x7f7f7f7f, 0x7f7f8181, 0x7f7f8181, 0x81818181, 0x817f7f7f, 0x81818181, +0x81817f81, 0x817f817f, 0x7f817f81, 0x7f7f8181, 0x7f7f7f81, 0x81817f81, 0x7f7f7f81, 0x817f7f7f, +0x817f817f, 0x817f8181, 0x7f817f81, 0x817f7f7f, 0x7f817f7f, 0x7f7f7f81, 0x81817f7f, 0x7f7f7f81, +0x7f817f7f, 0x7f81817f, 0x817f817f, 0x7f7f8181, 0x7f81817f, 0x8181817f, 0x7f7f7f7f, 0x7f7f817f, +0x7f7f7f7f, 0x7f817f7f, 0x7f7f817f, 0x7f817f81, 0x817f817f, 0x81817f7f, 0x7f81817f, 0x8181817f, +0x817f7f81, 0x7f7f7f81, 0x81818181, 0x7f817f81, 0x81818181, 0x7f818181, 0x817f7f7f, 0x81818181, +0x81818181, 0x7f7f7f7f, 0x8181817f, 0x7f7f7f81, 0x81818181, 0x7f7f817f, 0x817f7f81, 0x7f81817f, +0x817f817f, 0x7f7f7f81, 0x7f81817f, 0x7f81817f, 0x7f818181, 0x817f817f, 0x81817f7f, 0x7f7f817f, +0x7f818181, 0x7f817f7f, 0x7f7f8181, 0x817f817f, 0x81817f7f, 0x7f7f7f81, 0x817f817f, 0x7f7f817f, +0x81817f7f, 0x7f7f7f7f, 0x7f81817f, 0x817f7f7f, 0x7f7f8181, 0x7f7f8181, 0x7f7f817f, 0x7f81817f, +0x81817f81, 0x817f7f81, 0x7f81817f, 0x81817f7f, 0x817f8181, 0x817f8181, 0x817f7f7f, 0x7f7f7f7f, +0x7f7f7f7f, 0x81817f81, 0x817f817f, 0x81817f81, 0x7f7f7f7f, 0x817f817f, 0x817f7f81, 0x81817f81, +0x817f7f81, 0x817f7f7f, 0x81817f81, 0x817f7f81, 0x7f81817f, 0x81817f7f, 0x7f818181, 0x8181817f, +0x7f7f7f81, 0x81817f81, 0x7f817f81, 0x81818181, 0x817f8181, 0x7f7f8181, 0x81818181, 0x817f7f81, +0x817f8181, 0x81817f7f, 0x8181817f, 0x817f817f, 0x8181817f, 0x7f818181, 0x81817f81, 0x81818181, +0x817f817f, 0x817f7f7f, 0x7f7f7f7f, 0x7f7f7f81, 0x7f817f81, 0x81817f81, 0x7f7f8181, 0x8181817f, +0x817f7f81, 0x7f818181, 0x7f7f817f, 0x7f81817f, 0x81818181, 0x81817f7f, 0x7f817f7f, 0x817f8181, +0x7f7f8181, 0x81817f81, 0x7f7f8181, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x7f817f7f, 0x81817f81, +0x7f7f817f, 0x817f817f, 0x817f8181, 0x7f817f7f, 0x7f817f7f, 0x8181817f, 0x817f8181, 0x817f7f7f, +0x817f7f81, 0x817f8181, 0x8181817f, 0x7f818181, 0x7f818181, 0x7f7f7f81, 0x817f7f81, 0x7f817f81, +0x81817f81, 0x817f8181, 0x817f7f7f, 0x817f7f7f, 0x81818181, 0x7f7f7f81, 0x817f817f, 0x817f7f81, +0x81817f7f, 0x7f818181, 0x7f7f817f, 0x7f818181, 0x7f7f8181, 0x7f817f7f, 0x817f8181, 0x81817f81, +0x81818181, 0x7f817f7f, 0x81817f81, 0x7f817f81, 0x817f7f81, 0x7f7f7f81, 0x81817f7f, 0x7f817f7f, +0x7f7f8181, 0x817f8181, 0x7f818181, 0x8181817f, 0x81817f81, 0x817f7f7f, 0x7f817f81, 0x81818181, +0x817f817f, 0x81818181, 0x817f7f81, 0x817f7f7f, 0x817f817f, 0x7f817f81, 0x7f817f7f, 0x817f7f81, +0x7f818181, 0x817f7f7f, 0x7f81817f, 0x81817f7f, 0x817f8181, 0x817f8181, 0x81818181, 0x817f7f81, +0x7f7f817f, 0x81817f7f, 0x8181817f, 0x7f7f8181, 0x7f81817f, 0x8181817f, 0x817f7f81, 0x7f81817f, +0x817f817f, 0x7f7f817f, 0x817f7f7f, 0x8181817f, 0x7f7f817f, 0x7f81817f, 0x8181817f, 0x7f81817f, +0x81818181, 0x81818181, 0x7f81817f, 0x817f7f7f, 0x817f7f7f, 0x817f7f81, 0x7f818181, 0x817f7f7f, +0x817f7f7f, 0x81817f81, 0x7f817f81, 0x7f817f7f, 0x817f8181, 0x7f81817f, 0x7f7f817f, 0x81817f7f, +0x7f7f7f7f, 0x817f7f81, 0x81817f81, 0x817f817f, 0x7f7f8181, 0x817f7f81, 0x7f7f8181, 0x81818181, +0x7f7f7f81, 0x81817f7f, 0x817f7f81, 0x8181817f, 0x7f7f7f81, 0x817f7f7f, 0x81818181, 0x81817f7f, +0x7f817f7f, 0x817f7f7f, 0x81817f7f, 0x81817f7f, 0x817f7f7f, 0x81817f7f, 0x817f8181, 0x7f81817f, +0x7f7f7f7f, 0x817f817f, 0x817f7f81, 0x8181817f, 0x817f7f7f, 0x7f817f7f, 0x8181817f, 0x81817f81, +0x817f7f81, 0x7f7f7f81, 0x7f7f7f81, 0x7f818181, 0x7f817f81, 0x81818181, 0x817f8181, 0x8181817f, +0x7f817f7f, 0x817f8181, 0x7f818181, 0x817f7f81, 0x7f817f7f, 0x7f7f8181, 0x817f817f, 0x817f8181, +0x7f81817f, 0x7f7f817f, 0x7f7f8181, 0x7f7f817f, 0x8181817f, 0x81817f81, 0x817f8181, 0x7f7f7f81, +0x8181817f, 0x8181817f, 0x8181817f, 0x7f7f7f81, 0x8181817f, 0x7f818181, 0x81818181, 0x7f7f7f81, +0x7f817f7f, 0x81818181, 0x817f8181, 0x817f8181, 0x7f7f7f7f, 0x817f7f81, 0x817f817f, 0x817f7f81, +0x81818181, 0x817f8181, 0x7f7f8181, 0x7f7f8181, 0x817f817f, 0x81817f7f, 0x7f7f8181, 0x817f8181, +0x7f7f817f, 0x7f7f817f, 0x7f817f81, 0x7f817f7f, 0x7f818181, 0x7f81817f, 0x81817f7f, 0x817f8181, +0x817f8181, 0x7f818181, 0x8181817f, 0x7f817f7f, 0x81817f81, 0x7f7f8181, 0x817f7f81, 0x817f8181, +0x7f7f7f7f, 0x817f7f7f, 0x7f817f81, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f7f81, 0x81818181, 0x81817f81, +0x7f817f7f, 0x7f818181, 0x817f7f81, 0x8181817f, 0x81817f7f, 0x8181817f, 0x7f81817f, 0x7f817f81, +0x817f8181, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, 0x7f818181, 0x7f81817f, 0x7f7f8181, 0x7f7f7f81, +0x817f817f, 0x81817f7f, 0x7f81817f, 0x81817f7f, 0x7f818181, 0x817f8181, 0x81818181, 0x8181817f, +0x81817f7f, 0x7f817f7f, 0x817f8181, 0x81817f7f, 0x7f7f7f7f, 0x7f7f7f7f, 0x81817f81, 0x817f817f, +0x817f8181, 0x81817f81, 0x81817f81, 0x7f817f7f, 0x7f7f817f, 0x7f7f7f81, 0x7f7f7f7f, 0x8181817f, +0x81817f81, 0x7f817f7f, 0x81817f81, 0x81817f81, 0x7f817f7f, 0x7f7f817f, 0x81817f81, 0x817f8181, +0x81817f81, 0x8181817f, 0x7f7f7f81, 0x817f8181, 0x81817f7f, 0x81817f81, 0x817f8181, 0x817f8181, +0x817f817f, 0x817f7f7f, 0x81817f7f, 0x7f817f81, 0x7f7f8181, 0x7f7f7f7f, 0x7f818181, 0x81817f81, +0x817f8181, 0x817f7f7f, 0x817f817f, 0x8181817f, 0x81818181, 0x7f817f81, 0x81817f7f, 0x81818181, +0x7f818181, 0x817f817f, 0x7f7f7f81, 0x817f817f, 0x7f7f817f, 0x817f817f, 0x7f7f7f7f, 0x817f7f7f, +0x817f7f7f, 0x7f817f81, 0x7f7f817f, 0x81817f7f, 0x7f817f7f, 0x817f7f81, 0x817f817f, 0x817f817f, +0x817f817f, 0x7f817f81, 0x7f818181, 0x7f7f7f81, 0x7f817f81, 0x81817f81, 0x81818181, 0x7f817f7f, +0x7f7f7f81, 0x817f817f, 0x7f817f7f, 0x817f7f81, 0x7f7f7f7f, 0x817f8181, 0x7f7f7f7f, 0x817f817f, +0x81818181, 0x81818181, 0x8181817f, 0x81817f81, 0x81818181, 0x817f8181, 0x7f817f7f, 0x7f817f81, +0x817f8181, 0x7f818181, 0x817f7f81, 0x817f7f7f, 0x7f7f817f, 0x8181817f, 0x817f7f7f, 0x7f817f7f, +0x7f7f7f7f, 0x7f81817f, 0x7f7f817f, 0x7f81817f, 0x81817f81, 0x817f8181, 0x7f817f81, 0x817f8181, +0x81818181, 0x817f817f, 0x81817f81, 0x817f817f, 0x817f8181, 0x817f7f7f, 0x7f7f7f7f, 0x7f7f7f7f, +0x81817f81, 0x7f81817f, 0x817f7f81, 0x7f818181, 0x81817f7f, 0x7f7f8181, 0x7f7f8181, 0x7f817f7f, +0x8181817f, 0x7f81817f, 0x8181817f, 0x81818181, 0x81817f7f, 0x7f7f817f, 0x81817f81, 0x8181817f, +0x7f81817f, 0x7f817f81, 0x7f7f817f, 0x7f817f7f, 0x7f817f7f, 0x817f8181, 0x81817f81, 0x7f7f7f7f, +0x7f817f7f, 0x81817f7f, 0x7f7f7f7f, 0x817f817f, 0x7f7f7f81, 0x817f7f81, 0x7f818181, 0x7f7f8181, +0x7f817f81, 0x7f817f7f, 0x8181817f, 0x7f7f7f81, 0x7f7f8181, 0x81818181, 0x7f7f8181, 0x7f817f7f, +0x7f7f8181, 0x7f81817f, 0x7f7f8181, 0x81818181, 0x7f817f7f, 0x7f7f8181, 0x7f817f7f, 0x8181817f, +0x7f817f7f, 0x817f7f7f, 0x81817f81, 0x7f817f81, 0x817f817f, 0x7f817f81, 0x7f817f7f, 0x81818181, +0x7f817f81, 0x7f7f8181, 0x7f81817f, 0x81818181, 0x7f7f7f81, 0x7f817f7f, 0x7f7f7f7f, 0x7f817f81, +0x7f81817f, 0x817f7f7f, 0x817f8181, 0x7f817f7f, 0x817f7f81, 0x7f7f7f7f, 0x7f817f7f, 0x7f817f81, +0x817f7f7f, 0x817f8181, 0x7f7f7f7f, 0x817f7f81, 0x81817f7f, 0x81818181, 0x817f817f, 0x81818181, +0x7f7f7f81, 0x7f7f7f81, 0x7f7f7f7f, 0x81817f81, 0x7f81817f, 0x7f7f7f81, 0x7f7f7f81, 0x81817f7f, +0x817f7f81, 0x817f7f81, 0x7f817f7f, 0x7f7f7f7f, 0x8181817f, 0x81817f7f, 0x7f7f7f7f, 0x7f818181, +0x7f818181, 0x81818181, 0x817f8181, 0x81817f7f, 0x7f7f8181, 0x817f817f, 0x7f818181, 0x7f7f8181, +0x7f7f817f, 0x7f7f8181, 0x7f7f7f7f, 0x8181817f, 0x81818181, 0x817f8181, 0x7f7f7f7f, 0x7f81817f, +0x81817f7f, 0x7f7f8181, 0x81817f81, 0x817f7f7f, 0x817f7f7f, 0x7f817f81, 0x817f7f7f, 0x81817f7f, +0x817f817f, 0x7f7f7f81, 0x817f7f7f, 0x817f817f, 0x817f7f7f, 0x817f8181, 0x81818181, 0x7f81817f, +0x7f7f8181, 0x7f81817f, 0x81817f7f, 0x7f818181, 0x81817f81, 0x817f7f81, 0x7f7f8181, 0x7f817f81, +0x7f7f7f81, 0x7f7f7f81, 0x7f7f7f7f, 0x817f8181, 0x8181817f, 0x817f7f81, 0x817f7f81, 0x7f817f81, +0x7f7f817f, 0x7f817f81, 0x7f817f7f, 0x7f817f81, 0x81818181, 0x81818181, 0x7f7f7f7f, 0x81818181, +0x7f817f7f, 0x81818181, 0x7f81817f, 0x7f817f81, 0x81817f7f, 0x817f7f81, 0x7f7f817f, 0x7f817f7f, +0x7f818181, 0x7f7f817f, 0x7f817f7f, 0x7f7f7f7f, 0x817f8181, 0x81817f81, 0x817f7f81, 0x7f7f7f81, +0x81818181, 0x7f817f7f, 0x81818181, 0x7f818181, 0x817f7f81, 0x7f817f7f, 0x81818181, 0x81818181, +0x7f817f81, 0x7f7f7f7f, 0x81817f7f, 0x7f818181, 0x81817f7f, 0x81817f81, 0x8181817f, 0x817f817f, +0x817f817f, 0x817f7f81, 0x7f7f8181, 0x817f7f7f, 0x817f8181, 0x7f7f7f7f, 0x7f7f7f7f, 0x817f7f81, +0x817f7f7f, 0x7f7f8181, 0x7f817f81, 0x7f81817f, 0x817f7f7f, 0x7f7f7f7f, 0x7f7f8181, 0x7f7f817f, +0x817f7f7f, 0x81818181, 0x8181817f, 0x7f817f7f, 0x7f7f8181, 0x81817f81, 0x7f81817f, 0x7f7f8181, +0x817f7f7f, 0x817f7f81, 0x7f81817f, 0x7f7f7f81, 0x7f817f81, 0x8181817f, 0x817f7f81, 0x7f817f81, +0x81817f81, 0x81817f81, 0x81817f7f, 0x7f7f817f, 0x817f7f81, 0x7f81817f, 0x817f817f, 0x7f817f81, +0x817f7f81, 0x7f7f817f, 0x7f7f817f, 0x7f7f7f7f, 0x7f81817f, 0x7f7f817f, 0x8181817f, 0x817f7f7f, +0x7f817f81, 0x7f817f7f, 0x7f81817f, 0x81817f81, 0x7f817f7f, 0x7f81817f, 0x81817f7f, 0x7f7f7f7f, +0x817f817f, 0x7f7f7f7f, 0x7f81817f, 0x817f8181, 0x81818181, 0x7f81817f, 0x7f817f7f, 0x8181817f, +0x817f817f, 0x81818181, 0x81817f81, 0x81817f81, 0x7f7f8181, 0x7f7f7f81, 0x7f7f7f7f, 0x8181817f, +0x7f817f81, 0x817f7f81, 0x81818181, 0x7f7f8181, 0x81817f7f, 0x81818181, 0x7f81817f, 0x7f7f7f7f, +0x81818181, 0x7f81817f, 0x7f7f7f81, 0x8181817f, 0x7f7f7f81, 0x7f7f7f81, 0x7f7f8181, 0x81818181, +0x81818181, 0x7f7f8181, 0x7f817f81, 0x81817f7f, 0x7f7f817f, 0x817f7f81, 0x7f7f817f, 0x817f817f, +0x81817f7f, 0x7f7f817f, 0x7f7f7f7f, 0x81817f81, 0x817f7f7f, 0x7f7f7f7f, 0x8181817f, 0x7f7f8181, +0x7f817f7f, 0x7f817f81, 0x817f8181, 0x817f7f7f, 0x7f818181, 0x7f81817f, 0x817f817f, 0x7f7f8181, +0x81817f7f, 0x7f7f7f7f, 0x817f7f81, 0x8181817f, 0x81818181, 0x7f817f81, 0x7f817f81, 0x7f81817f, +0x7f81817f, 0x7f7f817f, 0x817f7f7f, 0x7f817f7f, 0x817f7f81, 0x7f7f817f, 0x817f8181, 0x7f7f7f7f, +0x7f81817f, 0x7f7f817f, 0x817f817f, 0x817f817f, 0x7f817f7f, 0x7f7f7f7f, 0x7f81817f, 0x7f7f8181, +0x817f7f81, 0x7f7f7f81, 0x7f818181, 0x817f817f, 0x7f7f817f, 0x7f817f81, 0x817f7f81, 0x81818181, +0x81817f81, 0x7f7f7f81, 0x7f817f7f, 0x7f818181, 0x817f817f, 0x7f7f8181, 0x7f81817f, 0x7f81817f, +0x817f7f81, 0x81818181, 0x7f817f81, 0x7f7f7f81, 0x7f81817f, 0x7f818181, 0x7f817f7f, 0x81817f81, +0x81818181, 0x7f817f7f, 0x81817f81, 0x81817f81, 0x7f7f817f, 0x817f8181, 0x81818181, 0x7f81817f, +0x81817f81, 0x7f7f817f, 0x7f7f817f, 0x7f817f81, 0x817f7f7f, 0x8181817f, 0x81817f81, 0x81818181, +0x7f817f81, 0x7f7f7f7f, 0x81818181, 0x7f7f8181, 0x81817f81, 0x81817f7f, 0x7f7f817f, 0x7f7f7f7f, +0x7f7f7f81, 0x7f7f8181, 0x7f7f7f7f, 0x7f7f7f81, 0x81817f81, 0x817f8181, 0x7f7f817f, 0x7f7f7f7f, +0x817f817f, 0x8181817f, 0x817f8181, 0x7f7f7f7f, 0x7f7f7f81, 0x81817f7f, 0x7f7f7f7f, 0x7f7f817f, +0x7f81817f, 0x7f7f7f81, 0x817f8181, 0x81817f81, 0x817f7f81, 0x7f7f817f, 0x7f81817f, 0x7f7f7f7f, +0x81818181, 0x81817f81, 0x7f818181, 0x7f817f7f, 0x7f7f8181, 0x81817f81, 0x8181817f, 0x7f81817f, +0x81818181, 0x7f818181, 0x817f7f81, 0x81817f7f, 0x81818181, 0x817f7f7f, 0x7f7f7f81, 0x817f7f7f, +0x817f817f, 0x817f7f81, 0x81818181, 0x81817f81, 0x7f818181, 0x81817f7f, 0x817f817f, 0x7f81817f, +0x817f7f81, 0x7f81817f, 0x8181817f, 0x7f7f7f7f, 0x8181817f, 0x7f7f817f, 0x7f817f7f, 0x817f7f7f, +0x7f818181, 0x817f8181, 0x7f7f817f, 0x817f7f81, 0x7f817f81, 0x81817f81, 0x7f7f8181, 0x7f81817f, +0x7f7f817f, 0x7f817f7f, 0x7f817f7f, 0x817f7f81, 0x817f817f, 0x817f7f81, 0x81818181, 0x7f7f817f, +0x817f8181, 0x7f81817f, 0x7f817f7f, 0x81817f81, 0x7f7f817f, 0x81817f81, 0x7f7f817f, 0x81818181, +0x81817f7f, 0x7f817f7f, 0x817f8181, 0x7f7f8181, 0x817f7f81, 0x81818181, 0x7f7f8181, 0x7f817f7f, +0x7f817f7f, 0x817f817f, 0x817f7f81, 0x81817f81, 0x7f7f8181, 0x817f7f7f, 0x7f7f8181, 0x81817f81, +0x7f817f7f, 0x817f8181, 0x817f817f, 0x81817f81, 0x81817f81, 0x7f7f817f, 0x7f7f8181, 0x7f7f7f81, +0x81817f81, 0x7f81817f, 0x7f7f8181, 0x81818181, 0x817f7f7f, 0x7f81817f, 0x7f7f817f, 0x7f817f7f, +0x7f818181, 0x7f81817f, 0x817f8181, 0x7f7f7f7f, 0x7f81817f, 0x7f818181, 0x817f817f, 0x81818181, +0x7f818181, 0x7f817f81, 0x7f81817f, 0x7f7f8181, 0x817f7f81, 0x7f7f8181, 0x7f81817f, 0x7f7f7f81, +0x7f818181, 0x7f817f7f, 0x7f7f8181, 0x7f7f8181, 0x7f818181, 0x81818181, 0x817f7f81, 0x817f8181, +0x7f7f7f7f, 0x7f818181, 0x7f7f7f7f, 0x7f7f7f81, 0x817f8181, 0x7f7f7f7f, 0x817f7f7f, 0x7f818181, +0x81818181, 0x7f7f817f, 0x81818181, 0x7f818181, 0x7f817f7f, 0x7f817f81, 0x7f7f8181, 0x81818181, +0x7f7f7f81, 0x817f8181, 0x7f7f7f7f, 0x81818181, 0x81817f7f, 0x7f817f7f, 0x7f817f7f, 0x817f817f, +0x7f817f81, 0x817f8181, 0x817f817f, 0x817f817f, 0x7f7f8181, 0x817f7f81, 0x7f7f8181, 0x7f7f7f81, +0x7f7f7f7f, 0x7f7f7f7f, 0x8181817f + +e = +34560 + +k = +6144 + +rv_index = +0 + +iter_max = +8 + +iter_min = +4 + +expected_iter_count = +8 + +ext_scale = +15 + +num_maps = +0 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_SOFT_OUTPUT, RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE, RTE_BBDEV_TURBO_POS_LLR_1_BIT_IN, +RTE_BBDEV_TURBO_NEG_LLR_1_BIT_SOFT_OUT + +expected_status = +OK \ No newline at end of file diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_dec_c2_k3136_r0_e4920_sbd_negllr.data b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c2_k3136_r0_e4920_sbd_negllr.data new file mode 100644 index 00000000..cbf4e72e --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c2_k3136_r0_e4920_sbd_negllr.data @@ -0,0 +1,676 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_DEC + +input0 = +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0xD0000000, 0x33CDC8CE, 0x4FAEE4CC, 0xC7DC23C3, 0xC306D6CA, 0x2B360A24, 0xE91C423D, +0x1F323427, 0x4B1C33B6, 0x3EC9D0E7, 0x39204DD1, 0xCCD73C37, 0xC6F6D8E1, 0x1DF828F3, 0xDEE34025, +0xC41EC235, 0xD035E1D8, 0x3CC32843, 0x29B60C3C, 0xE92E122E, 0x454BD4C8, 0x35D02543, 0x33E4D0AC, +0x07AF2228, 0x38C62DD1, 0x233800C5, 0x3CC02DD1, 0x35E73B19, 0xDEBED026, 0x33C5EDD9, 0x33E51527, +0x1F21EA18, 0xCC3E3BD2, 0x013CC14C, 0x3724D23B, 0x23CDED2D, 0x21424630, 0xC5B0E64D, 0xCFC03BEF, +0x24294241, 0x46C526BC, 0xD82F334C, 0x1E283DCB, 0x3F3F33C7, 0x4A14D2A1, 0xD92F2AF3, 0xCFA820CC, +0xDD30C6CB, 0x2B3ACAB7, 0x4AFE29CD, 0x25BD3200, 0x2D293323, 0xD32A3B29, 0x29E64D0C, 0xBE4BFB3C, +0xB1DD242D, 0x0427F11D, 0x3046CFDA, 0xD633C0D5, 0xC0DDBE01, 0xDA3DD9B4, 0x3BCE3638, 0x23262CC6, +0x432BD2AE, 0x23A9E1C7, 0x1BE45609, 0x22CD35AA, 0x32D6371F, 0xCE27352A, 0xE2B73F40, 0xC2D0CE2C, +0xE72E3A3E, 0x2FF52147, 0xEFCF2E33, 0x003103CA, 0xC6C14A33, 0x45CAC0CF, 0x011E1FD3, 0x23D3C72B, +0x2EB644C4, 0x484BD930, 0x3AC22E1A, 0x1AB6DD42, 0xCEDDE232, 0x3ACCCC31, 0xB830CA2D, 0xC930D2D7, +0xD0E049BB, 0xE6243524, 0x3EF53914, 0xCB083BE3, 0x411EDBE9, 0x30C3C3E8, 0x3721F5E7, 0xBBDF3FD9, +0x2AD21F33, 0xEE402858, 0xD3AFD237, 0x3D39401D, 0x30003623, 0x4734DC46, 0x192ACD32, 0x3324CAC2, +0xD714D4B5, 0xDC2E21D0, 0x2327CDBA, 0x15B739C3, 0x31C9EEB8, 0xFAB5372E, 0xC239B5DA, 0xE628440F, +0x34D425C6, 0x293BE6FD, 0xBDC04BBC, 0xE0DDB7CF, 0xD627BC34, 0xA5422FCB, 0x13211ABC, 0x1708E51C, +0xCE33DAD1, 0x51AEC731, 0x2E383824, 0xC722D0D3, 0xCB32292B, 0xCECC402B, 0x1B312EF0, 0x34CF46CA, +0xE3BD252C, 0x0C21EC36, 0x4ADFECBE, 0xC4D2E62C, 0xADE5BAD3, 0xDDCC1D18, 0x521AC135, 0x3F302D37, +0x22CE2DCC, 0x262A1BCF, 0x4EE7D7CA, 0xD43D3526, 0x1840E020, 0xC3201F2F, 0x3ACB453E, 0x41EAD5B6, +0xB6C4CF17, 0x243A55D4, 0xC7501E28, 0x351DCD1C, 0xE3A8D502, 0x3235271C, 0xAC342F00, 0x34B4E3D8, +0x36473AE4, 0xAE403CEB, 0xBC3FDA3F, 0x552CDBEC, 0xE7C42B5C, 0xE743E13D, 0x18D44650, 0xBA2EDF31, +0xB5C3EC42, 0x432EBE2E, 0xADDE4F2A, 0x401BCAD6, 0x03D6EAEB, 0xD0DEB133, 0x153FC4D4, 0x30DA2FD3, +0xEE282FD9, 0x242ADC39, 0xD72B42D9, 0x44C7E0D5, 0xC72CB3C7, 0x27C9472B, 0x0036DFB6, 0xD9C53112, +0xC0360E0E, 0x4EEA0E27, 0xEDCF27C1, 0x1739D5DC, 0x22DE2CE1, 0xDCCAF633, 0x31D02417, 0xD7DCD0DA, +0xDE1DE0DC, 0xD827DD22, 0x1CDBD7EC, 0xCE171BCB, 0xEE2347CF, 0xC22C312D, 0xDBCC2A38, 0xBE0ADED4, +0xDC20CC2D, 0xD5380DB2, 0x2A32C701, 0x3EC7D41E, 0xCDD7D74C, 0xE83E2A44, 0xC12B26E6, 0xE70046D7, +0xD626D1EC, 0x35DDE41D, 0x303F27BD, 0xC94D4517, 0x31D1E229, 0xAD3529BF, 0x3AC5D21C, 0xCF1DB731, +0x2738DBEB, 0x1F382B33, 0x1ED52213, 0xFCE22CD2, 0x1330331E, 0x4A30BA0F, 0x1F38B42A, 0x2E34CB1F, +0xE3E125D5, 0x3AC72D17, 0xBD3C342C, 0x37272810, 0x1D34E4C7, 0xE5CC2EF8, 0x1E2FF1DF, 0xB9294236, +0x26F10017, 0xBDE3FBD1, 0xB7DB52D0, 0x2E3CB8E9, 0xB3343D3C, 0x3E3BDFDD, 0x1F46393A, 0x25FA20C1, +0x2D194CD1, 0xE9471D26, 0x34C3242E, 0xAE012034, 0xC8F3FED7, 0xB825362B, 0x322DC136, 0x42F41511, +0x181855C2, 0x4937D5EB, 0xE13CDC23, 0x08242A28, 0x2DCCDFC0, 0x13CEC64F, 0xB3E6F7ED, 0xD93401EE, +0x3D02DEE0, 0xCFC52300, 0xD2F1BE38, 0x2D092924, 0x253E2AF0, 0x37DACAD0, 0xDB1ECFE7, 0x40CED84D, +0x3706C42A, 0x4231D91A, 0xDD30471D, 0xBD272DC8, 0xCB1941D9, 0xC22FC7C5, 0x333F3C0C, 0x3F3D2518, +0xDDCCD143, 0xB3D834D5, 0xD231BEE4, 0x2BDB3B3D, 0x2F2754E7, 0xEAF3CE64, 0x3632E4C0, 0xD83324C2, +0x11A4D5CC, 0x00D627D3, 0xD51F2731, 0x17C92BE2, 0x1AD62E1E, 0x16DB2418, 0x16B83823, 0x2711D54B, +0xDFB02BDF, 0x28CDA600, 0x2AE9ED3E, 0xD314CD20, 0xD04F272D, 0xD0F9CD37, 0xCC1735E5, 0xD328EC37, +0xA7D31A26, 0x214BD9A5, 0xCFCB4226, 0x233C4AD5, 0x26EF37D8, 0xD83E3230, 0x3EC8C93C, 0x07B9282B, +0xD1E6DAF2, 0x31C52EDD, 0x3A0038A5, 0x56C1E0DF, 0x22E437C0, 0xDBC8143B, 0xCF1DB03F, 0x1D4BE636, +0xB8D3C23A, 0xBCC3372E, 0xAF284228, 0xDCE02F1E, 0xB9D4E3D4, 0x56D3D0D8, 0x494816C1, 0xFB1240C3, +0x23B1BDD2, 0x1236322E, 0xC2CBCB47, 0x102FE7C8, 0xBB0AE1D9, 0x3831BC29, 0x39AECBD9, 0xC7CF1ED2, +0x0AE43913, 0xDDD62F2C, 0x4FD522E6, 0x3CB90A55, 0x2101C8EC, 0x432604BD, 0x29E3E31E, 0xC25B3C29, +0x20E927D4, 0xD4300C41, 0xD9AECBD5, 0x48D8E143, 0x2CEE272F, 0xD2C11FBA, 0x2F2DB8D7, 0x1CF4DFE6, +0xCCDCCEBF, 0xE052D0EA, 0xDADDEB1E, 0xC72ACB3A, 0x332E2B1B, 0x2D26E827, 0x2A29E6A6, 0xE614D6D5, +0x221D1CC5, 0x51102031, 0x1735C248, 0x31D4DCD5, 0x3C20DC00, 0x2D231732, 0xE54AD9CD, 0x1D24D826, +0xE936373A, 0x20C0D3C9, 0x2B231520, 0xD6CED133, 0xDFC8C7B5, 0xDC2FE8B8, 0x3332249B, 0xC6E6E11F, +0xB7DA17AF, 0x1FD91321, 0xEE29CD41, 0xE1262FEF, 0xB6181DC6, 0x24F618C6, 0x3BE5DA2D, 0xFC2C35C8, +0x51C232CC, 0x36183EDF, 0x2B2CD6EF, 0x2F2DBDB9, 0x00D03413, 0xCDC9CBC4, 0xDB3309CC, 0x264EB22C, +0xCED1EAC9, 0x38D53F27, 0x2EE7D6F3, 0xC7232C23, 0xCCD42515, 0xC9C8C8E3, 0x231E3930, 0x5BBEDFCA, +0x34D7E5D7, 0x25CCE9A7, 0xCD4BF1E8, 0x5E2542E6, 0x38DCE43E, 0xDA34F0CA, 0x2EBFF841, 0xDA422D3E, +0x13CA2231, 0xD5C6DA27, 0xBAC0C6B6, 0xC82C49FF, 0x31312123, 0xBF00DFBF, 0xE8F23819, 0x2C34E81E, +0xDC4C2B0C, 0x3A5650F2, 0x2CCB0FD0, 0xC42D4C28, 0x4F19DA4B, 0xF63DE630, 0x29F51C1F, 0xD6BE14C4, +0xD2DDCDD6, 0xE0D02825, 0x2FE0E3D1, 0x41342FE8, 0x25D547ED, 0xC5A732B5, 0x19BD503F, 0x4CD9DA3D, +0x24D013BF, 0x4FE93FCA, 0x12DBE8B9, 0x33DF151F, 0x1E24AECD, 0x2E2E3656, 0x133500AD, 0x3641403C, +0xBDDF2BD4, 0xE33130E0, 0x3DC02726, 0x18390C3C, 0x26C04DC9, 0xDBF22D35, 0xC3E1EC45, 0xCED12623, +0x1CE92E39, 0xCF2FD222, 0xD8DE31E6, 0x1EE916D1, 0x35DCB621, 0x19C2EB54, 0x41B34ED4, 0xD9373BDF, +0x203ED6D8, 0x2CD4A8C8, 0x14E0C6C6, 0xDF242131, 0xCDD921CF, 0x33AACBCC, 0x2AD1A5C6, 0x3ED42B00, +0xBF181FD0, 0xC0CC2329, 0x1BD1EDC9, 0x3A2D47CB, 0xDD27D8CB, 0xD8E8BAE6, 0xD5DE3FF5, 0x2C30D641, +0xDCDCDF26, 0xC2C34433, 0x2837F53D, 0x29D8E12D, 0xBF3FEBD0, 0x34CACC45, 0x30E1DA21, 0x39DDDCB3, +0x30193518, 0xC0132E19, 0x313D3736, 0xB2D4FF34, 0xF4D92945, 0xC70D3AEA, 0x3E3034F0, 0x00DA2F07, +0x1DD415EB, 0xCC1AC4B2, 0x232EB21C, 0xD7431E16, 0x2FCE0835, 0xCF39242C, 0x1BCA2CD4, 0xD8B6D0FA, +0x18C3262F, 0x2CE3BDCB, 0xD839DC38, 0x26161D24, 0x1223B3DC, 0xD6D63515, 0xCFBABFC6, 0xC1BF18C7, +0x2637C715, 0x264020EA, 0xD024D3C4, 0x4126A11D, 0x2E34442D, 0x32BB1CC9, 0x36E7DD40, 0x3113C3C5, +0xD100C8CE, 0xD21651E8, 0xC3CFCDD9, 0x403139F1, 0xD3D02119, 0x32171AC8, 0xD2F9B62E, 0xB5B335D4, +0xC1462835, 0x1F1C2C35, 0xD8DDBF35, 0x422D2DDA, 0x38ED19D7, 0x3326BD2F, 0x4534DFCE, 0x46403132, +0x19CDD51E, 0xCBC3CBC0, 0xE3CF2139, 0xC646ECD8, 0x29E7C7DD, 0xE94CCAF8, 0xB8C6EA37, 0xEC2B09DA, +0xC8C82738, 0xC6CCDDFB, 0x3FCA3B4D, 0xC6482BD1, 0x2DE1D333, 0x3530BB3A, 0xBF4838EA, 0xE4411ACB, +0x3514D939, 0xCD372B27, 0x4BEC1C39, 0xAB21C221, 0xD8D83F37, 0xCCE0512D, 0xDA4640C2, 0x36AE1CD6, +0xBC2F303B, 0x20C3B8CD, 0xBFE6D4E0, 0xBECCC4EB, 0xC7CC1E4E, 0xCC2BD03C, 0x3738C518, 0x23B824CC, +0x2BECD0BC, 0x2107B945, 0x2AC21B00, 0xBE49F2EB, 0x31D2E3DC, 0xDAD23F11, 0xCDED3137, 0x3ABADA3A, +0xD9C0C12C, 0x36CEE8CF, 0xD0D5BDD3, 0x2AC9CFCC, 0xE61FD747, 0xC7B7B12A, 0x2BD520DF, 0xD5BC1D31, +0xE6D8423B, 0xCC21EB43, 0x1F46292D, 0x171EE038, 0xDFA5E244, 0x1BD5C4AD, 0xC622C2B5, 0x4E33CFD3, +0x3F263F30, 0x34E4CF15, 0x0041CC36, 0x4021DB29, 0xD62E2231, 0x2C22AE21, 0x1A201F44, 0xC6D7C100, +0x3A335125, 0x42473A2A, 0xDF32DA1F, 0xE5D8B51F, 0x3049E832, 0xE6DED531, 0x315DD01E, 0xDDD92034, +0x2EBF0AEB, 0x4B4639DD, 0xBED6C8DA, 0xDABA3714, 0xDD133BD7, 0xC934EACF, 0x0D35C3E2, 0xB4CCC213, +0x37EE202B, 0x0AD21AD9, 0x28D2C9CD, 0xD800DB2D, 0xD34B274A, 0xBDD33644, 0x38EFF1CC, 0xBDABD734, +0xB11E3E32, 0x2F1C23DC, 0xE80BD7DE, 0xDBE7D6F7, 0x240248BF, 0x23CEF1F6, 0x1F56E634, 0x33CD2230, +0xB4DB182F, 0xD338BA36, 0x33392B24, 0xBB3B3649, 0x282F30E7, 0x45CC35B1, 0x46261B30, 0x1FD9A722, +0x3BE3E1C3, 0x25334FF0, 0xB8BB28C0, 0xD751E2CC, 0x403600E1, 0x461FC3ED, 0xCB30C202, 0xD8D2B92A, +0x2923D71B, 0xB9DDE6CC, 0xB935CB0A, 0xEF37D723, 0xD1DA06CD, 0x303CE928, 0xF8B9BA27, 0x37370A1F, +0xD6C1DDD9, 0x18B6463F, 0x28C71FC3, 0x38D8B53A, 0xD22DE9A1, 0x18DDB236, 0x0A28D515, 0x363B263D, +0x1F3A122D, 0x202AE323, 0xC72337E8, 0x32ED08CC, 0x2658C347, 0xC6F4E200, 0x45D1C2C3, 0xF92D343A, +0x3EBE1E65, 0x10EB2FD3, 0xC64FD5E5, 0xC5D3F521, 0xE9CA1A37, 0xD736CDD6, 0xDCC7C233, 0xC9C8E2D4, +0x2B99D60A, 0x1EE3DA0D, 0x30BDD8E1, 0x16D3BE1F, 0x1C4524D4, 0xC8D5D432, 0x19AEDA3D, 0xCDD4CAD5, +0xD835332E, 0x2412DA1B, 0xD0CBEBD7, 0xC3CEF425, 0xD1D743E3, 0x000A163B, 0x23DC3129, 0x202332D1, +0x322F2216, 0x28F5131A, 0xD316CF0D, 0xD0344C30, 0x423325D7, 0x37204237, 0xEEDDC721, 0x2038E805, +0xD525C522, 0xBEDDE327, 0x3AC8BCC5, 0x45402E27, 0x34DFC1D9, 0x2DB0D049, 0x322F2ACF, 0xDB3AC3C5, +0xD8D0EB4F, 0x2A1DCCCC, 0xD0D33DC2, 0x1349DC45, 0x39CBBC32, 0x30301EBF, 0x2C00DFC0, 0x5DD0CA11, +0xD0220C3A, 0xC527CD1A, 0x31CD372C, 0x2E0ED7DC, 0xC6F92338, 0xF537E8D0, 0x3626DBBF, 0xC9C3C6C3, +0x30DDBE17, 0x50394C39, 0x3A11343B, 0x27E8DC2E, 0x0DDB303F, 0x1818BCF0, 0xC5DC17DB, 0x24C53A26, +0xECB0E7E2, 0xC21B1928, 0x15D7C920, 0x4126AACE, 0xE214D8C7, 0xD40BCD42, 0xB2CBD018, 0x554436DB, +0x3C2D25D9, 0x21CEC0DC, 0xE9251919, 0xE41ACDC3, 0xDECF67C3, 0xEDDE3225, 0xCA30EEDD, 0xCFE01E37, +0xD231BF41, 0x34DF1EBD, 0x1DD2CC2C, 0x21142E29, 0xEA322B37, 0xCEC311DE, 0xF0F738DD, 0xEBED1A40, +0xDCE640E8, 0xD0D1BA2B, 0xE906D6DB, 0xC71938B1, 0x13C59F39, 0x1ED6DBB6, 0x231BD3A9, 0x2C2B30DC, +0x0ED50000, 0x2DD4EED6, 0xD9D21BEE, 0xE1442E36, 0x38CEC4AE, 0xC626CCF4, 0xC2CD3F36, 0xD71F2BD9, +0x2F421837, 0x31B11A23, 0xD4D5282D, 0xD0E6BED7, 0x30D737CB, 0xC8142A09, 0xD8D2CFDC, 0xC22D3C44, +0xF62DF1C9, 0x3D38DBC9, 0xBFC2E5D3, 0xDCE42128, 0x3A304038, 0xDBE6D626, 0x28250FE0, 0x27B52BCF, +0xE0D8DC3C, 0xC8DA09DA, 0x2DF02FD4, 0xE539D7D1, 0xCDE729C2, 0x4816E313, 0x4E392AC5, 0x112CF50F, +0x3AD94924, 0x4B22273A, 0x43D2DE9E, 0xDA22DE30, 0x3FE324DB, 0xD9CC2811, 0x29D253D4, 0xE2BC353C, +0x0CC92147, 0xB8F0C6F5, 0x3D35DEBF, 0xED4ACBEB, 0xAC34E532, 0xEA2AC526, 0xEAC0BFC8, 0x4431D9DD, +0xC0CE4E21, 0x0000F2DD, 0xE0D8C7BA, 0xDDDC2E36, 0xDA42C22B, 0x2B312614, 0xDC391EF4, 0xCC2E31D8, +0x3DF4D630, 0xC72C21D9, 0xDB2BD531, 0xD916DB3C, 0xBEC11DD2, 0xD1D5CEA4, 0xBA072ED7, 0x402F2829, +0xE3C0F627, 0xC7CC3CBD, 0xF41D3345, 0xD706DB14, 0xB2DF36AC, 0xEBC4551F, 0x32B01929, 0xC5B6DDEC, +0xD91DE130, 0xE1213222, 0x4126D927, 0xAF52300D, 0x291B301C, 0x241A1843, 0xE43C1FDC, 0xBBCC191C, +0x2C25E011, 0xBD3EE9DB, 0xC9BF1345, 0x2F333AD2, 0x1CD822DB, 0x23BE43EA, 0x2527EADC, 0xB52E1C1E, +0x391CCF17, 0xD2C82C0C, 0x242E3830, 0xD328D808, 0x35CEF4CD, 0xD81BE035, 0xD0DF38DA, 0xD1D3C42E, +0x3A2C1EEA, 0xC6D03AD2, 0x1E362E4E, 0xE6080000, 0xD018C312, 0xEFAA323B, 0xD1294BD2, 0xBAD52C3B, +0xE0373E10, 0x260CDDE2, 0xC902E040, 0x30472BD5, 0x534BE7D2, 0x20CCB523, 0x21CCD843, 0x2259E24E, +0xDE35FDD1, 0xD4D91D3E, 0xCFCA11D5, 0x35D724BD, 0x3C3F0CB8, 0x3EC447CB, 0xE835DDC4, 0xC3C73E31, +0xDD432B2F, 0x2FDCBDC2, 0x2D3CCE17, 0x25B8AB31, 0x2ACCFFE5, 0x40E03C2B, 0xE7DA1841, 0x17E53020, +0xD44E0B2F, 0x46DA22EC, 0x33422425, 0x211840C5, 0x23C1CFD5, 0xC8350BE5, 0xD833CDD0, 0xE63DC528, +0x292ADC17, 0xD8BCD3C1, 0xCD39F02E, 0xA71EC343, 0xD6CC3A3E, 0x2E12D2CF, 0xC40B1FB1, 0x37DCDC33, +0x20E52EEA, 0x271629E5, 0xC4BEE0C2, 0x1B3CDBD6, 0x00004934, 0x18E145C7, 0x4BE93522, 0x3029C309, +0x31D1D50F, 0x3DB42C1A, 0x21A643B2, 0x3E26D8C2, 0xCDDE28D1, 0x44E835E1, 0xEDC4DED1, 0xC4F9C9ED, +0xC5E1BBCC, 0x4426C1D4, 0xD9BB372B, 0x37232E0D, 0xE5D0D0AE, 0xC9D23CC5, 0xF638E2C7, 0x2DB6E6D9, +0xC0C9B72D, 0xDF28E6D0, 0x11E2DCC8, 0xD8E2F31C, 0xEA3937DE, 0x2741B953, 0x19D5CEE6, 0x33DDC22D, +0x2BF72326, 0x2143363C, 0x13CA1A1F, 0xB9D7DAC9, 0x22C6B72E, 0xC5DA2631, 0xCB32ECCF, 0x032D15D4, +0xBB1EE613, 0x38C42452, 0xDBCD2726, 0x1AF44039, 0x22CF2EC6, 0xCAEDE6B4, 0x2823D423, 0xDCC61C39, +0xC12D19BF, 0xC612C31B, 0x31F1D32D, 0x3E1C09E4, 0xD824C5E7, 0xDFFC2839, 0x2AD40000, 0xDFCCBBBC, +0x3FC9CC12, 0xEF2626E1, 0x0FCDEA2A, 0xEBD735CF, 0x4E1A2834, 0xCDEDE1D5, 0x31CCCD29, 0x2F4618DF, +0x48C11936, 0x29DADE41, 0xD71C1AE0, 0xDDD2285A, 0xB13DCDC8, 0xF020B8DB, 0xD0B837C2, 0xDC3B48E8, +0x18C3C530, 0x192C2940, 0x2ADA3AE4, 0xEA34C55A, 0xDAB2BED4, 0x1A51E838, 0xC832AEBE, 0x10DE2DE2, +0x3B2408CC, 0x22CEBD1E, 0xC919D6BE, 0x21D2D128, 0xCFE03A2A, 0xB7DEDC3B, 0x33C630ED, 0x5A0EEAD1, +0x3CB03FCE, 0x24EC15E4, 0xD0385BBF, 0x20CB4E4B, 0x30E0C535, 0x301BCD3C, 0x3642DDD3, 0xD90FDC50, +0xCA35BBCC, 0x393B3B2F, 0xE2BE352D, 0x362BE3BF, 0x1922B73D, 0xDE3448BD, 0xF7C0C02D, 0x00002F43, +0x2FCAD0E4, 0xDE412A34, 0xFBF339DA, 0x2BB94330, 0x39B521D5, 0x2DE31F43, 0x382C4F23, 0x35314C2D, +0x2DE3DBDC, 0x3A18EAB9, 0xD0DF131C, 0x45EA41E0, 0x48BBC1BF, 0x2220D7E4, 0x29C8D0C5, 0x3C0527CA, +0xC1ED061C, 0xCB48CA3E, 0xE846E633, 0x0EB2D7CF, 0x44C42335, 0x27F0D440, 0x332FD847, 0xD72F2CE3, +0xD7ECD6C9, 0x4026D51F, 0xD334BEE6, 0xE01B40C5, 0xD62BDA2F, 0xDEEA36AF, 0xABD039C7, 0x2E123FCE, +0xFD3924A8, 0xD0C6D529, 0xD52AEED4, 0x211D2A35, 0x1EE9C5F1, 0xCECAF921, 0x38C92740, 0x3E4E2A1A, +0xB8FAC7C3, 0xF11F1C36, 0xD4DA163A, 0x2701DC38, 0x0E3019D5, 0x57DB23E1, 0x43CD26D0, 0xADCC4934, +0xE0D3D706, 0x403A0000, 0xDECED5E1, 0xDAE6CF1D, 0xD5E33C40, 0x35B3D038, 0x24D73231, 0x2EC01D40, +0xC2D23CDC, 0x44C20BC6, 0x44C1D7A8, 0xE9D1C745, 0x36E5C2C6, 0xB81B2D42, 0x413917CC, 0xD8E031BA, +0xCE3CC1C4, 0x3EC91AC5, 0x0DE406CE, 0x35D1DA27, 0xC8E5CAE3, 0x313EC127, 0xB61631CA, 0xD73B3444, +0xD834C850, 0x37D31DE2, 0xCCC9153D, 0xBF41E9CA, 0xD649F9DC, 0x3A31D501, 0x50CD1DE3, 0xC8E54012, +0x4432D4D5, 0xCBB93023, 0xE8FCD6C2, 0xCEE0CAD1, 0xE41B2E39, 0x28CC0E40, 0xDFEAEFFD, 0xB3EA2723, +0x3534B935, 0x42D12A09, 0xED3DCA0A, 0x263C352B, 0xAD224EDF, 0x2CDF2A10, 0xF0BE4CAD, 0xBD45D8D5, +0x263F27D4, 0xE2372FB4, 0xC35240DF, 0x1EF3DCDA, 0x3AD0D224, 0x27DAD527, 0x303A2CBE, 0xCDDCC829, +0xE22B35C9, 0xD5DC222F, 0xEEBDCB63, 0x2042DAD8, 0x06CC3524, 0xD11BBB3E, 0xAE2229E6, 0x1FC81E1F, +0xEF4B12DA, 0xE1D846DD, 0xBA3B1CDA, 0xE154C801, 0xD6092CD2, 0xD0F132CB, 0x3A40D027, 0x123530BB, +0x1EC5B4DC, 0xD11825CB, 0xC603DECC, 0x28BCC5ED, 0x2C03E1CD, 0xE9AA12A5, 0xDB2DD53A, 0xCA32D01E, +0x2F4A09C5, 0xFE411E26, 0x1354CEC8, 0x3736C5F4, 0xC62F3C3B, 0x252DDFB9, 0xD0353723, 0xD8DDC21E, +0xE3DF21ED, 0xDAE33AC0, 0x2ADA323E, 0xDC43E1D9, 0x3A23463E, 0xD71934F8, 0xDB1037C8, 0x38B7BCCC, +0xDA2C1CC1, 0x3EEEE349, 0x2ACB2A26, 0xD4BCBCD8, 0x4AB50000, 0x9EC941D8, 0x3824C9D0, 0x203CCD36, +0x0DCDEDD8, 0x5B333036, 0x3125C7DD, 0xD7383F30, 0xD6C72617, 0xE1C245DD, 0xBDD3E818, 0x3210422F, +0xD00ECF2E, 0x44411A3A, 0x02E72E22, 0xF106D24C, 0xCFC847CF, 0x0421AFCF, 0xD2C525A9, 0x3CBCE63A, +0xE02A1E36, 0xE036E3E4, 0xDDC22B25, 0x2EC7452E, 0xE1E3EFE3, 0xD82ED6CE, 0x36C1E115, 0xD129E026, +0x251E1C44, 0xE8E5D7CA, 0xDDE7D40F, 0xC5D8D9D8, 0xA1C52550, 0x3ACDDA25, 0x3548C4E1, 0x2913D9C8, +0x1E28243A, 0xCA3324E2, 0xD0D7D1C0, 0xBDE9D0C4, 0x35BF4FE0, 0xCA42E33C, 0xBBFA33DA, 0xD753C9CD, +0xDA2CE11D, 0xD9BEE407, 0xE31DD4C1, 0xCBCCD51B, 0x3BB8D6C9, 0x0000DDDD, 0x28D3CDDC, 0xD62445B5, +0xB3211F13, 0xD7B9BF29, 0xD429D5E5, 0xB9C23CCE, 0xC219EDC1, 0x211E0B34, 0xB8F0E1C1, 0x33C0EF4C, +0xD8D6C7D0, 0x30E8D3D8, 0xB06514D1, 0x21B9D915, 0xE2323DCD, 0xDA41311F, 0xE24AB11F, 0xE9B92BE7, +0x33EA23C3, 0x04C430C0, 0xE849EF0D, 0xD1232C2C, 0x260FD13E, 0x002DCBDD, 0xEB3129E2, 0x20B4C7C6, +0x25D32335, 0x2CCB39D0, 0xEA13DAC6, 0xCDEE2E14, 0xCDAA2CD0, 0xE1D339E6, 0x36DCD03B, 0xF5453114, +0x34D428ED, 0xEAEDF458, 0xD226CC2D, 0xEDD03AD8, 0xEE1CC83D, 0x10DBB5E5, 0xF018CBDB, 0x5E33E31E, +0xC6D9232B, 0x1A373130, 0xDFE7DDCB, 0xB21FC5C6, 0xC1CDB1B7, 0x3F2DF128, 0x482BB426, 0x323E0000, +0xF5D43741, 0xCC1C1FB9, 0x00004E2E, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x13000000, 0xC9112F26, 0xE3DF1FED, 0x284020CF, 0xC6ED13CA, 0x38B1E217, 0xD90FE8DB, +0xD7C0C3B7, 0x341336CE, 0x45E2D239, 0x3E22F0D8, 0x461D2C29, 0x3825D628, 0xD733E5F1, 0xD7B6C532, +0xC62B28C7, 0xD0BF3024, 0xF0BA3A26, 0x2D1D2836, 0x1BCF2DDE, 0xD02B251A, 0xB938CD2C, 0x34E220D9, +0xC2BEC23B, 0xCC2908CD, 0x1D080039, 0x42BB25C5, 0x32EB3206, 0x36D3D1E0, 0xC32E10C3, 0x243B13E6, +0xC02A1FF0, 0xC325D6DD, 0x23E33D2C, 0xDCD7DED5, 0x9C23C21D, 0x32E62056, 0x06314421, 0xE9E4ED24, +0xC0C9C233, 0x44BE3FDD, 0xD25134E5, 0x292EE2E7, 0x4EE0D1DF, 0x29C82E3A, 0xF5392A22, 0x4BE2D133, +0x58C2C4C8, 0xD2212920, 0x2935D4DB, 0xDECF4400, 0x31D7CCC9, 0x12DCFB2D, 0x2B2F0723, 0x2DE7B54E, +0x2AD24ADA, 0x261E2931, 0xD538CEDF, 0xE12A1C2C, 0x33DDEDE9, 0x2FFC36C4, 0xE61823CE, 0xC225E1DF, +0xC2102FEF, 0xCFD22617, 0x3EFBDA37, 0xDAD6BCD8, 0x110623E7, 0x3320E2D1, 0x1932B8D5, 0xCCEBBFC9, +0x47234011, 0x0A2B45D8, 0x25C11244, 0x00EF3221, 0xC634CC38, 0x0F34D0E6, 0x1A3F40EA, 0x2DEDC82D, +0xFFD2C8BD, 0x40E321CD, 0x24441BD3, 0xD932EDDD, 0x48362127, 0x1230EADE, 0x39E632F9, 0xD0D451DC, +0x18DD2DC0, 0xC22AC72E, 0xD71E172E, 0xAFDED12B, 0x1DBAA948, 0xF927B647, 0xD7C31C23, 0x3328E73B, +0xDBD72DD9, 0x3E23D424, 0xCB09E025, 0x16BA21CF, 0xDD00B9CB, 0x29E3CAD7, 0x3A19D011, 0xEA0EC4E0, +0xD51CDDBF, 0x2AD1C640, 0x2CD6CBD3, 0x36D6D43C, 0xA72851C2, 0xE7EE3C3F, 0x432A4721, 0x46C726E5, +0x253FD426, 0xD03B2C27, 0x1AEC343B, 0x5A2BC735, 0xCE31D2D2, 0xCFEDF30D, 0x492B4141, 0x20DC1BAD, +0xDAF81CD9, 0xD6EAD21D, 0x2339CD25, 0xA31546BB, 0xD0C0F63F, 0x382BE719, 0xDA34BFDB, 0x1EDBBADF, +0x5151E646, 0xC164D5E2, 0xD8292FCA, 0x33CF1F3B, 0x3FF531F4, 0xD3EB3453, 0xD433E236, 0x1B102B1C, +0xC6C320E8, 0xD8393C2E, 0x17EAB7EB, 0xE0C42CE5, 0x313720CC, 0x3F39C42D, 0x33374AD5, 0xCE1ED41E, +0xF7D2BE0D, 0xF114311E, 0xE22CD6EB, 0x4E23CA2A, 0xCDCF36D3, 0x463B1EC4, 0x312DE400, 0x1A1219CE, +0x0CD1E2C3, 0x3850CBEA, 0x31D7CCCD, 0x3F2D3936, 0xC518E33E, 0xC31F31B7, 0xC0AED6C9, 0x2AC6E5D9, +0xC334B8BF, 0x35EFD72E, 0xD72B3AD8, 0x31073AD7, 0x25E22D38, 0xF0CD4A64, 0x28DFDB2D, 0x22D9E714, +0xB8E43935, 0x35FCBFEC, 0x1DC1391E, 0xD6E731A5, 0xBBC8B837, 0xBD3FBE3A, 0x00BADC53, 0x28C43F33, +0x1CE0BAC2, 0xE0E125D9, 0x3BD6CB3D, 0x313230BE, 0xC4243D34, 0x1F24D2B1, 0x2058384C, 0x2AE9192D, +0x302CCCEE, 0xD435C4C7, 0xB0D53217, 0xE52D32D9, 0xE0AF2127, 0x3E47D0CA, 0x4112BFCA, 0x27D0D0BD, +0xE81E5CF9, 0xD50CBCE5, 0x2DC9B4F7, 0x3FBA33BC, 0xCBCC2821, 0xDFD51326, 0xD20ADA3A, 0x2000C3BB, +0xC4DC4BE5, 0xCD39D7A7, 0xC40FF54F, 0xCC3BD1CC, 0x2C19EACC, 0xCD2E2129, 0x44D9F0DB, 0x2B40E11C, +0x31343F4C, 0x0E142817, 0xD0104135, 0xE304CAC7, 0x259A1FD4, 0xC3EAED3C, 0x1A311AD7, 0xD8242BEE, +0x20BF4EFD, 0x3FCA16C5, 0xDDEED03A, 0x0A22C52E, 0xD53B2D3A, 0xE22A2CB5, 0x291D3228, 0x2CCC2E45, +0x48BA00D5, 0x4FE12D1D, 0xD8D91DD1, 0xCEDBEC31, 0xFBD526B9, 0x3C1DE845, 0xAFDCE91C, 0xD622D31E, +0xCF2620D9, 0x28D63427, 0x362C3DDF, 0xD72C2530, 0xD6A7C203, 0x442C3233, 0x29B24E3F, 0xDD21412C, +0x400BD6D0, 0xC8D4B831, 0x171EF12A, 0x34FFC9D0, 0x3B3ED02D, 0x0BC7E2CB, 0xC43D28C6, 0x42F6C948, +0xD24AD429, 0xCB2C2A00, 0x43E2A236, 0xBC353312, 0xD91324B5, 0x30B4D013, 0xC92FDE2F, 0xBFD6D432, +0xC82DCABC, 0xC032DC1D, 0x30C9D2CB, 0xE0341DC5, 0xDDCEBE39, 0x2CC41D58, 0xE213BF14, 0xDFCDCBD9, +0x2E3C36DD, 0xE5262833, 0xF841291D, 0x15E324F5, 0x0315D732, 0xD1C5C5CA, 0xD73144D3, 0x2ABEBA1A, +0xBCF42E2C, 0x00E1E53B, 0x4423AD2C, 0xB021CF20, 0xD8D5EEB6, 0xBD35BCCD, 0xE6D85334, 0xD549DF2D, +0x35C74D19, 0x3F4BBE3D, 0x1F3FA3EB, 0xCEB2B335, 0x35D0BCBA, 0xD5D8CF27, 0xB11DCFC4, 0x24D3D408, +0x183B35D0, 0x1C14D619, 0x35D445BB, 0xDE452444, 0xC83315C5, 0x3F2122D0, 0xB623C737, 0x33321DD0, +0xD6DF33DE, 0xDE39AA20, 0xCF00F239, 0x412410D5, 0xC5DC2339, 0x1D3BD1B6, 0xC0E3481E, 0x1030451A, +0x30EA30D1, 0xC6DCB9C7, 0x2C36DC17, 0xCBB0ED30, 0x26C20FBE, 0x35D6D528, 0xD6CC2242, 0xEDE4241E, +0xC036DED9, 0x26303627, 0xB6B6371B, 0x26E3D6DF, 0x0D36C729, 0xBEEE4FD7, 0xE3C839D1, 0xC0313031, +0xAEAD2828, 0x36323D23, 0x13342728, 0x2904EE20, 0x261DDCD6, 0x0CBCE1E1, 0xD2CBD5C3, 0xFD1C4B2C, +0x1CD8CDDB, 0xD4BE53B1, 0x2BD6CDDE, 0xC61A01E1, 0x3D41BED3, 0xE0333BD0, 0x4040E821, 0xC4A32449, +0xD3D3A8BF, 0x36E11C2C, 0x3CC1321B, 0xC5D03E1C, 0xB813D9DD, 0xD6C51838, 0xCEBED141, 0x1FEA3813, +0x3611CAC0, 0xD12EE2C6, 0xE1BFD22D, 0xD0DAE9CC, 0x0E03DB00, 0xBC37393F, 0x422BC916, 0x49C4D72A, +0xBFC0243B, 0xC1C2F1C3, 0xDF36CC32, 0x49E53EE4, 0xBFC02CDF, 0x2BEE2BE1, 0xC23224C5, 0xD63F16DA, +0x15DBC21F, 0x49E4CC57, 0xBE272CC7, 0x274E0643, 0x5C1DDBE7, 0x34B73DF0, 0xCD2D2EB9, 0x1AD43BCD, +0xFD3047D3, 0xDAC53936, 0x1328DCC8, 0xBC32331D, 0x001F2634, 0xC845410F, 0xDBDB1C2B, 0xD1D819D0, +0xEBDDE12D, 0x37E335D7, 0xCDA84635, 0x0F362FBE, 0x30C229C4, 0xBC1DC43E, 0x3CD02F3D, 0x2EF5F5F4, +0x462D15E6, 0x362F1FC5, 0xC0C9CD49, 0x14C92639, 0x2BD135D0, 0x49C6B8DA, 0xDBC7E1D9, 0xC6382328, +0xEDC217BC, 0xE4D1331C, 0xDCC741CF, 0x283A341C, 0x38B93420, 0x46001BE7, 0x3E1030C4, 0x42A8C1CD, +0xBB3339C5, 0xE538DD20, 0xDF3F2602, 0xE52BE5C1, 0xBE2B41BA, 0x3BD4D337, 0x2D39C3DB, 0xC3BB36F0, +0xD12CCF17, 0x4130B93A, 0xE31DEAEC, 0xF9DEF52D, 0xADD35231, 0xD62ED0EF, 0xBE303521, 0x1E2ADB41, +0x310AC62F, 0x22DA2DC5, 0x26E1D9D0, 0x1F4D2AC1, 0xC8E6DCE1, 0xC6B3D62E, 0xD6D20019, 0x4EDC2D39, +0x342AE224, 0x07CBD4CB, 0xE43FB3BE, 0x1BE53748, 0xDD44C8A2, 0xBC34B7BA, 0xA1DD1D43, 0x23D31827, +0x3519E93F, 0xC83B3720, 0x1BF0B72A, 0xEAC7DCC0, 0xCA4B28EC, 0xD54D242F, 0xF35435D1, 0x3DDEC824, +0xC31128CE, 0x0B0FD136, 0x442D1E2B, 0xCBD1C3EC, 0xD635341B, 0xDD1F2931, 0x3442CDD0, 0xDADD3600, +0xC31E40E3, 0xB0BD1B22, 0x38CA1F0D, 0xD42D3043, 0xE65127E4, 0x2F15C9D6, 0x3E360B30, 0xB2CB1524, +0xC7BBC4DF, 0x38D244C7, 0x483B31C0, 0xDB43C8D1, 0x43DCDBBC, 0xDCC7571D, 0x24BAC2C6, 0x3B4ED6DC, +0x3AE03245, 0x0C40E1E2, 0x493724C0, 0xD4ECC537, 0x36CC30BA, 0x1C303FC6, 0xD12F27D5, 0x00C4272F, +0xFFFD061D, 0xC2D9251D, 0xC1D7D625, 0xBFCD3AD3, 0xB42E11D4, 0xC7230E1E, 0x24CD2C21, 0x0BD436F9, +0xCECCD331, 0xE6BDE0D5, 0xD2C6C941, 0xE1D1D3EF, 0xEB4123CD, 0xDA1755E5, 0xCEE4C529, 0xDC43C5D1, +0xD8FDDFEA, 0x1AC9C317, 0x22E13DCE, 0x01D7BA29, 0x1551E449, 0x43CD363E, 0xC6281B24, 0xCBD23725, +0x3200D61C, 0x35D6CCBC, 0x262CB22C, 0xE2E43DB1, 0x26493335, 0x3AD72ACA, 0xC4244430, 0x3AD7C9C3, +0xC91ED8D1, 0x40EAD2AC, 0x38C9CDAC, 0x413BC137, 0xCD304B1F, 0xEFCA2323, 0xAC3FE4CD, 0x3616F6E0, +0x41C2272A, 0xDCD7B2E2, 0x27D9E018, 0xDEE2D943, 0xD5C01F22, 0xB0D93540, 0xB411C13D, 0xDE39CBE0, +0xE2C6F126, 0x31450828, 0xD2BC2927, 0xBACA1DD1, 0x34DDCDC3, 0xD8302AD5, 0xE2D02824, 0x0B2BE03E, +0x3BCAFD2C, 0x2E461D45, 0xBBF6F32E, 0x31E5DED2, 0x2ED4E847, 0x3BDDCD29, 0x231331D8, 0xF521CF2E, +0xD024C718, 0x20D91011, 0xDB1E0F34, 0xD931C0BB, 0xBB310837, 0x29B7C5CB, 0x3B31DEE0, 0x0836B926, +0x28CB1FD4, 0x31072A30, 0xEE41F300, 0x35CB501E, 0x603A2B47, 0x2C2739CE, 0xC0CFC4D9, 0xE0CED43C, +0x22BDC9CA, 0xBAEC122D, 0x4D16F2C5, 0xB6E5C3DA, 0x19D5C9D0, 0x1D1DDFC2, 0x223A23FB, 0xCD382B09, +0x3025C2E6, 0x3918BDD5, 0x33DFEF0E, 0x291A302B, 0x14184930, 0xD414D233, 0x473CCED7, 0xD2D318B7, +0x3E2738D2, 0x2928D537, 0x002E14BB, 0xE2E03734, 0x1B1EDFD1, 0xCF483FD8, 0x3D043844, 0xD52E1C4C, +0x2823EAD6, 0x1F2D241A, 0xBC4B22E6, 0x2EE7DFB8, 0x3424E2F0, 0xD41BD7CB, 0x2BBCDF1E, 0x155B5033, +0xB527D543, 0xE3C5E9F4, 0xD9B5C92B, 0xD3CAE0DF, 0x31C4CED7, 0x0BD33B25, 0xCF3B111D, 0xE62F241F, +0x22D93D22, 0xCF27EF43, 0xDAD2C5DD, 0xC60019BE, 0x2935D5C3, 0x2D0BCD22, 0x224714EA, 0xE7CAB817, +0xD1282222, 0x3C25EBD0, 0x45F0A8DC, 0x3ACFB92D, 0x34B4E7E6, 0xE93523E0, 0x2C292FCA, 0x44D4C7AC, +0xEA22C3FE, 0xEF37E524, 0x313ADCDA, 0x2212D846, 0x1BE0FF4B, 0x1CBC2E34, 0x174AF041, 0xD245223E, +0xE1DB2BDC, 0xF2E9A515, 0xE3C33BCC, 0x43DBD5CE, 0xF11700DB, 0xDE44DD36, 0x26E1D604, 0xC2ACE419, +0xB9D8C7D4, 0x37EFE21E, 0xE2BAC2D1, 0x3926E91A, 0xD013E8BC, 0x2C43ECD9, 0x232B30F8, 0xD6C9E0BD, +0x26A718BC, 0xEFF6C840, 0xC13332C2, 0xC4B9BD36, 0x1B4921BD, 0xD0EBECE0, 0xBCE4D647, 0x35D4DCD5, +0x1BBE4C27, 0xECC1B7CB, 0xD820284D, 0x0BC33DBF, 0x34D03140, 0x38EC3900, 0x29EBDBC9, 0xD43A3041, +0x3AE0DD0C, 0x09CFCFDB, 0x44E0C7CE, 0x20DBC4E3, 0x26DC41DE, 0xBA3AD2E5, 0x2ABA34E1, 0xD70FBEB7, +0x40E03FE2, 0xF8D62041, 0xBD47B7EA, 0xD2E03E34, 0x1FC6D2DE, 0xCF1CD155, 0x32D52630, 0x3C463D2C, +0x253130E3, 0x1ED8BB4A, 0x411BDFDD, 0x3FD8EE35, 0x33D02BBB, 0x002ECAAE, 0xDF1FDFEB, 0x2CD8DA3C, +0xB6352D59, 0xD5BDD3B9, 0x1946630D, 0xD9181731, 0xD419D31F, 0xCE2FB7D0, 0x32EFE3C3, 0xCDDB2FD3, +0xD247BB3E, 0xBA41DECA, 0x23244AD2, 0xDEAAF2B2, 0xB8D525F2, 0xD327A4D0, 0x3718C0E2, 0x2D3E20D9, +0x24CFDF12, 0x26D0BE1C, 0xC8D9282E, 0xEB54CADF, 0x44DB1AED, 0xC927BF29, 0x2E002B39, 0xBB2625E4, +0xBB4421D2, 0x18CBAA3A, 0x213ECD37, 0x0F33163C, 0x3622E432, 0xE5C7C62B, 0xDE39E7C7, 0xF3151CCA, +0x2ACED6D2, 0x2ADADFC4, 0xD6241ACC, 0xB92F0622, 0x1BC639E4, 0x33D92BC4, 0xAF14E0DD, 0xE6EEC2C9, +0x16BDCAE3, 0x272E1AC3, 0x292E1EBE, 0xDCE439B1, 0x4225E1C6, 0xCD372420, 0x27D716B1, 0xD83EE339, +0xDBCAD4B3, 0x26385033, 0xC5373528, 0x2DD2C936, 0xF9D5BAE6, 0x2E2DC7C2, 0xCD4BD045, 0x1DDF2DE8, +0x0E3740CE, 0xE5D82C36, 0x29D1360A, 0xCD03C8C6, 0x37CCCCD8, 0xD83AD330, 0xBFD619CD, 0xDEDDC432, +0xD0DADAEC, 0xD3E14F26, 0xA8CD35D9, 0xA3F325BD, 0xBC1626D7, 0x3AB81BD5, 0xBBFF1832, 0xC9DB2126, +0x16250000, 0xE003BB39, 0x35D533CE, 0x1801C610, 0xD9D44644, 0x474BD3C3, 0x1C30C51E, 0x2E3342C6, +0xB7D6B3E9, 0x5AE44BD2, 0x46D62830, 0xDBDD4B0C, 0x38CF2F1D, 0x2D331ACB, 0x36CCDED6, 0x10D7DA10, +0xD42246E0, 0x0FC73FC0, 0x2C3E1E37, 0x2CF0D1B8, 0x17412EDD, 0x2630D0DD, 0xDBD535E4, 0xD927DA43, +0xB636C11E, 0x25CBE2D8, 0x4711F64B, 0xBC2C2029, 0xD0E3F538, 0xB92C27DF, 0x272DE2D2, 0x0CC5BCEB, +0xC0BC2EDF, 0x26D131C9, 0x1F092FEE, 0x2D342B34, 0x31B825D5, 0xC6BBC9C1, 0x38F1D0E4, 0xCBF7FDC7, +0xC3CAC9B6, 0xDBC436C6, 0x19DC1FD8, 0x28CCAED1, 0x1921BDB4, 0x24BCBF33, 0x301EE244, 0x36BDBDE1, +0x31C5B724, 0x00003ECD, 0x220B1EE7, 0xDEBAD3DC, 0x21CAB942, 0xC9193B33, 0xE3CD3A0A, 0x0DC22CC3, +0x3EC0C7CB, 0x43F2BFE4, 0x0ECFE9C6, 0x3ECA2FD5, 0xCB492DD9, 0x23E7BDC9, 0xD33DC151, 0xFCBE2933, +0x5ACEC51E, 0xF2172E1C, 0x3C24DE23, 0x3320D4D5, 0x2AB53C20, 0xC328323A, 0x45EA4133, 0x24391A32, +0xC7D8DA2B, 0xF4EAAF39, 0xDFCA36E3, 0xDEBED5C8, 0x36252C36, 0x04D7DBC7, 0x30CAC23D, 0xB60BF73D, +0xEACA2A30, 0x34C12ABE, 0xE2D747D8, 0x19D21FE3, 0x1DFD24EC, 0xB6B5D9C4, 0xBF2C1DD2, 0xD2B442E0, +0x37CCE1BD, 0x2AD1E1CC, 0x18DAC6BD, 0x213AD93E, 0x2031D8CB, 0xD82F2D12, 0xD112D2F2, 0x19CD43F1, +0x36BCB9CA, 0xD7C5DDFC, 0x34011D28, 0x3BDB0000, 0xCD00E1AA, 0x5228252C, 0x3512CA34, 0x0EE53A41, +0x242CD2C2, 0xF425DD2A, 0xF02FDDF8, 0xD82026D3, 0x4ABEE7B8, 0x2DC9DDDC, 0x3BEECD3C, 0x2ED1442A, +0xC713E92A, 0x310CC12B, 0xE128D520, 0x48E5C1CF, 0xC91CE3ED, 0xCCCBC72F, 0x0B23E3E9, 0xC1D2C8B6, +0x511ED534, 0x2AC7D4C3, 0xCFDCE335, 0xD4C9F643, 0xD433EBF1, 0x1BF8DA31, 0x223FC515, 0xCC2F13C2, +0xF825FFC8, 0x35E51633, 0x36293C4A, 0xC72ACA36, 0x4544E736, 0x1BD9315C, 0xCF16DFD4, 0xD0AFEFC3, +0xCECBBDE2, 0x05D6AA32, 0xD851F045, 0x240FD948, 0x3D35B2E6, 0x17D7C7EE, 0x0B22DACE, 0xF53AD637, +0xB2372DDB, 0xF6DF14CB, 0xBED95B24, 0xD4D5190B, 0x0000CB29, 0x3DC1ECF5, 0x30C6B34F, 0x1D2629B6, +0xD429D1CC, 0xD50716E3, 0xD0C3C2CC, 0xC82CC7D4, 0x5F27C718, 0xE6201BE4, 0xDACFD42F, 0x2E2EE9D4, +0x3AD036C0, 0x3828DB20, 0x27D63439, 0xD21026DE, 0xD60710BC, 0x13AEB9E9, 0x28C41944, 0x49D5E3D7, +0x27163429, 0x1BBAD528, 0xB12A14BC, 0xC72FD9E2, 0x1F43213A, 0x133B3EE5, 0x3622FE21, 0xD438DCBE, +0x41CB25E8, 0xD0D83E2C, 0x32BFC2CE, 0x362BE1DA, 0xC3CBCE24, 0x22DECDC2, 0x30CBD5CD, 0xD42DBC26, +0xDDF6AAD0, 0xCADC0EC8, 0xDC333434, 0x333114F5, 0xCE24361A, 0xB1C5D123, 0x4B25380B, 0x2FEC3634, +0x3EE2A610, 0xB4CEE72F, 0x29BCC721, 0xB23850CD, 0xA51AE4DD, 0xC81BBC2F, 0xD4DF0000, 0x2DE5D1E8, +0x2E0A28D0, 0x29E3C1E1, 0x56F62DD6, 0x3828DD4B, 0xC41ACFDF, 0xD2390A3A, 0xDF3028CC, 0xE8C5E22B, +0xD11CDCC1, 0x44D1CBEC, 0x33C73DDC, 0x1D25B7DC, 0xE7DD0525, 0x2743C41A, 0x2AE2D3DC, 0x21D6E82A, +0x133CBAC7, 0x593F34B0, 0xDA5FE339, 0x5206252C, 0x394FD5BD, 0x21D2C92C, 0x292426E0, 0x48E0362B, +0xCC272DE4, 0xC815C4DE, 0x49BD5D37, 0x1912342F, 0x2CE9E01B, 0x30D8D2D9, 0xDEBF46E8, 0x392F473A, +0x3A18D324, 0x2FE31FD0, 0x2B2B2710, 0xB53B2AC7, 0xB3481CFF, 0x2DD7CFBC, 0x2232351A, 0xD7C91C16, +0x1CB4DA16, 0x1BCBDF3C, 0xD2213822, 0x2121DB1F, 0xDBD1C02A, 0x27DA33C7, 0x351FC4CD, 0x0000B9D8, +0xC2142C29, 0x38B9DD31, 0x54E60CDF, 0x2A4A42DB, 0x35CF2BB5, 0x0CEA3323, 0xC2D8BEE2, 0xC61F3B33, +0x10CE11D0, 0x3336CE2E, 0x303ADD25, 0x3347B7D9, 0x33222C1A, 0x1A43E4C1, 0x4326E019, 0x2AE2E92E, +0xD2C5DC40, 0xBA45D7D6, 0xDF341BDC, 0x3A361925, 0xD4FEF4D2, 0x3018DA31, 0xDDC12817, 0x3038DE24, +0xEDC0292B, 0x3E3E3336, 0xCB1BEF17, 0xE10B2EBF, 0x2B20BABF, 0xC80C421B, 0xC2411D21, 0xECD8C7D6, +0xC824E4AD, 0x65B53DD7, 0x304325D4, 0xDD1DEAE6, 0xD33131CF, 0x2F36452B, 0x23B219E9, 0xC1C23529, +0x21DC3836, 0xD618E933, 0x393E2C14, 0x362ACA3A, 0xD6E2C237, 0xD6085111, 0xD721FCB1, 0xCCE6DEBC, +0xD1DCD920, 0xC3320000, 0xEA26CB19, 0x18BB26B7, 0x233BDF4F, 0xDE23FD3F, 0x2ADA1ECB, 0x3122E839, +0xEF42D7D7, 0xCAC7202D, 0xCD4CDED1, 0x2633C824, 0xD0123E4E, 0x104429B6, 0xC8CD4B00, 0x5136B738, +0xCFC8202B, 0x163135DD, 0x2C3821E8, 0x29314CE0, 0x0F3CE4EC, 0xE5432DAF, 0xC63AE340, 0x4639DB18, +0xDF282511, 0x25B5CBD3, 0xCAF50F37, 0x1E443B33, 0xE32CD32D, 0x10DA47B9, 0xDA410526, 0x34F0E4EA, +0xCEED3134, 0xC0B31AA7, 0xD12B1AD3, 0xC722C743, 0xBFBEDD1F, 0xD328D52B, 0xCC384024, 0x18B43FB8, +0x281742BB, 0xD8322FD2, 0xB5D628C6, 0xE420C0CB, 0xC9EB391D, 0x41E6B419, 0xBCC543E1, 0x2C37DDEF, +0x1F2D41E9, 0xD0F0D205, 0x1FEAE5C5, 0x15DF14B8, 0x47E327D9, 0xD328CE25, 0x36DB2E24, 0xD92E1EBF, +0x2B3230F4, 0xD1EAD5D9, 0xC7212ADA, 0xCD1235DC, 0xC519D5DB, 0xBFC631DF, 0xD5222B35, 0x35B6DEC6, +0x153C4428, 0xC5C3E42F, 0x1B2E2D2A, 0x3CD422A6, 0xC22842CC, 0xCADAC43D, 0x2C10DAD2, 0x18C708B5, +0x36D636BD, 0xCD29C5D3, 0xBFED433A, 0xE0384939, 0x005629F2, 0x36E2C425, 0x2830DF1C, 0xC9CB351C, +0xB0D7D3E6, 0xF3CEC214, 0xDC25D1C7, 0x1BD5CBCC, 0x19DD21C9, 0xC854EE30, 0x4FE14EE3, 0x38D5D3E8, +0xC9D126E4, 0x2FDD2130, 0xCE2130D1, 0xCFCAB0C9, 0xC71F0E1A, 0x2627D4DE, 0xCBE9DACD, 0xE7121F22, +0x2CC8E435, 0x3838391D, 0xDD37242E, 0xEBBE2F10, 0x233C0000, 0xAFC91CE7, 0xCE3536F6, 0x0F25D248, +0x2D04EBE4, 0x3FF634E5, 0xE929CE26, 0x24E0BABE, 0x302CD52B, 0xD1E2073F, 0x4333D11C, 0xDA48C2D1, +0x2D33F315, 0x29B2C8C3, 0x3B26E3EB, 0x3821C6DA, 0x32163DD6, 0x1DC737BC, 0x463CCD24, 0x19D63DDD, +0x200BDDB7, 0xBDD41E22, 0xC40A444F, 0x34BF49ED, 0xD4C0C0F4, 0xA41FB721, 0xBEDF51E8, 0x15D82A17, +0x383FDF21, 0x2B2918D0, 0xE6EB4718, 0x244530DB, 0x25DD3919, 0x3BA819BE, 0xDAEFBE38, 0xEB3FCCD9, +0xD1DF2E1C, 0x2520C3E3, 0x32D0BFD5, 0x37312BD5, 0xDC29D0C9, 0xBED02432, 0x310C3A18, 0x411ED8D7, +0xA4E5CE22, 0x163CBD30, 0xC3E2BE45, 0xCFBBAAC3, 0xC8C0E4CF, 0x0000CEFC, 0xE8CC2433, 0x39ED223F, +0x243AD83B, 0xD44D39C6, 0xCABF24DE, 0x252050EB, 0x29C73B40, 0xD4F434CF, 0xCAE4D5DF, 0xC5F2EEF1, +0x2919C8D2, 0x2FC4D20B, 0x1CBF5528, 0x3DBBD52D, 0xE03AD031, 0x3D2CCA17, 0xBE23BDB8, 0xE7DA3B30, +0xD1D2D331, 0xCC1825EA, 0x2F2D39D2, 0xF9313526, 0xCFBCC2E1, 0xE5BE3AD4, 0xBF59BE1C, 0x22FFC8B5, +0x32B14015, 0x31D6BEE6, 0x46DF3843, 0xDCD1D8B2, 0xE728C8BE, 0xEA1E2A28, 0xD1D8C7D2, 0x30CAF134, +0xEDD0EADA, 0x13C1E3D0, 0xB9D7DC29, 0xEAB93624, 0x24E49F38, 0x2E3FDEDC, 0xE6D0D23D, 0x1FBB3ED2, +0x23D540DD, 0xD727D92B, 0x251E33E6, 0x4BA7EECE, 0xC1CD203C, 0xC5CFED32, 0xB9CF35C3, 0x29710000, +0x29D7C636, 0x3A3ED1D0, 0xCC1B271F, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 + +hard_output0 = +0x42441F44, 0x006DFAE9, 0xDBB63AC2, 0xA3365CB3, 0xFA44372B, 0xC2603862, 0xA6C86E09, 0x416F0BF7, +0x8E58011D, 0x47C54B99, 0x269AD14C, 0x0B286902, 0x093E4C64, 0x8D84D8E1, 0xB8AE0993, 0x49567118, +0x8CEAABD2, 0x12D33409, 0xE3358237, 0x130176F6, 0x0986718F, 0xA18C83B5, 0x04EEF058, 0x59704040, +0x7478FE81, 0x01D27193, 0xEA1AD7F3, 0x21E6C2B9, 0x6478A1FF, 0xED9959AD, 0xE39E57BE, 0x4D859105, +0x056EF72D, 0xBD170BB7, 0xF01F9ADD, 0x99BF0C05, 0x44BAD09F, 0xF6ABDD61, 0xC0F8F116, 0x2972B53C, +0x0FEC944B, 0xD1675432, 0x0025F563, 0xF42B2EA8, 0x808E5C37, 0xB6E79AD5, 0x3706284D, 0xB9C0AFA5, +0xFF0E7E28, 0x5FA45C6F, 0x9CD5244E, 0xA013DDEA, 0x0D27D1A2, 0x2AE414AD, 0x41614379, 0xE6B68872, +0x5585D926, 0x5098D45B, 0xF8980ABD, 0x65821418, 0xEF8968A3, 0x301DEC3A, 0x57EF2A7F, 0xC17BE446, +0x94B65D62, 0xC9E6F350, 0x2E6130BC, 0x235F2E5E, 0xF1C13241, 0x94B291D6, 0x8C342458, 0x175B1FCA, +0xE07310FD, 0xD24934DD, 0x3CE8D053, 0x5C8F243D, 0x945B0AB3, 0x50EB8CC1, 0x8EF499D4, 0xA67801BF, +0x1680F061, 0x283FE705, 0xB8D7E773, 0x13AD3D2F, 0x4A6C305B, 0x1C1E5B12, 0x6F57D880, 0x8A666E5E, +0x26511296, 0xFFC09750, 0xFF91760A, 0xAC26795C, 0x75F321FD, 0x4221B9CB, 0x2E119188, 0x0772A832, +0x74D6036B, 0xA001B764, 0xF0D72816, 0xCBE119F5, 0xC5D6B248, 0x1BBAB091, 0x1BBE518E, 0xC647DCB1, +0x240CE252, 0x3F8AAE65, 0xE0CDEAAB, 0xBE724062, 0x5720636C, 0xC816E67F, 0x8E5A6C7F, 0x9E2738B3, +0x2419CBDF, 0xA6BF9B75, 0x48B21FC3, 0x8C4AED8D, 0xC1D8BCA8, 0x01353DE3, 0xA99D7D85, 0x9C6DDD20, +0xA650398D, 0x4E7E5777, 0x623FA183, 0xF02B261F, 0xF0BBA66C, 0x35AC701F, 0xD818E590, 0x7CA713BE, +0x01153743, 0xD1302B91, 0x55459E3C, 0x63FDA677, 0xECF93759, 0xBBE0E349, 0x73FD4FDE, 0x5C48C9E8, +0x70F46F73, 0x0B15437C, 0x9F4159A0, 0x7C343E78, 0xB3D085E1, 0x04266351, 0x1933DAD9, 0x83AF8C77, +0x2AC11BE1, 0x76C25CF6, 0x083E6CF7, 0x00DDBA22, 0x6AD2F284, 0xF87DBC6D, 0x7D40B45D, 0x6EBF21BD, +0x972E8535, 0x632B51D8, 0x5BFBA23A, 0xCDF1D44D, 0x11749158, 0x06DC45D1, 0x93D63A59, 0x9D0552F7, +0x61C8048F, 0xBFDE9FD2, 0x77827B70, 0xFFB130DC, 0xF18BBCE9, 0xD50C324F, 0x15521E17, 0x7D2470C3, +0x5EE34C4C, 0x2AC0C8AB, 0xB2283269, 0x4107FE15, 0xE6DA1ED3, 0x66A2A3A7, 0xB204B429, 0x0E02F9D0, +0xE5A45550, 0xEAF94102, 0x18F57A81, 0x8EEC9C68, 0xD575F397, 0xFC1BE8BF, 0x104E073F, 0xDF219F97, +0x2EC93D4A, 0x6033EE18, 0xC1351A38, 0x420C6A87, 0x0B5DC831, 0xD90D23E2, 0x13B9E883, 0xCD65BE3C, +0xDB17A9CC, 0xC73C2A7E, 0x3281 + +c = +2 + +cab = +1 + +ea = +4918 + +eb = +4920 + +c_neg = +0 + +k_neg = +3072 + +k_pos = +3136 + +rv_index = +0 + +iter_max = +8 + +iter_min = +4 + +expected_iter_count = +8 + +ext_scale = +15 + +num_maps = +0 + +code_block_mode = +0 + +op_flags = +RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE, RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN + +expected_status = +OK \ No newline at end of file diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_dec_c2_k3136_r0_e4920_sbd_negllr_crc24b.data b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c2_k3136_r0_e4920_sbd_negllr_crc24b.data new file mode 100644 index 00000000..a866878a --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_dec_c2_k3136_r0_e4920_sbd_negllr_crc24b.data @@ -0,0 +1,677 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_DEC + +input0 = +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0xD0000000, 0x33CDC8CE, 0x4FAEE4CC, 0xC7DC23C3, 0xC306D6CA, 0x2B360A24, 0xE91C423D, +0x1F323427, 0x4B1C33B6, 0x3EC9D0E7, 0x39204DD1, 0xCCD73C37, 0xC6F6D8E1, 0x1DF828F3, 0xDEE34025, +0xC41EC235, 0xD035E1D8, 0x3CC32843, 0x29B60C3C, 0xE92E122E, 0x454BD4C8, 0x35D02543, 0x33E4D0AC, +0x07AF2228, 0x38C62DD1, 0x233800C5, 0x3CC02DD1, 0x35E73B19, 0xDEBED026, 0x33C5EDD9, 0x33E51527, +0x1F21EA18, 0xCC3E3BD2, 0x013CC14C, 0x3724D23B, 0x23CDED2D, 0x21424630, 0xC5B0E64D, 0xCFC03BEF, +0x24294241, 0x46C526BC, 0xD82F334C, 0x1E283DCB, 0x3F3F33C7, 0x4A14D2A1, 0xD92F2AF3, 0xCFA820CC, +0xDD30C6CB, 0x2B3ACAB7, 0x4AFE29CD, 0x25BD3200, 0x2D293323, 0xD32A3B29, 0x29E64D0C, 0xBE4BFB3C, +0xB1DD242D, 0x0427F11D, 0x3046CFDA, 0xD633C0D5, 0xC0DDBE01, 0xDA3DD9B4, 0x3BCE3638, 0x23262CC6, +0x432BD2AE, 0x23A9E1C7, 0x1BE45609, 0x22CD35AA, 0x32D6371F, 0xCE27352A, 0xE2B73F40, 0xC2D0CE2C, +0xE72E3A3E, 0x2FF52147, 0xEFCF2E33, 0x003103CA, 0xC6C14A33, 0x45CAC0CF, 0x011E1FD3, 0x23D3C72B, +0x2EB644C4, 0x484BD930, 0x3AC22E1A, 0x1AB6DD42, 0xCEDDE232, 0x3ACCCC31, 0xB830CA2D, 0xC930D2D7, +0xD0E049BB, 0xE6243524, 0x3EF53914, 0xCB083BE3, 0x411EDBE9, 0x30C3C3E8, 0x3721F5E7, 0xBBDF3FD9, +0x2AD21F33, 0xEE402858, 0xD3AFD237, 0x3D39401D, 0x30003623, 0x4734DC46, 0x192ACD32, 0x3324CAC2, +0xD714D4B5, 0xDC2E21D0, 0x2327CDBA, 0x15B739C3, 0x31C9EEB8, 0xFAB5372E, 0xC239B5DA, 0xE628440F, +0x34D425C6, 0x293BE6FD, 0xBDC04BBC, 0xE0DDB7CF, 0xD627BC34, 0xA5422FCB, 0x13211ABC, 0x1708E51C, +0xCE33DAD1, 0x51AEC731, 0x2E383824, 0xC722D0D3, 0xCB32292B, 0xCECC402B, 0x1B312EF0, 0x34CF46CA, +0xE3BD252C, 0x0C21EC36, 0x4ADFECBE, 0xC4D2E62C, 0xADE5BAD3, 0xDDCC1D18, 0x521AC135, 0x3F302D37, +0x22CE2DCC, 0x262A1BCF, 0x4EE7D7CA, 0xD43D3526, 0x1840E020, 0xC3201F2F, 0x3ACB453E, 0x41EAD5B6, +0xB6C4CF17, 0x243A55D4, 0xC7501E28, 0x351DCD1C, 0xE3A8D502, 0x3235271C, 0xAC342F00, 0x34B4E3D8, +0x36473AE4, 0xAE403CEB, 0xBC3FDA3F, 0x552CDBEC, 0xE7C42B5C, 0xE743E13D, 0x18D44650, 0xBA2EDF31, +0xB5C3EC42, 0x432EBE2E, 0xADDE4F2A, 0x401BCAD6, 0x03D6EAEB, 0xD0DEB133, 0x153FC4D4, 0x30DA2FD3, +0xEE282FD9, 0x242ADC39, 0xD72B42D9, 0x44C7E0D5, 0xC72CB3C7, 0x27C9472B, 0x0036DFB6, 0xD9C53112, +0xC0360E0E, 0x4EEA0E27, 0xEDCF27C1, 0x1739D5DC, 0x22DE2CE1, 0xDCCAF633, 0x31D02417, 0xD7DCD0DA, +0xDE1DE0DC, 0xD827DD22, 0x1CDBD7EC, 0xCE171BCB, 0xEE2347CF, 0xC22C312D, 0xDBCC2A38, 0xBE0ADED4, +0xDC20CC2D, 0xD5380DB2, 0x2A32C701, 0x3EC7D41E, 0xCDD7D74C, 0xE83E2A44, 0xC12B26E6, 0xE70046D7, +0xD626D1EC, 0x35DDE41D, 0x303F27BD, 0xC94D4517, 0x31D1E229, 0xAD3529BF, 0x3AC5D21C, 0xCF1DB731, +0x2738DBEB, 0x1F382B33, 0x1ED52213, 0xFCE22CD2, 0x1330331E, 0x4A30BA0F, 0x1F38B42A, 0x2E34CB1F, +0xE3E125D5, 0x3AC72D17, 0xBD3C342C, 0x37272810, 0x1D34E4C7, 0xE5CC2EF8, 0x1E2FF1DF, 0xB9294236, +0x26F10017, 0xBDE3FBD1, 0xB7DB52D0, 0x2E3CB8E9, 0xB3343D3C, 0x3E3BDFDD, 0x1F46393A, 0x25FA20C1, +0x2D194CD1, 0xE9471D26, 0x34C3242E, 0xAE012034, 0xC8F3FED7, 0xB825362B, 0x322DC136, 0x42F41511, +0x181855C2, 0x4937D5EB, 0xE13CDC23, 0x08242A28, 0x2DCCDFC0, 0x13CEC64F, 0xB3E6F7ED, 0xD93401EE, +0x3D02DEE0, 0xCFC52300, 0xD2F1BE38, 0x2D092924, 0x253E2AF0, 0x37DACAD0, 0xDB1ECFE7, 0x40CED84D, +0x3706C42A, 0x4231D91A, 0xDD30471D, 0xBD272DC8, 0xCB1941D9, 0xC22FC7C5, 0x333F3C0C, 0x3F3D2518, +0xDDCCD143, 0xB3D834D5, 0xD231BEE4, 0x2BDB3B3D, 0x2F2754E7, 0xEAF3CE64, 0x3632E4C0, 0xD83324C2, +0x11A4D5CC, 0x00D627D3, 0xD51F2731, 0x17C92BE2, 0x1AD62E1E, 0x16DB2418, 0x16B83823, 0x2711D54B, +0xDFB02BDF, 0x28CDA600, 0x2AE9ED3E, 0xD314CD20, 0xD04F272D, 0xD0F9CD37, 0xCC1735E5, 0xD328EC37, +0xA7D31A26, 0x214BD9A5, 0xCFCB4226, 0x233C4AD5, 0x26EF37D8, 0xD83E3230, 0x3EC8C93C, 0x07B9282B, +0xD1E6DAF2, 0x31C52EDD, 0x3A0038A5, 0x56C1E0DF, 0x22E437C0, 0xDBC8143B, 0xCF1DB03F, 0x1D4BE636, +0xB8D3C23A, 0xBCC3372E, 0xAF284228, 0xDCE02F1E, 0xB9D4E3D4, 0x56D3D0D8, 0x494816C1, 0xFB1240C3, +0x23B1BDD2, 0x1236322E, 0xC2CBCB47, 0x102FE7C8, 0xBB0AE1D9, 0x3831BC29, 0x39AECBD9, 0xC7CF1ED2, +0x0AE43913, 0xDDD62F2C, 0x4FD522E6, 0x3CB90A55, 0x2101C8EC, 0x432604BD, 0x29E3E31E, 0xC25B3C29, +0x20E927D4, 0xD4300C41, 0xD9AECBD5, 0x48D8E143, 0x2CEE272F, 0xD2C11FBA, 0x2F2DB8D7, 0x1CF4DFE6, +0xCCDCCEBF, 0xE052D0EA, 0xDADDEB1E, 0xC72ACB3A, 0x332E2B1B, 0x2D26E827, 0x2A29E6A6, 0xE614D6D5, +0x221D1CC5, 0x51102031, 0x1735C248, 0x31D4DCD5, 0x3C20DC00, 0x2D231732, 0xE54AD9CD, 0x1D24D826, +0xE936373A, 0x20C0D3C9, 0x2B231520, 0xD6CED133, 0xDFC8C7B5, 0xDC2FE8B8, 0x3332249B, 0xC6E6E11F, +0xB7DA17AF, 0x1FD91321, 0xEE29CD41, 0xE1262FEF, 0xB6181DC6, 0x24F618C6, 0x3BE5DA2D, 0xFC2C35C8, +0x51C232CC, 0x36183EDF, 0x2B2CD6EF, 0x2F2DBDB9, 0x00D03413, 0xCDC9CBC4, 0xDB3309CC, 0x264EB22C, +0xCED1EAC9, 0x38D53F27, 0x2EE7D6F3, 0xC7232C23, 0xCCD42515, 0xC9C8C8E3, 0x231E3930, 0x5BBEDFCA, +0x34D7E5D7, 0x25CCE9A7, 0xCD4BF1E8, 0x5E2542E6, 0x38DCE43E, 0xDA34F0CA, 0x2EBFF841, 0xDA422D3E, +0x13CA2231, 0xD5C6DA27, 0xBAC0C6B6, 0xC82C49FF, 0x31312123, 0xBF00DFBF, 0xE8F23819, 0x2C34E81E, +0xDC4C2B0C, 0x3A5650F2, 0x2CCB0FD0, 0xC42D4C28, 0x4F19DA4B, 0xF63DE630, 0x29F51C1F, 0xD6BE14C4, +0xD2DDCDD6, 0xE0D02825, 0x2FE0E3D1, 0x41342FE8, 0x25D547ED, 0xC5A732B5, 0x19BD503F, 0x4CD9DA3D, +0x24D013BF, 0x4FE93FCA, 0x12DBE8B9, 0x33DF151F, 0x1E24AECD, 0x2E2E3656, 0x133500AD, 0x3641403C, +0xBDDF2BD4, 0xE33130E0, 0x3DC02726, 0x18390C3C, 0x26C04DC9, 0xDBF22D35, 0xC3E1EC45, 0xCED12623, +0x1CE92E39, 0xCF2FD222, 0xD8DE31E6, 0x1EE916D1, 0x35DCB621, 0x19C2EB54, 0x41B34ED4, 0xD9373BDF, +0x203ED6D8, 0x2CD4A8C8, 0x14E0C6C6, 0xDF242131, 0xCDD921CF, 0x33AACBCC, 0x2AD1A5C6, 0x3ED42B00, +0xBF181FD0, 0xC0CC2329, 0x1BD1EDC9, 0x3A2D47CB, 0xDD27D8CB, 0xD8E8BAE6, 0xD5DE3FF5, 0x2C30D641, +0xDCDCDF26, 0xC2C34433, 0x2837F53D, 0x29D8E12D, 0xBF3FEBD0, 0x34CACC45, 0x30E1DA21, 0x39DDDCB3, +0x30193518, 0xC0132E19, 0x313D3736, 0xB2D4FF34, 0xF4D92945, 0xC70D3AEA, 0x3E3034F0, 0x00DA2F07, +0x1DD415EB, 0xCC1AC4B2, 0x232EB21C, 0xD7431E16, 0x2FCE0835, 0xCF39242C, 0x1BCA2CD4, 0xD8B6D0FA, +0x18C3262F, 0x2CE3BDCB, 0xD839DC38, 0x26161D24, 0x1223B3DC, 0xD6D63515, 0xCFBABFC6, 0xC1BF18C7, +0x2637C715, 0x264020EA, 0xD024D3C4, 0x4126A11D, 0x2E34442D, 0x32BB1CC9, 0x36E7DD40, 0x3113C3C5, +0xD100C8CE, 0xD21651E8, 0xC3CFCDD9, 0x403139F1, 0xD3D02119, 0x32171AC8, 0xD2F9B62E, 0xB5B335D4, +0xC1462835, 0x1F1C2C35, 0xD8DDBF35, 0x422D2DDA, 0x38ED19D7, 0x3326BD2F, 0x4534DFCE, 0x46403132, +0x19CDD51E, 0xCBC3CBC0, 0xE3CF2139, 0xC646ECD8, 0x29E7C7DD, 0xE94CCAF8, 0xB8C6EA37, 0xEC2B09DA, +0xC8C82738, 0xC6CCDDFB, 0x3FCA3B4D, 0xC6482BD1, 0x2DE1D333, 0x3530BB3A, 0xBF4838EA, 0xE4411ACB, +0x3514D939, 0xCD372B27, 0x4BEC1C39, 0xAB21C221, 0xD8D83F37, 0xCCE0512D, 0xDA4640C2, 0x36AE1CD6, +0xBC2F303B, 0x20C3B8CD, 0xBFE6D4E0, 0xBECCC4EB, 0xC7CC1E4E, 0xCC2BD03C, 0x3738C518, 0x23B824CC, +0x2BECD0BC, 0x2107B945, 0x2AC21B00, 0xBE49F2EB, 0x31D2E3DC, 0xDAD23F11, 0xCDED3137, 0x3ABADA3A, +0xD9C0C12C, 0x36CEE8CF, 0xD0D5BDD3, 0x2AC9CFCC, 0xE61FD747, 0xC7B7B12A, 0x2BD520DF, 0xD5BC1D31, +0xE6D8423B, 0xCC21EB43, 0x1F46292D, 0x171EE038, 0xDFA5E244, 0x1BD5C4AD, 0xC622C2B5, 0x4E33CFD3, +0x3F263F30, 0x34E4CF15, 0x0041CC36, 0x4021DB29, 0xD62E2231, 0x2C22AE21, 0x1A201F44, 0xC6D7C100, +0x3A335125, 0x42473A2A, 0xDF32DA1F, 0xE5D8B51F, 0x3049E832, 0xE6DED531, 0x315DD01E, 0xDDD92034, +0x2EBF0AEB, 0x4B4639DD, 0xBED6C8DA, 0xDABA3714, 0xDD133BD7, 0xC934EACF, 0x0D35C3E2, 0xB4CCC213, +0x37EE202B, 0x0AD21AD9, 0x28D2C9CD, 0xD800DB2D, 0xD34B274A, 0xBDD33644, 0x38EFF1CC, 0xBDABD734, +0xB11E3E32, 0x2F1C23DC, 0xE80BD7DE, 0xDBE7D6F7, 0x240248BF, 0x23CEF1F6, 0x1F56E634, 0x33CD2230, +0xB4DB182F, 0xD338BA36, 0x33392B24, 0xBB3B3649, 0x282F30E7, 0x45CC35B1, 0x46261B30, 0x1FD9A722, +0x3BE3E1C3, 0x25334FF0, 0xB8BB28C0, 0xD751E2CC, 0x403600E1, 0x461FC3ED, 0xCB30C202, 0xD8D2B92A, +0x2923D71B, 0xB9DDE6CC, 0xB935CB0A, 0xEF37D723, 0xD1DA06CD, 0x303CE928, 0xF8B9BA27, 0x37370A1F, +0xD6C1DDD9, 0x18B6463F, 0x28C71FC3, 0x38D8B53A, 0xD22DE9A1, 0x18DDB236, 0x0A28D515, 0x363B263D, +0x1F3A122D, 0x202AE323, 0xC72337E8, 0x32ED08CC, 0x2658C347, 0xC6F4E200, 0x45D1C2C3, 0xF92D343A, +0x3EBE1E65, 0x10EB2FD3, 0xC64FD5E5, 0xC5D3F521, 0xE9CA1A37, 0xD736CDD6, 0xDCC7C233, 0xC9C8E2D4, +0x2B99D60A, 0x1EE3DA0D, 0x30BDD8E1, 0x16D3BE1F, 0x1C4524D4, 0xC8D5D432, 0x19AEDA3D, 0xCDD4CAD5, +0xD835332E, 0x2412DA1B, 0xD0CBEBD7, 0xC3CEF425, 0xD1D743E3, 0x000A163B, 0x23DC3129, 0x202332D1, +0x322F2216, 0x28F5131A, 0xD316CF0D, 0xD0344C30, 0x423325D7, 0x37204237, 0xEEDDC721, 0x2038E805, +0xD525C522, 0xBEDDE327, 0x3AC8BCC5, 0x45402E27, 0x34DFC1D9, 0x2DB0D049, 0x322F2ACF, 0xDB3AC3C5, +0xD8D0EB4F, 0x2A1DCCCC, 0xD0D33DC2, 0x1349DC45, 0x39CBBC32, 0x30301EBF, 0x2C00DFC0, 0x5DD0CA11, +0xD0220C3A, 0xC527CD1A, 0x31CD372C, 0x2E0ED7DC, 0xC6F92338, 0xF537E8D0, 0x3626DBBF, 0xC9C3C6C3, +0x30DDBE17, 0x50394C39, 0x3A11343B, 0x27E8DC2E, 0x0DDB303F, 0x1818BCF0, 0xC5DC17DB, 0x24C53A26, +0xECB0E7E2, 0xC21B1928, 0x15D7C920, 0x4126AACE, 0xE214D8C7, 0xD40BCD42, 0xB2CBD018, 0x554436DB, +0x3C2D25D9, 0x21CEC0DC, 0xE9251919, 0xE41ACDC3, 0xDECF67C3, 0xEDDE3225, 0xCA30EEDD, 0xCFE01E37, +0xD231BF41, 0x34DF1EBD, 0x1DD2CC2C, 0x21142E29, 0xEA322B37, 0xCEC311DE, 0xF0F738DD, 0xEBED1A40, +0xDCE640E8, 0xD0D1BA2B, 0xE906D6DB, 0xC71938B1, 0x13C59F39, 0x1ED6DBB6, 0x231BD3A9, 0x2C2B30DC, +0x0ED50000, 0x2DD4EED6, 0xD9D21BEE, 0xE1442E36, 0x38CEC4AE, 0xC626CCF4, 0xC2CD3F36, 0xD71F2BD9, +0x2F421837, 0x31B11A23, 0xD4D5282D, 0xD0E6BED7, 0x30D737CB, 0xC8142A09, 0xD8D2CFDC, 0xC22D3C44, +0xF62DF1C9, 0x3D38DBC9, 0xBFC2E5D3, 0xDCE42128, 0x3A304038, 0xDBE6D626, 0x28250FE0, 0x27B52BCF, +0xE0D8DC3C, 0xC8DA09DA, 0x2DF02FD4, 0xE539D7D1, 0xCDE729C2, 0x4816E313, 0x4E392AC5, 0x112CF50F, +0x3AD94924, 0x4B22273A, 0x43D2DE9E, 0xDA22DE30, 0x3FE324DB, 0xD9CC2811, 0x29D253D4, 0xE2BC353C, +0x0CC92147, 0xB8F0C6F5, 0x3D35DEBF, 0xED4ACBEB, 0xAC34E532, 0xEA2AC526, 0xEAC0BFC8, 0x4431D9DD, +0xC0CE4E21, 0x0000F2DD, 0xE0D8C7BA, 0xDDDC2E36, 0xDA42C22B, 0x2B312614, 0xDC391EF4, 0xCC2E31D8, +0x3DF4D630, 0xC72C21D9, 0xDB2BD531, 0xD916DB3C, 0xBEC11DD2, 0xD1D5CEA4, 0xBA072ED7, 0x402F2829, +0xE3C0F627, 0xC7CC3CBD, 0xF41D3345, 0xD706DB14, 0xB2DF36AC, 0xEBC4551F, 0x32B01929, 0xC5B6DDEC, +0xD91DE130, 0xE1213222, 0x4126D927, 0xAF52300D, 0x291B301C, 0x241A1843, 0xE43C1FDC, 0xBBCC191C, +0x2C25E011, 0xBD3EE9DB, 0xC9BF1345, 0x2F333AD2, 0x1CD822DB, 0x23BE43EA, 0x2527EADC, 0xB52E1C1E, +0x391CCF17, 0xD2C82C0C, 0x242E3830, 0xD328D808, 0x35CEF4CD, 0xD81BE035, 0xD0DF38DA, 0xD1D3C42E, +0x3A2C1EEA, 0xC6D03AD2, 0x1E362E4E, 0xE6080000, 0xD018C312, 0xEFAA323B, 0xD1294BD2, 0xBAD52C3B, +0xE0373E10, 0x260CDDE2, 0xC902E040, 0x30472BD5, 0x534BE7D2, 0x20CCB523, 0x21CCD843, 0x2259E24E, +0xDE35FDD1, 0xD4D91D3E, 0xCFCA11D5, 0x35D724BD, 0x3C3F0CB8, 0x3EC447CB, 0xE835DDC4, 0xC3C73E31, +0xDD432B2F, 0x2FDCBDC2, 0x2D3CCE17, 0x25B8AB31, 0x2ACCFFE5, 0x40E03C2B, 0xE7DA1841, 0x17E53020, +0xD44E0B2F, 0x46DA22EC, 0x33422425, 0x211840C5, 0x23C1CFD5, 0xC8350BE5, 0xD833CDD0, 0xE63DC528, +0x292ADC17, 0xD8BCD3C1, 0xCD39F02E, 0xA71EC343, 0xD6CC3A3E, 0x2E12D2CF, 0xC40B1FB1, 0x37DCDC33, +0x20E52EEA, 0x271629E5, 0xC4BEE0C2, 0x1B3CDBD6, 0x00004934, 0x18E145C7, 0x4BE93522, 0x3029C309, +0x31D1D50F, 0x3DB42C1A, 0x21A643B2, 0x3E26D8C2, 0xCDDE28D1, 0x44E835E1, 0xEDC4DED1, 0xC4F9C9ED, +0xC5E1BBCC, 0x4426C1D4, 0xD9BB372B, 0x37232E0D, 0xE5D0D0AE, 0xC9D23CC5, 0xF638E2C7, 0x2DB6E6D9, +0xC0C9B72D, 0xDF28E6D0, 0x11E2DCC8, 0xD8E2F31C, 0xEA3937DE, 0x2741B953, 0x19D5CEE6, 0x33DDC22D, +0x2BF72326, 0x2143363C, 0x13CA1A1F, 0xB9D7DAC9, 0x22C6B72E, 0xC5DA2631, 0xCB32ECCF, 0x032D15D4, +0xBB1EE613, 0x38C42452, 0xDBCD2726, 0x1AF44039, 0x22CF2EC6, 0xCAEDE6B4, 0x2823D423, 0xDCC61C39, +0xC12D19BF, 0xC612C31B, 0x31F1D32D, 0x3E1C09E4, 0xD824C5E7, 0xDFFC2839, 0x2AD40000, 0xDFCCBBBC, +0x3FC9CC12, 0xEF2626E1, 0x0FCDEA2A, 0xEBD735CF, 0x4E1A2834, 0xCDEDE1D5, 0x31CCCD29, 0x2F4618DF, +0x48C11936, 0x29DADE41, 0xD71C1AE0, 0xDDD2285A, 0xB13DCDC8, 0xF020B8DB, 0xD0B837C2, 0xDC3B48E8, +0x18C3C530, 0x192C2940, 0x2ADA3AE4, 0xEA34C55A, 0xDAB2BED4, 0x1A51E838, 0xC832AEBE, 0x10DE2DE2, +0x3B2408CC, 0x22CEBD1E, 0xC919D6BE, 0x21D2D128, 0xCFE03A2A, 0xB7DEDC3B, 0x33C630ED, 0x5A0EEAD1, +0x3CB03FCE, 0x24EC15E4, 0xD0385BBF, 0x20CB4E4B, 0x30E0C535, 0x301BCD3C, 0x3642DDD3, 0xD90FDC50, +0xCA35BBCC, 0x393B3B2F, 0xE2BE352D, 0x362BE3BF, 0x1922B73D, 0xDE3448BD, 0xF7C0C02D, 0x00002F43, +0x2FCAD0E4, 0xDE412A34, 0xFBF339DA, 0x2BB94330, 0x39B521D5, 0x2DE31F43, 0x382C4F23, 0x35314C2D, +0x2DE3DBDC, 0x3A18EAB9, 0xD0DF131C, 0x45EA41E0, 0x48BBC1BF, 0x2220D7E4, 0x29C8D0C5, 0x3C0527CA, +0xC1ED061C, 0xCB48CA3E, 0xE846E633, 0x0EB2D7CF, 0x44C42335, 0x27F0D440, 0x332FD847, 0xD72F2CE3, +0xD7ECD6C9, 0x4026D51F, 0xD334BEE6, 0xE01B40C5, 0xD62BDA2F, 0xDEEA36AF, 0xABD039C7, 0x2E123FCE, +0xFD3924A8, 0xD0C6D529, 0xD52AEED4, 0x211D2A35, 0x1EE9C5F1, 0xCECAF921, 0x38C92740, 0x3E4E2A1A, +0xB8FAC7C3, 0xF11F1C36, 0xD4DA163A, 0x2701DC38, 0x0E3019D5, 0x57DB23E1, 0x43CD26D0, 0xADCC4934, +0xE0D3D706, 0x403A0000, 0xDECED5E1, 0xDAE6CF1D, 0xD5E33C40, 0x35B3D038, 0x24D73231, 0x2EC01D40, +0xC2D23CDC, 0x44C20BC6, 0x44C1D7A8, 0xE9D1C745, 0x36E5C2C6, 0xB81B2D42, 0x413917CC, 0xD8E031BA, +0xCE3CC1C4, 0x3EC91AC5, 0x0DE406CE, 0x35D1DA27, 0xC8E5CAE3, 0x313EC127, 0xB61631CA, 0xD73B3444, +0xD834C850, 0x37D31DE2, 0xCCC9153D, 0xBF41E9CA, 0xD649F9DC, 0x3A31D501, 0x50CD1DE3, 0xC8E54012, +0x4432D4D5, 0xCBB93023, 0xE8FCD6C2, 0xCEE0CAD1, 0xE41B2E39, 0x28CC0E40, 0xDFEAEFFD, 0xB3EA2723, +0x3534B935, 0x42D12A09, 0xED3DCA0A, 0x263C352B, 0xAD224EDF, 0x2CDF2A10, 0xF0BE4CAD, 0xBD45D8D5, +0x263F27D4, 0xE2372FB4, 0xC35240DF, 0x1EF3DCDA, 0x3AD0D224, 0x27DAD527, 0x303A2CBE, 0xCDDCC829, +0xE22B35C9, 0xD5DC222F, 0xEEBDCB63, 0x2042DAD8, 0x06CC3524, 0xD11BBB3E, 0xAE2229E6, 0x1FC81E1F, +0xEF4B12DA, 0xE1D846DD, 0xBA3B1CDA, 0xE154C801, 0xD6092CD2, 0xD0F132CB, 0x3A40D027, 0x123530BB, +0x1EC5B4DC, 0xD11825CB, 0xC603DECC, 0x28BCC5ED, 0x2C03E1CD, 0xE9AA12A5, 0xDB2DD53A, 0xCA32D01E, +0x2F4A09C5, 0xFE411E26, 0x1354CEC8, 0x3736C5F4, 0xC62F3C3B, 0x252DDFB9, 0xD0353723, 0xD8DDC21E, +0xE3DF21ED, 0xDAE33AC0, 0x2ADA323E, 0xDC43E1D9, 0x3A23463E, 0xD71934F8, 0xDB1037C8, 0x38B7BCCC, +0xDA2C1CC1, 0x3EEEE349, 0x2ACB2A26, 0xD4BCBCD8, 0x4AB50000, 0x9EC941D8, 0x3824C9D0, 0x203CCD36, +0x0DCDEDD8, 0x5B333036, 0x3125C7DD, 0xD7383F30, 0xD6C72617, 0xE1C245DD, 0xBDD3E818, 0x3210422F, +0xD00ECF2E, 0x44411A3A, 0x02E72E22, 0xF106D24C, 0xCFC847CF, 0x0421AFCF, 0xD2C525A9, 0x3CBCE63A, +0xE02A1E36, 0xE036E3E4, 0xDDC22B25, 0x2EC7452E, 0xE1E3EFE3, 0xD82ED6CE, 0x36C1E115, 0xD129E026, +0x251E1C44, 0xE8E5D7CA, 0xDDE7D40F, 0xC5D8D9D8, 0xA1C52550, 0x3ACDDA25, 0x3548C4E1, 0x2913D9C8, +0x1E28243A, 0xCA3324E2, 0xD0D7D1C0, 0xBDE9D0C4, 0x35BF4FE0, 0xCA42E33C, 0xBBFA33DA, 0xD753C9CD, +0xDA2CE11D, 0xD9BEE407, 0xE31DD4C1, 0xCBCCD51B, 0x3BB8D6C9, 0x0000DDDD, 0x28D3CDDC, 0xD62445B5, +0xB3211F13, 0xD7B9BF29, 0xD429D5E5, 0xB9C23CCE, 0xC219EDC1, 0x211E0B34, 0xB8F0E1C1, 0x33C0EF4C, +0xD8D6C7D0, 0x30E8D3D8, 0xB06514D1, 0x21B9D915, 0xE2323DCD, 0xDA41311F, 0xE24AB11F, 0xE9B92BE7, +0x33EA23C3, 0x04C430C0, 0xE849EF0D, 0xD1232C2C, 0x260FD13E, 0x002DCBDD, 0xEB3129E2, 0x20B4C7C6, +0x25D32335, 0x2CCB39D0, 0xEA13DAC6, 0xCDEE2E14, 0xCDAA2CD0, 0xE1D339E6, 0x36DCD03B, 0xF5453114, +0x34D428ED, 0xEAEDF458, 0xD226CC2D, 0xEDD03AD8, 0xEE1CC83D, 0x10DBB5E5, 0xF018CBDB, 0x5E33E31E, +0xC6D9232B, 0x1A373130, 0xDFE7DDCB, 0xB21FC5C6, 0xC1CDB1B7, 0x3F2DF128, 0x482BB426, 0x323E0000, +0xF5D43741, 0xCC1C1FB9, 0x00004E2E, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x13000000, 0xC9112F26, 0xE3DF1FED, 0x284020CF, 0xC6ED13CA, 0x38B1E217, 0xD90FE8DB, +0xD7C0C3B7, 0x341336CE, 0x45E2D239, 0x3E22F0D8, 0x461D2C29, 0x3825D628, 0xD733E5F1, 0xD7B6C532, +0xC62B28C7, 0xD0BF3024, 0xF0BA3A26, 0x2D1D2836, 0x1BCF2DDE, 0xD02B251A, 0xB938CD2C, 0x34E220D9, +0xC2BEC23B, 0xCC2908CD, 0x1D080039, 0x42BB25C5, 0x32EB3206, 0x36D3D1E0, 0xC32E10C3, 0x243B13E6, +0xC02A1FF0, 0xC325D6DD, 0x23E33D2C, 0xDCD7DED5, 0x9C23C21D, 0x32E62056, 0x06314421, 0xE9E4ED24, +0xC0C9C233, 0x44BE3FDD, 0xD25134E5, 0x292EE2E7, 0x4EE0D1DF, 0x29C82E3A, 0xF5392A22, 0x4BE2D133, +0x58C2C4C8, 0xD2212920, 0x2935D4DB, 0xDECF4400, 0x31D7CCC9, 0x12DCFB2D, 0x2B2F0723, 0x2DE7B54E, +0x2AD24ADA, 0x261E2931, 0xD538CEDF, 0xE12A1C2C, 0x33DDEDE9, 0x2FFC36C4, 0xE61823CE, 0xC225E1DF, +0xC2102FEF, 0xCFD22617, 0x3EFBDA37, 0xDAD6BCD8, 0x110623E7, 0x3320E2D1, 0x1932B8D5, 0xCCEBBFC9, +0x47234011, 0x0A2B45D8, 0x25C11244, 0x00EF3221, 0xC634CC38, 0x0F34D0E6, 0x1A3F40EA, 0x2DEDC82D, +0xFFD2C8BD, 0x40E321CD, 0x24441BD3, 0xD932EDDD, 0x48362127, 0x1230EADE, 0x39E632F9, 0xD0D451DC, +0x18DD2DC0, 0xC22AC72E, 0xD71E172E, 0xAFDED12B, 0x1DBAA948, 0xF927B647, 0xD7C31C23, 0x3328E73B, +0xDBD72DD9, 0x3E23D424, 0xCB09E025, 0x16BA21CF, 0xDD00B9CB, 0x29E3CAD7, 0x3A19D011, 0xEA0EC4E0, +0xD51CDDBF, 0x2AD1C640, 0x2CD6CBD3, 0x36D6D43C, 0xA72851C2, 0xE7EE3C3F, 0x432A4721, 0x46C726E5, +0x253FD426, 0xD03B2C27, 0x1AEC343B, 0x5A2BC735, 0xCE31D2D2, 0xCFEDF30D, 0x492B4141, 0x20DC1BAD, +0xDAF81CD9, 0xD6EAD21D, 0x2339CD25, 0xA31546BB, 0xD0C0F63F, 0x382BE719, 0xDA34BFDB, 0x1EDBBADF, +0x5151E646, 0xC164D5E2, 0xD8292FCA, 0x33CF1F3B, 0x3FF531F4, 0xD3EB3453, 0xD433E236, 0x1B102B1C, +0xC6C320E8, 0xD8393C2E, 0x17EAB7EB, 0xE0C42CE5, 0x313720CC, 0x3F39C42D, 0x33374AD5, 0xCE1ED41E, +0xF7D2BE0D, 0xF114311E, 0xE22CD6EB, 0x4E23CA2A, 0xCDCF36D3, 0x463B1EC4, 0x312DE400, 0x1A1219CE, +0x0CD1E2C3, 0x3850CBEA, 0x31D7CCCD, 0x3F2D3936, 0xC518E33E, 0xC31F31B7, 0xC0AED6C9, 0x2AC6E5D9, +0xC334B8BF, 0x35EFD72E, 0xD72B3AD8, 0x31073AD7, 0x25E22D38, 0xF0CD4A64, 0x28DFDB2D, 0x22D9E714, +0xB8E43935, 0x35FCBFEC, 0x1DC1391E, 0xD6E731A5, 0xBBC8B837, 0xBD3FBE3A, 0x00BADC53, 0x28C43F33, +0x1CE0BAC2, 0xE0E125D9, 0x3BD6CB3D, 0x313230BE, 0xC4243D34, 0x1F24D2B1, 0x2058384C, 0x2AE9192D, +0x302CCCEE, 0xD435C4C7, 0xB0D53217, 0xE52D32D9, 0xE0AF2127, 0x3E47D0CA, 0x4112BFCA, 0x27D0D0BD, +0xE81E5CF9, 0xD50CBCE5, 0x2DC9B4F7, 0x3FBA33BC, 0xCBCC2821, 0xDFD51326, 0xD20ADA3A, 0x2000C3BB, +0xC4DC4BE5, 0xCD39D7A7, 0xC40FF54F, 0xCC3BD1CC, 0x2C19EACC, 0xCD2E2129, 0x44D9F0DB, 0x2B40E11C, +0x31343F4C, 0x0E142817, 0xD0104135, 0xE304CAC7, 0x259A1FD4, 0xC3EAED3C, 0x1A311AD7, 0xD8242BEE, +0x20BF4EFD, 0x3FCA16C5, 0xDDEED03A, 0x0A22C52E, 0xD53B2D3A, 0xE22A2CB5, 0x291D3228, 0x2CCC2E45, +0x48BA00D5, 0x4FE12D1D, 0xD8D91DD1, 0xCEDBEC31, 0xFBD526B9, 0x3C1DE845, 0xAFDCE91C, 0xD622D31E, +0xCF2620D9, 0x28D63427, 0x362C3DDF, 0xD72C2530, 0xD6A7C203, 0x442C3233, 0x29B24E3F, 0xDD21412C, +0x400BD6D0, 0xC8D4B831, 0x171EF12A, 0x34FFC9D0, 0x3B3ED02D, 0x0BC7E2CB, 0xC43D28C6, 0x42F6C948, +0xD24AD429, 0xCB2C2A00, 0x43E2A236, 0xBC353312, 0xD91324B5, 0x30B4D013, 0xC92FDE2F, 0xBFD6D432, +0xC82DCABC, 0xC032DC1D, 0x30C9D2CB, 0xE0341DC5, 0xDDCEBE39, 0x2CC41D58, 0xE213BF14, 0xDFCDCBD9, +0x2E3C36DD, 0xE5262833, 0xF841291D, 0x15E324F5, 0x0315D732, 0xD1C5C5CA, 0xD73144D3, 0x2ABEBA1A, +0xBCF42E2C, 0x00E1E53B, 0x4423AD2C, 0xB021CF20, 0xD8D5EEB6, 0xBD35BCCD, 0xE6D85334, 0xD549DF2D, +0x35C74D19, 0x3F4BBE3D, 0x1F3FA3EB, 0xCEB2B335, 0x35D0BCBA, 0xD5D8CF27, 0xB11DCFC4, 0x24D3D408, +0x183B35D0, 0x1C14D619, 0x35D445BB, 0xDE452444, 0xC83315C5, 0x3F2122D0, 0xB623C737, 0x33321DD0, +0xD6DF33DE, 0xDE39AA20, 0xCF00F239, 0x412410D5, 0xC5DC2339, 0x1D3BD1B6, 0xC0E3481E, 0x1030451A, +0x30EA30D1, 0xC6DCB9C7, 0x2C36DC17, 0xCBB0ED30, 0x26C20FBE, 0x35D6D528, 0xD6CC2242, 0xEDE4241E, +0xC036DED9, 0x26303627, 0xB6B6371B, 0x26E3D6DF, 0x0D36C729, 0xBEEE4FD7, 0xE3C839D1, 0xC0313031, +0xAEAD2828, 0x36323D23, 0x13342728, 0x2904EE20, 0x261DDCD6, 0x0CBCE1E1, 0xD2CBD5C3, 0xFD1C4B2C, +0x1CD8CDDB, 0xD4BE53B1, 0x2BD6CDDE, 0xC61A01E1, 0x3D41BED3, 0xE0333BD0, 0x4040E821, 0xC4A32449, +0xD3D3A8BF, 0x36E11C2C, 0x3CC1321B, 0xC5D03E1C, 0xB813D9DD, 0xD6C51838, 0xCEBED141, 0x1FEA3813, +0x3611CAC0, 0xD12EE2C6, 0xE1BFD22D, 0xD0DAE9CC, 0x0E03DB00, 0xBC37393F, 0x422BC916, 0x49C4D72A, +0xBFC0243B, 0xC1C2F1C3, 0xDF36CC32, 0x49E53EE4, 0xBFC02CDF, 0x2BEE2BE1, 0xC23224C5, 0xD63F16DA, +0x15DBC21F, 0x49E4CC57, 0xBE272CC7, 0x274E0643, 0x5C1DDBE7, 0x34B73DF0, 0xCD2D2EB9, 0x1AD43BCD, +0xFD3047D3, 0xDAC53936, 0x1328DCC8, 0xBC32331D, 0x001F2634, 0xC845410F, 0xDBDB1C2B, 0xD1D819D0, +0xEBDDE12D, 0x37E335D7, 0xCDA84635, 0x0F362FBE, 0x30C229C4, 0xBC1DC43E, 0x3CD02F3D, 0x2EF5F5F4, +0x462D15E6, 0x362F1FC5, 0xC0C9CD49, 0x14C92639, 0x2BD135D0, 0x49C6B8DA, 0xDBC7E1D9, 0xC6382328, +0xEDC217BC, 0xE4D1331C, 0xDCC741CF, 0x283A341C, 0x38B93420, 0x46001BE7, 0x3E1030C4, 0x42A8C1CD, +0xBB3339C5, 0xE538DD20, 0xDF3F2602, 0xE52BE5C1, 0xBE2B41BA, 0x3BD4D337, 0x2D39C3DB, 0xC3BB36F0, +0xD12CCF17, 0x4130B93A, 0xE31DEAEC, 0xF9DEF52D, 0xADD35231, 0xD62ED0EF, 0xBE303521, 0x1E2ADB41, +0x310AC62F, 0x22DA2DC5, 0x26E1D9D0, 0x1F4D2AC1, 0xC8E6DCE1, 0xC6B3D62E, 0xD6D20019, 0x4EDC2D39, +0x342AE224, 0x07CBD4CB, 0xE43FB3BE, 0x1BE53748, 0xDD44C8A2, 0xBC34B7BA, 0xA1DD1D43, 0x23D31827, +0x3519E93F, 0xC83B3720, 0x1BF0B72A, 0xEAC7DCC0, 0xCA4B28EC, 0xD54D242F, 0xF35435D1, 0x3DDEC824, +0xC31128CE, 0x0B0FD136, 0x442D1E2B, 0xCBD1C3EC, 0xD635341B, 0xDD1F2931, 0x3442CDD0, 0xDADD3600, +0xC31E40E3, 0xB0BD1B22, 0x38CA1F0D, 0xD42D3043, 0xE65127E4, 0x2F15C9D6, 0x3E360B30, 0xB2CB1524, +0xC7BBC4DF, 0x38D244C7, 0x483B31C0, 0xDB43C8D1, 0x43DCDBBC, 0xDCC7571D, 0x24BAC2C6, 0x3B4ED6DC, +0x3AE03245, 0x0C40E1E2, 0x493724C0, 0xD4ECC537, 0x36CC30BA, 0x1C303FC6, 0xD12F27D5, 0x00C4272F, +0xFFFD061D, 0xC2D9251D, 0xC1D7D625, 0xBFCD3AD3, 0xB42E11D4, 0xC7230E1E, 0x24CD2C21, 0x0BD436F9, +0xCECCD331, 0xE6BDE0D5, 0xD2C6C941, 0xE1D1D3EF, 0xEB4123CD, 0xDA1755E5, 0xCEE4C529, 0xDC43C5D1, +0xD8FDDFEA, 0x1AC9C317, 0x22E13DCE, 0x01D7BA29, 0x1551E449, 0x43CD363E, 0xC6281B24, 0xCBD23725, +0x3200D61C, 0x35D6CCBC, 0x262CB22C, 0xE2E43DB1, 0x26493335, 0x3AD72ACA, 0xC4244430, 0x3AD7C9C3, +0xC91ED8D1, 0x40EAD2AC, 0x38C9CDAC, 0x413BC137, 0xCD304B1F, 0xEFCA2323, 0xAC3FE4CD, 0x3616F6E0, +0x41C2272A, 0xDCD7B2E2, 0x27D9E018, 0xDEE2D943, 0xD5C01F22, 0xB0D93540, 0xB411C13D, 0xDE39CBE0, +0xE2C6F126, 0x31450828, 0xD2BC2927, 0xBACA1DD1, 0x34DDCDC3, 0xD8302AD5, 0xE2D02824, 0x0B2BE03E, +0x3BCAFD2C, 0x2E461D45, 0xBBF6F32E, 0x31E5DED2, 0x2ED4E847, 0x3BDDCD29, 0x231331D8, 0xF521CF2E, +0xD024C718, 0x20D91011, 0xDB1E0F34, 0xD931C0BB, 0xBB310837, 0x29B7C5CB, 0x3B31DEE0, 0x0836B926, +0x28CB1FD4, 0x31072A30, 0xEE41F300, 0x35CB501E, 0x603A2B47, 0x2C2739CE, 0xC0CFC4D9, 0xE0CED43C, +0x22BDC9CA, 0xBAEC122D, 0x4D16F2C5, 0xB6E5C3DA, 0x19D5C9D0, 0x1D1DDFC2, 0x223A23FB, 0xCD382B09, +0x3025C2E6, 0x3918BDD5, 0x33DFEF0E, 0x291A302B, 0x14184930, 0xD414D233, 0x473CCED7, 0xD2D318B7, +0x3E2738D2, 0x2928D537, 0x002E14BB, 0xE2E03734, 0x1B1EDFD1, 0xCF483FD8, 0x3D043844, 0xD52E1C4C, +0x2823EAD6, 0x1F2D241A, 0xBC4B22E6, 0x2EE7DFB8, 0x3424E2F0, 0xD41BD7CB, 0x2BBCDF1E, 0x155B5033, +0xB527D543, 0xE3C5E9F4, 0xD9B5C92B, 0xD3CAE0DF, 0x31C4CED7, 0x0BD33B25, 0xCF3B111D, 0xE62F241F, +0x22D93D22, 0xCF27EF43, 0xDAD2C5DD, 0xC60019BE, 0x2935D5C3, 0x2D0BCD22, 0x224714EA, 0xE7CAB817, +0xD1282222, 0x3C25EBD0, 0x45F0A8DC, 0x3ACFB92D, 0x34B4E7E6, 0xE93523E0, 0x2C292FCA, 0x44D4C7AC, +0xEA22C3FE, 0xEF37E524, 0x313ADCDA, 0x2212D846, 0x1BE0FF4B, 0x1CBC2E34, 0x174AF041, 0xD245223E, +0xE1DB2BDC, 0xF2E9A515, 0xE3C33BCC, 0x43DBD5CE, 0xF11700DB, 0xDE44DD36, 0x26E1D604, 0xC2ACE419, +0xB9D8C7D4, 0x37EFE21E, 0xE2BAC2D1, 0x3926E91A, 0xD013E8BC, 0x2C43ECD9, 0x232B30F8, 0xD6C9E0BD, +0x26A718BC, 0xEFF6C840, 0xC13332C2, 0xC4B9BD36, 0x1B4921BD, 0xD0EBECE0, 0xBCE4D647, 0x35D4DCD5, +0x1BBE4C27, 0xECC1B7CB, 0xD820284D, 0x0BC33DBF, 0x34D03140, 0x38EC3900, 0x29EBDBC9, 0xD43A3041, +0x3AE0DD0C, 0x09CFCFDB, 0x44E0C7CE, 0x20DBC4E3, 0x26DC41DE, 0xBA3AD2E5, 0x2ABA34E1, 0xD70FBEB7, +0x40E03FE2, 0xF8D62041, 0xBD47B7EA, 0xD2E03E34, 0x1FC6D2DE, 0xCF1CD155, 0x32D52630, 0x3C463D2C, +0x253130E3, 0x1ED8BB4A, 0x411BDFDD, 0x3FD8EE35, 0x33D02BBB, 0x002ECAAE, 0xDF1FDFEB, 0x2CD8DA3C, +0xB6352D59, 0xD5BDD3B9, 0x1946630D, 0xD9181731, 0xD419D31F, 0xCE2FB7D0, 0x32EFE3C3, 0xCDDB2FD3, +0xD247BB3E, 0xBA41DECA, 0x23244AD2, 0xDEAAF2B2, 0xB8D525F2, 0xD327A4D0, 0x3718C0E2, 0x2D3E20D9, +0x24CFDF12, 0x26D0BE1C, 0xC8D9282E, 0xEB54CADF, 0x44DB1AED, 0xC927BF29, 0x2E002B39, 0xBB2625E4, +0xBB4421D2, 0x18CBAA3A, 0x213ECD37, 0x0F33163C, 0x3622E432, 0xE5C7C62B, 0xDE39E7C7, 0xF3151CCA, +0x2ACED6D2, 0x2ADADFC4, 0xD6241ACC, 0xB92F0622, 0x1BC639E4, 0x33D92BC4, 0xAF14E0DD, 0xE6EEC2C9, +0x16BDCAE3, 0x272E1AC3, 0x292E1EBE, 0xDCE439B1, 0x4225E1C6, 0xCD372420, 0x27D716B1, 0xD83EE339, +0xDBCAD4B3, 0x26385033, 0xC5373528, 0x2DD2C936, 0xF9D5BAE6, 0x2E2DC7C2, 0xCD4BD045, 0x1DDF2DE8, +0x0E3740CE, 0xE5D82C36, 0x29D1360A, 0xCD03C8C6, 0x37CCCCD8, 0xD83AD330, 0xBFD619CD, 0xDEDDC432, +0xD0DADAEC, 0xD3E14F26, 0xA8CD35D9, 0xA3F325BD, 0xBC1626D7, 0x3AB81BD5, 0xBBFF1832, 0xC9DB2126, +0x16250000, 0xE003BB39, 0x35D533CE, 0x1801C610, 0xD9D44644, 0x474BD3C3, 0x1C30C51E, 0x2E3342C6, +0xB7D6B3E9, 0x5AE44BD2, 0x46D62830, 0xDBDD4B0C, 0x38CF2F1D, 0x2D331ACB, 0x36CCDED6, 0x10D7DA10, +0xD42246E0, 0x0FC73FC0, 0x2C3E1E37, 0x2CF0D1B8, 0x17412EDD, 0x2630D0DD, 0xDBD535E4, 0xD927DA43, +0xB636C11E, 0x25CBE2D8, 0x4711F64B, 0xBC2C2029, 0xD0E3F538, 0xB92C27DF, 0x272DE2D2, 0x0CC5BCEB, +0xC0BC2EDF, 0x26D131C9, 0x1F092FEE, 0x2D342B34, 0x31B825D5, 0xC6BBC9C1, 0x38F1D0E4, 0xCBF7FDC7, +0xC3CAC9B6, 0xDBC436C6, 0x19DC1FD8, 0x28CCAED1, 0x1921BDB4, 0x24BCBF33, 0x301EE244, 0x36BDBDE1, +0x31C5B724, 0x00003ECD, 0x220B1EE7, 0xDEBAD3DC, 0x21CAB942, 0xC9193B33, 0xE3CD3A0A, 0x0DC22CC3, +0x3EC0C7CB, 0x43F2BFE4, 0x0ECFE9C6, 0x3ECA2FD5, 0xCB492DD9, 0x23E7BDC9, 0xD33DC151, 0xFCBE2933, +0x5ACEC51E, 0xF2172E1C, 0x3C24DE23, 0x3320D4D5, 0x2AB53C20, 0xC328323A, 0x45EA4133, 0x24391A32, +0xC7D8DA2B, 0xF4EAAF39, 0xDFCA36E3, 0xDEBED5C8, 0x36252C36, 0x04D7DBC7, 0x30CAC23D, 0xB60BF73D, +0xEACA2A30, 0x34C12ABE, 0xE2D747D8, 0x19D21FE3, 0x1DFD24EC, 0xB6B5D9C4, 0xBF2C1DD2, 0xD2B442E0, +0x37CCE1BD, 0x2AD1E1CC, 0x18DAC6BD, 0x213AD93E, 0x2031D8CB, 0xD82F2D12, 0xD112D2F2, 0x19CD43F1, +0x36BCB9CA, 0xD7C5DDFC, 0x34011D28, 0x3BDB0000, 0xCD00E1AA, 0x5228252C, 0x3512CA34, 0x0EE53A41, +0x242CD2C2, 0xF425DD2A, 0xF02FDDF8, 0xD82026D3, 0x4ABEE7B8, 0x2DC9DDDC, 0x3BEECD3C, 0x2ED1442A, +0xC713E92A, 0x310CC12B, 0xE128D520, 0x48E5C1CF, 0xC91CE3ED, 0xCCCBC72F, 0x0B23E3E9, 0xC1D2C8B6, +0x511ED534, 0x2AC7D4C3, 0xCFDCE335, 0xD4C9F643, 0xD433EBF1, 0x1BF8DA31, 0x223FC515, 0xCC2F13C2, +0xF825FFC8, 0x35E51633, 0x36293C4A, 0xC72ACA36, 0x4544E736, 0x1BD9315C, 0xCF16DFD4, 0xD0AFEFC3, +0xCECBBDE2, 0x05D6AA32, 0xD851F045, 0x240FD948, 0x3D35B2E6, 0x17D7C7EE, 0x0B22DACE, 0xF53AD637, +0xB2372DDB, 0xF6DF14CB, 0xBED95B24, 0xD4D5190B, 0x0000CB29, 0x3DC1ECF5, 0x30C6B34F, 0x1D2629B6, +0xD429D1CC, 0xD50716E3, 0xD0C3C2CC, 0xC82CC7D4, 0x5F27C718, 0xE6201BE4, 0xDACFD42F, 0x2E2EE9D4, +0x3AD036C0, 0x3828DB20, 0x27D63439, 0xD21026DE, 0xD60710BC, 0x13AEB9E9, 0x28C41944, 0x49D5E3D7, +0x27163429, 0x1BBAD528, 0xB12A14BC, 0xC72FD9E2, 0x1F43213A, 0x133B3EE5, 0x3622FE21, 0xD438DCBE, +0x41CB25E8, 0xD0D83E2C, 0x32BFC2CE, 0x362BE1DA, 0xC3CBCE24, 0x22DECDC2, 0x30CBD5CD, 0xD42DBC26, +0xDDF6AAD0, 0xCADC0EC8, 0xDC333434, 0x333114F5, 0xCE24361A, 0xB1C5D123, 0x4B25380B, 0x2FEC3634, +0x3EE2A610, 0xB4CEE72F, 0x29BCC721, 0xB23850CD, 0xA51AE4DD, 0xC81BBC2F, 0xD4DF0000, 0x2DE5D1E8, +0x2E0A28D0, 0x29E3C1E1, 0x56F62DD6, 0x3828DD4B, 0xC41ACFDF, 0xD2390A3A, 0xDF3028CC, 0xE8C5E22B, +0xD11CDCC1, 0x44D1CBEC, 0x33C73DDC, 0x1D25B7DC, 0xE7DD0525, 0x2743C41A, 0x2AE2D3DC, 0x21D6E82A, +0x133CBAC7, 0x593F34B0, 0xDA5FE339, 0x5206252C, 0x394FD5BD, 0x21D2C92C, 0x292426E0, 0x48E0362B, +0xCC272DE4, 0xC815C4DE, 0x49BD5D37, 0x1912342F, 0x2CE9E01B, 0x30D8D2D9, 0xDEBF46E8, 0x392F473A, +0x3A18D324, 0x2FE31FD0, 0x2B2B2710, 0xB53B2AC7, 0xB3481CFF, 0x2DD7CFBC, 0x2232351A, 0xD7C91C16, +0x1CB4DA16, 0x1BCBDF3C, 0xD2213822, 0x2121DB1F, 0xDBD1C02A, 0x27DA33C7, 0x351FC4CD, 0x0000B9D8, +0xC2142C29, 0x38B9DD31, 0x54E60CDF, 0x2A4A42DB, 0x35CF2BB5, 0x0CEA3323, 0xC2D8BEE2, 0xC61F3B33, +0x10CE11D0, 0x3336CE2E, 0x303ADD25, 0x3347B7D9, 0x33222C1A, 0x1A43E4C1, 0x4326E019, 0x2AE2E92E, +0xD2C5DC40, 0xBA45D7D6, 0xDF341BDC, 0x3A361925, 0xD4FEF4D2, 0x3018DA31, 0xDDC12817, 0x3038DE24, +0xEDC0292B, 0x3E3E3336, 0xCB1BEF17, 0xE10B2EBF, 0x2B20BABF, 0xC80C421B, 0xC2411D21, 0xECD8C7D6, +0xC824E4AD, 0x65B53DD7, 0x304325D4, 0xDD1DEAE6, 0xD33131CF, 0x2F36452B, 0x23B219E9, 0xC1C23529, +0x21DC3836, 0xD618E933, 0x393E2C14, 0x362ACA3A, 0xD6E2C237, 0xD6085111, 0xD721FCB1, 0xCCE6DEBC, +0xD1DCD920, 0xC3320000, 0xEA26CB19, 0x18BB26B7, 0x233BDF4F, 0xDE23FD3F, 0x2ADA1ECB, 0x3122E839, +0xEF42D7D7, 0xCAC7202D, 0xCD4CDED1, 0x2633C824, 0xD0123E4E, 0x104429B6, 0xC8CD4B00, 0x5136B738, +0xCFC8202B, 0x163135DD, 0x2C3821E8, 0x29314CE0, 0x0F3CE4EC, 0xE5432DAF, 0xC63AE340, 0x4639DB18, +0xDF282511, 0x25B5CBD3, 0xCAF50F37, 0x1E443B33, 0xE32CD32D, 0x10DA47B9, 0xDA410526, 0x34F0E4EA, +0xCEED3134, 0xC0B31AA7, 0xD12B1AD3, 0xC722C743, 0xBFBEDD1F, 0xD328D52B, 0xCC384024, 0x18B43FB8, +0x281742BB, 0xD8322FD2, 0xB5D628C6, 0xE420C0CB, 0xC9EB391D, 0x41E6B419, 0xBCC543E1, 0x2C37DDEF, +0x1F2D41E9, 0xD0F0D205, 0x1FEAE5C5, 0x15DF14B8, 0x47E327D9, 0xD328CE25, 0x36DB2E24, 0xD92E1EBF, +0x2B3230F4, 0xD1EAD5D9, 0xC7212ADA, 0xCD1235DC, 0xC519D5DB, 0xBFC631DF, 0xD5222B35, 0x35B6DEC6, +0x153C4428, 0xC5C3E42F, 0x1B2E2D2A, 0x3CD422A6, 0xC22842CC, 0xCADAC43D, 0x2C10DAD2, 0x18C708B5, +0x36D636BD, 0xCD29C5D3, 0xBFED433A, 0xE0384939, 0x005629F2, 0x36E2C425, 0x2830DF1C, 0xC9CB351C, +0xB0D7D3E6, 0xF3CEC214, 0xDC25D1C7, 0x1BD5CBCC, 0x19DD21C9, 0xC854EE30, 0x4FE14EE3, 0x38D5D3E8, +0xC9D126E4, 0x2FDD2130, 0xCE2130D1, 0xCFCAB0C9, 0xC71F0E1A, 0x2627D4DE, 0xCBE9DACD, 0xE7121F22, +0x2CC8E435, 0x3838391D, 0xDD37242E, 0xEBBE2F10, 0x233C0000, 0xAFC91CE7, 0xCE3536F6, 0x0F25D248, +0x2D04EBE4, 0x3FF634E5, 0xE929CE26, 0x24E0BABE, 0x302CD52B, 0xD1E2073F, 0x4333D11C, 0xDA48C2D1, +0x2D33F315, 0x29B2C8C3, 0x3B26E3EB, 0x3821C6DA, 0x32163DD6, 0x1DC737BC, 0x463CCD24, 0x19D63DDD, +0x200BDDB7, 0xBDD41E22, 0xC40A444F, 0x34BF49ED, 0xD4C0C0F4, 0xA41FB721, 0xBEDF51E8, 0x15D82A17, +0x383FDF21, 0x2B2918D0, 0xE6EB4718, 0x244530DB, 0x25DD3919, 0x3BA819BE, 0xDAEFBE38, 0xEB3FCCD9, +0xD1DF2E1C, 0x2520C3E3, 0x32D0BFD5, 0x37312BD5, 0xDC29D0C9, 0xBED02432, 0x310C3A18, 0x411ED8D7, +0xA4E5CE22, 0x163CBD30, 0xC3E2BE45, 0xCFBBAAC3, 0xC8C0E4CF, 0x0000CEFC, 0xE8CC2433, 0x39ED223F, +0x243AD83B, 0xD44D39C6, 0xCABF24DE, 0x252050EB, 0x29C73B40, 0xD4F434CF, 0xCAE4D5DF, 0xC5F2EEF1, +0x2919C8D2, 0x2FC4D20B, 0x1CBF5528, 0x3DBBD52D, 0xE03AD031, 0x3D2CCA17, 0xBE23BDB8, 0xE7DA3B30, +0xD1D2D331, 0xCC1825EA, 0x2F2D39D2, 0xF9313526, 0xCFBCC2E1, 0xE5BE3AD4, 0xBF59BE1C, 0x22FFC8B5, +0x32B14015, 0x31D6BEE6, 0x46DF3843, 0xDCD1D8B2, 0xE728C8BE, 0xEA1E2A28, 0xD1D8C7D2, 0x30CAF134, +0xEDD0EADA, 0x13C1E3D0, 0xB9D7DC29, 0xEAB93624, 0x24E49F38, 0x2E3FDEDC, 0xE6D0D23D, 0x1FBB3ED2, +0x23D540DD, 0xD727D92B, 0x251E33E6, 0x4BA7EECE, 0xC1CD203C, 0xC5CFED32, 0xB9CF35C3, 0x29710000, +0x29D7C636, 0x3A3ED1D0, 0xCC1B271F, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, +0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000 + +hard_output0 = +0x42441F44, 0x006DFAE9, 0xDBB63AC2, 0xA3365CB3, 0xFA44372B, 0xC2603862, 0xA6C86E09, 0x416F0BF7, +0x8E58011D, 0x47C54B99, 0x269AD14C, 0x0B286902, 0x093E4C64, 0x8D84D8E1, 0xB8AE0993, 0x49567118, +0x8CEAABD2, 0x12D33409, 0xE3358237, 0x130176F6, 0x0986718F, 0xA18C83B5, 0x04EEF058, 0x59704040, +0x7478FE81, 0x01D27193, 0xEA1AD7F3, 0x21E6C2B9, 0x6478A1FF, 0xED9959AD, 0xE39E57BE, 0x4D859105, +0x056EF72D, 0xBD170BB7, 0xF01F9ADD, 0x99BF0C05, 0x44BAD09F, 0xF6ABDD61, 0xC0F8F116, 0x2972B53C, +0x0FEC944B, 0xD1675432, 0x0025F563, 0xF42B2EA8, 0x808E5C37, 0xB6E79AD5, 0x3706284D, 0xB9C0AFA5, +0xFF0E7E28, 0x5FA45C6F, 0x9CD5244E, 0xA013DDEA, 0x0D27D1A2, 0x2AE414AD, 0x41614379, 0xE6B68872, +0x5585D926, 0x5098D45B, 0xF8980ABD, 0x65821418, 0xEF8968A3, 0x301DEC3A, 0x57EF2A7F, 0xC17BE446, +0x94B65D62, 0xC9E6F350, 0x2E6130BC, 0x235F2E5E, 0xF1C13241, 0x94B291D6, 0x8C342458, 0x175B1FCA, +0xE07310FD, 0xD24934DD, 0x3CE8D053, 0x5C8F243D, 0x945B0AB3, 0x50EB8CC1, 0x8EF499D4, 0xA67801BF, +0x1680F061, 0x283FE705, 0xB8D7E773, 0x13AD3D2F, 0x4A6C305B, 0x1C1E5B12, 0x6F57D880, 0x8A666E5E, +0x26511296, 0xFFC09750, 0xFF91760A, 0xAC26795C, 0x75F321FD, 0x4221B9CB, 0x2E119188, 0x0772A832, +0x74D6036B, 0x505D6D64, 0x16A001B7, 0xF5F0D728, 0x48CBE119, 0x91C5D6B2, 0x8E1BBAB0, 0xB11BBE51, +0x52C647DC, 0x65240CE2, 0xAB3F8AAE, 0x62E0CDEA, 0x6CBE7240, 0x7F572063, 0x7FC816E6, 0xB38E5A6C, +0xDF9E2738, 0x752419CB, 0xC3A6BF9B, 0x8D48B21F, 0xA88C4AED, 0xE3C1D8BC, 0x8501353D, 0x20A99D7D, +0x8D9C6DDD, 0x77A65039, 0x834E7E57, 0x1F623FA1, 0x6CF02B26, 0x1FF0BBA6, 0x9035AC70, 0xBED818E5, +0x437CA713, 0x91011537, 0x3CD1302B, 0x7755459E, 0x5963FDA6, 0x49ECF937, 0xDEBBE0E3, 0xE873FD4F, +0x735C48C9, 0x7C70F46F, 0xA00B1543, 0x789F4159, 0xE17C343E, 0x51B3D085, 0xD9042663, 0x771933DA, +0xE183AF8C, 0xF62AC11B, 0xF776C25C, 0x22083E6C, 0x8400DDBA, 0x6D6AD2F2, 0x5DF87DBC, 0xBD7D40B4, +0x356EBF21, 0xD8972E85, 0x3A632B51, 0x4D5BFBA2, 0x58CDF1D4, 0xD1117491, 0x5906DC45, 0xF793D63A, +0x8F9D0552, 0xD261C804, 0x70BFDE9F, 0xDC77827B, 0xE9FFB130, 0x4FF18BBC, 0x17D50C32, 0xC315521E, +0x4C7D2470, 0xAB5EE34C, 0x692AC0C8, 0x15B22832, 0xD34107FE, 0xA7E6DA1E, 0x2966A2A3, 0xD0B204B4, +0x500E02F9, 0x02E5A455, 0x81EAF941, 0x6818F57A, 0x978EEC9C, 0xBFD575F3, 0x3FFC1BE8, 0x97104E07, +0x4ADF219F, 0x182EC93D, 0x386033EE, 0x87C1351A, 0x31420C6A, 0xE20B5DC8, 0x83D90D23, 0x3C13B9E8, +0xCCCD65BE, 0x7EDB17A9, 0x81C73C2A, 0xE202E332 + +c = +2 + +cab = +1 + +ea = +4918 + +eb = +4920 + +c_neg = +0 + +k_neg = +3072 + +k_pos = +3136 + +rv_index = +0 + +iter_max = +8 + +iter_min = +4 + +expected_iter_count = +8 + +ext_scale = +15 + +num_maps = +0 + +code_block_mode = +0 + +op_flags = +RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE, RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN, +RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP + +expected_status = +OK \ No newline at end of file diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1190_rm.data b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1190_rm.data new file mode 100644 index 00000000..6221756a --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1190_rm.data @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_ENC + +input0 = +0x11D2BCAC, 0x4D + +output0 = +0xD2399179, 0x640EB999, 0x2CBAF577, 0xAF224AE2, 0x9D139927, 0xE6909B29, 0xA25B7F47, 0x2AA224CE, +0x399179F2, 0x0EB999D2, 0xBAF57764, 0x224AE22C, 0x139927AF, 0x909B299D, 0x5B7F47E6, 0xA224CEA2, +0x9179F22A, 0xB999D239, 0xF577640E, 0x4AE22CBA, 0x9927AF22, 0x9B299D13, 0x7F47E690, 0x24CEA25B, +0x79F22AA2, 0x99D23991, 0x77640EB9, 0xE22CBAF5, 0x27AF224A, 0x299D1399, 0x47E6909B, 0xCEA25B7F, +0xF22AA224, 0xD2399179, 0x640EB999, 0x2CBAF577, 0xAF224AE2, 0x24 + +e = +1190 + +k = +40 + +ncb = +192 + +rv_index = +0 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_RATE_MATCH + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1194_rm.data b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1194_rm.data new file mode 100644 index 00000000..c569abd7 --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1194_rm.data @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_ENC + +input0 = +0x11D2BCAC, 0x4D + +output0 = +0xB3E8D6DF, 0xBC8A2889, 0x744E649E, 0x99436EA6, 0x8B6EFD1D, 0xAB889238, 0xE744E6C9, 0x39E4664A, +0xE8D6DF91, 0x8A2889B3, 0x4E649EBC, 0x436EA674, 0x6EFD1D99, 0x8892388B, 0x44E6C9AB, 0xE4664AE7, +0xD6DF9139, 0x2889B3E8, 0x649EBC8A, 0x6EA6744E, 0xFD1D9943, 0x92388B6E, 0xE6C9AB88, 0x664AE744, +0xDF9139E4, 0x89B3E8D6, 0x9EBC8A28, 0xA6744E64, 0x1D99436E, 0x388B6EFD, 0xC9AB8892, 0x4AE744E6, +0x9139E466, 0xB3E8D6DF, 0xBC8A2889, 0x744E649E, 0x99436EA6, 0xC01D + +e = +1194 + +k = +40 + +ncb = +192 + +rv_index = +2 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_RATE_MATCH + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1196_rm.data b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1196_rm.data new file mode 100644 index 00000000..72be6f5d --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e1196_rm.data @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_ENC + +input0 = +0x11D2BCAC, 0x4D + +output0 = +0xBC8A2889, 0x744E649E, 0x99436EA6, 0x8B6EFD1D, 0xAB889238, 0xE744E6C9, 0x39E4664A, 0xE8D6DF91, +0x8A2889B3, 0x4E649EBC, 0x436EA674, 0x6EFD1D99, 0x8892388B, 0x44E6C9AB, 0xE4664AE7, 0xD6DF9139, +0x2889B3E8, 0x649EBC8A, 0x6EA6744E, 0xFD1D9943, 0x92388B6E, 0xE6C9AB88, 0x664AE744, 0xDF9139E4, +0x89B3E8D6, 0x9EBC8A28, 0xA6744E64, 0x1D99436E, 0x388B6EFD, 0xC9AB8892, 0x4AE744E6, 0x9139E466, +0xB3E8D6DF, 0xBC8A2889, 0x744E649E, 0x99436EA6, 0x8B6EFD1D, 0x9038 + +e = +1196 + +k = +40 + +ncb = +192 + +rv_index = +3 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_RATE_MATCH + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e272_rm.data b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e272_rm.data new file mode 100644 index 00000000..883a76cf --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k40_r0_e272_rm.data @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_ENC + +input0 = +0x11d2bcac, 0x4d + +output0 = +0xd2399179, 0x640eb999, 0x2cbaf577, 0xaf224ae2, 0x9d139927, 0xe6909b29, 0xa25b7f47, 0x2aa224ce, +0x79f2 + +e = +272 + +k = +40 + +ncb = +192 + +rv_index = +0 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_RATE_MATCH + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e120_rm_rvidx.data b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e120_rm_rvidx.data new file mode 100644 index 00000000..b3867f35 --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e120_rm_rvidx.data @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_ENC + +input0 = +0x11d2bcac, 0x7715ae4d, 0xc9f4dbc6, 0x2fda3c1a, 0x09349b16, 0x2cd3c189, +0xe5650429, 0xc73c42c1, 0xe7336bb7, 0xbb7de593, 0x83f986aa, 0xc0ade12e, +0x6730b095, 0x78f7c059, 0xaa907199, 0x75c323d1, 0x7061f9ba, 0x97e067bf, +0xc155cd55, 0x6a4cb08d, 0x4260e2fa, 0xff35e496, 0x37f251d3, 0x02fd9f9e, +0x6f53345c, 0x790cda6d, 0x3b8549e4, 0x56d0c6ea, 0x70a38006, 0xfff18223, +0x6f2035b5, 0x6cf508d3, 0x98a0a308, 0x432353fc, 0x0eb818eb, 0xdd9a128b, +0xf92431b2, 0xad788286, 0xda07d5de, 0x44b4b3a8, 0xbe62fd67, 0x61a4eb54, +0x90d7ac9f, 0xc4d9a930, 0xfddc3c24, 0xf5e3b1c8, 0x38143538, 0xcb1d3062, +0xcae36df3, 0x50a73437, 0x542dcab7, 0x875973a0, 0x34690309, 0x49cb1ddb, +0xf8a62b92, 0x82af6103, 0xc9f8c928, 0x3835b822, 0x16b044a6, 0xdae89096, +0xa5be47d4, 0xaf61189e, 0x5cd70faf, 0x037331cf, 0xef7fa5f6, 0xb2f2b41a, +0xa6f222c6, 0xdb60fe4c, 0x2a857a9a, 0x0b821f9d, 0x348afd17, 0x7eecbaeb, +0x92bb9509, 0x8a3cec24, 0xd02549a2, 0x155ffa81, 0x2b7feac6, 0x3ee461e7, +0xc981f936, 0x89b544c7, 0x9a431e36, 0x62511734, 0x769f9647, 0x211a747e, +0x567abef4, 0xad87e2b4, 0xa3e0c3bf, 0x6d325dd5, 0xf561cc46, 0x39925735, +0x3d8abbfd, 0xc3724c88, 0x8bdf03c9, 0x1b02a12a, 0x4f233a0c, 0x9ca9444a, +0xc5d1f7e6, 0x4d995f37, 0xd9aefb32, 0xd0465248, 0x0f3a3b21, 0x62ea8c0c, +0x91f8d54e, 0x5cf75514, 0x14618a01, 0x8fe9b87e, 0xf2b424f9, 0x49724ce2, +0xa1464587, 0x5e00dc83, 0x59475455, 0x444119b1, 0x4fb9f036, 0x65fcbc1a, +0x1c63a990, 0x767a6114, 0xb0ede06a, 0xcfb91ae3, 0x7874af5f, 0xf78772a3, +0xa2932c81, 0x77f2759d, 0x930dc8f1, 0x95ce14a5, 0x134363ee, 0x61ee143f, +0xf0034b35, 0xfdc75fce, 0x3be2dcf3, 0xff3a07eb, 0xdc43f0eb, 0x23ba73bb, +0x45f7649e, 0xcacc297c, 0xa3dd98db, 0x058d46a9, 0x6bcfc154, 0x7be8e1e6, +0x618a4754, 0x8d193c46, 0xba39e1ce, 0xc3b85cfc, 0xd80d853b, 0x38d6440d, +0x9c1a6185, 0x90c9dfcb, 0x01c6e841, 0xeedfe6ac, 0x7b61a3ae, 0xad3924dd, +0x514bc1a8, 0x6ed60d7e, 0x1b74b79b, 0xea295947, 0x5f9a5d33, 0xcd24311f, +0x0bd3d10c, 0xd214ecfe, 0xcb37035d, 0x8ce95168, 0x7020cb52, 0xe432107d, +0x042d63ac, 0x6201c8dd, 0x4bea65a2, 0x1a3cf453, 0x5b8e868d, 0x3d653bb0, +0x24c1967e, 0x37f183a9, 0x3700e703, 0xb168a02b, 0x2592cd82, 0x5c0cdb66, +0x70b64ebd, 0xf8dcb9a7, 0x634a8335, 0x06642398, 0xfe2f4497, 0x2e775256, +0x30f85cbe, 0xf4d2fdaf, 0x46ca6c9d, 0x022097d6, 0xcedeb03e, 0x3d8353bc, +0x0d005559, 0xd457eb38, 0xb1931313, 0xd4f12824, 0xcc444f1b, 0xfaa42c5f + +output0 = +0xd0d936d6, 0x870c26f3, 0x3f12805d, 0x4a6016 + +e = +120 + +k = +6144 + +ncb = +18528 + +rv_index = +1 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_RATE_MATCH, RTE_BBDEV_TURBO_RV_INDEX_BYPASS + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e18444.data b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e18444.data new file mode 100644 index 00000000..dcad5587 --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e18444.data @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_ENC + +input0 = +0x11d2bcac, 0x7715ae4d, 0xc9f4dbc6, 0x2fda3c1a, 0x09349b16, 0x2cd3c189, +0xe5650429, 0xc73c42c1, 0xe7336bb7, 0xbb7de593, 0x83f986aa, 0xc0ade12e, +0x6730b095, 0x78f7c059, 0xaa907199, 0x75c323d1, 0x7061f9ba, 0x97e067bf, +0xc155cd55, 0x6a4cb08d, 0x4260e2fa, 0xff35e496, 0x37f251d3, 0x02fd9f9e, +0x6f53345c, 0x790cda6d, 0x3b8549e4, 0x56d0c6ea, 0x70a38006, 0xfff18223, +0x6f2035b5, 0x6cf508d3, 0x98a0a308, 0x432353fc, 0x0eb818eb, 0xdd9a128b, +0xf92431b2, 0xad788286, 0xda07d5de, 0x44b4b3a8, 0xbe62fd67, 0x61a4eb54, +0x90d7ac9f, 0xc4d9a930, 0xfddc3c24, 0xf5e3b1c8, 0x38143538, 0xcb1d3062, +0xcae36df3, 0x50a73437, 0x542dcab7, 0x875973a0, 0x34690309, 0x49cb1ddb, +0xf8a62b92, 0x82af6103, 0xc9f8c928, 0x3835b822, 0x16b044a6, 0xdae89096, +0xa5be47d4, 0xaf61189e, 0x5cd70faf, 0x037331cf, 0xef7fa5f6, 0xb2f2b41a, +0xa6f222c6, 0xdb60fe4c, 0x2a857a9a, 0x0b821f9d, 0x348afd17, 0x7eecbaeb, +0x92bb9509, 0x8a3cec24, 0xd02549a2, 0x155ffa81, 0x2b7feac6, 0x3ee461e7, +0xc981f936, 0x89b544c7, 0x9a431e36, 0x62511734, 0x769f9647, 0x211a747e, +0x567abef4, 0xad87e2b4, 0xa3e0c3bf, 0x6d325dd5, 0xf561cc46, 0x39925735, +0x3d8abbfd, 0xc3724c88, 0x8bdf03c9, 0x1b02a12a, 0x4f233a0c, 0x9ca9444a, +0xc5d1f7e6, 0x4d995f37, 0xd9aefb32, 0xd0465248, 0x0f3a3b21, 0x62ea8c0c, +0x91f8d54e, 0x5cf75514, 0x14618a01, 0x8fe9b87e, 0xf2b424f9, 0x49724ce2, +0xa1464587, 0x5e00dc83, 0x59475455, 0x444119b1, 0x4fb9f036, 0x65fcbc1a, +0x1c63a990, 0x767a6114, 0xb0ede06a, 0xcfb91ae3, 0x7874af5f, 0xf78772a3, +0xa2932c81, 0x77f2759d, 0x930dc8f1, 0x95ce14a5, 0x134363ee, 0x61ee143f, +0xf0034b35, 0xfdc75fce, 0x3be2dcf3, 0xff3a07eb, 0xdc43f0eb, 0x23ba73bb, +0x45f7649e, 0xcacc297c, 0xa3dd98db, 0x058d46a9, 0x6bcfc154, 0x7be8e1e6, +0x618a4754, 0x8d193c46, 0xba39e1ce, 0xc3b85cfc, 0xd80d853b, 0x38d6440d, +0x9c1a6185, 0x90c9dfcb, 0x01c6e841, 0xeedfe6ac, 0x7b61a3ae, 0xad3924dd, +0x514bc1a8, 0x6ed60d7e, 0x1b74b79b, 0xea295947, 0x5f9a5d33, 0xcd24311f, +0x0bd3d10c, 0xd214ecfe, 0xcb37035d, 0x8ce95168, 0x7020cb52, 0xe432107d, +0x042d63ac, 0x6201c8dd, 0x4bea65a2, 0x1a3cf453, 0x5b8e868d, 0x3d653bb0, +0x24c1967e, 0x37f183a9, 0x3700e703, 0xb168a02b, 0x2592cd82, 0x5c0cdb66, +0x70b64ebd, 0xf8dcb9a7, 0x634a8335, 0x06642398, 0xfe2f4497, 0x2e775256, +0x30f85cbe, 0xf4d2fdaf, 0x46ca6c9d, 0x022097d6, 0xcedeb03e, 0x3d8353bc, +0x0d005559, 0xd457eb38, 0xb1931313, 0xd4f12824, 0xcc444f1b, 0xfaa42c5f + +output0 = +0x11d2bcac, 0x7715ae4d, 0xc9f4dbc6, 0x2fda3c1a, 0x09349b16, 0x2cd3c189, +0xe5650429, 0xc73c42c1, 0xe7336bb7, 0xbb7de593, 0x83f986aa, 0xc0ade12e, +0x6730b095, 0x78f7c059, 0xaa907199, 0x75c323d1, 0x7061f9ba, 0x97e067bf, +0xc155cd55, 0x6a4cb08d, 0x4260e2fa, 0xff35e496, 0x37f251d3, 0x02fd9f9e, +0x6f53345c, 0x790cda6d, 0x3b8549e4, 0x56d0c6ea, 0x70a38006, 0xfff18223, +0x6f2035b5, 0x6cf508d3, 0x98a0a308, 0x432353fc, 0x0eb818eb, 0xdd9a128b, +0xf92431b2, 0xad788286, 0xda07d5de, 0x44b4b3a8, 0xbe62fd67, 0x61a4eb54, +0x90d7ac9f, 0xc4d9a930, 0xfddc3c24, 0xf5e3b1c8, 0x38143538, 0xcb1d3062, +0xcae36df3, 0x50a73437, 0x542dcab7, 0x875973a0, 0x34690309, 0x49cb1ddb, +0xf8a62b92, 0x82af6103, 0xc9f8c928, 0x3835b822, 0x16b044a6, 0xdae89096, +0xa5be47d4, 0xaf61189e, 0x5cd70faf, 0x037331cf, 0xef7fa5f6, 0xb2f2b41a, +0xa6f222c6, 0xdb60fe4c, 0x2a857a9a, 0x0b821f9d, 0x348afd17, 0x7eecbaeb, +0x92bb9509, 0x8a3cec24, 0xd02549a2, 0x155ffa81, 0x2b7feac6, 0x3ee461e7, +0xc981f936, 0x89b544c7, 0x9a431e36, 0x62511734, 0x769f9647, 0x211a747e, +0x567abef4, 0xad87e2b4, 0xa3e0c3bf, 0x6d325dd5, 0xf561cc46, 0x39925735, +0x3d8abbfd, 0xc3724c88, 0x8bdf03c9, 0x1b02a12a, 0x4f233a0c, 0x9ca9444a, +0xc5d1f7e6, 0x4d995f37, 0xd9aefb32, 0xd0465248, 0x0f3a3b21, 0x62ea8c0c, +0x91f8d54e, 0x5cf75514, 0x14618a01, 0x8fe9b87e, 0xf2b424f9, 0x49724ce2, +0xa1464587, 0x5e00dc83, 0x59475455, 0x444119b1, 0x4fb9f036, 0x65fcbc1a, +0x1c63a990, 0x767a6114, 0xb0ede06a, 0xcfb91ae3, 0x7874af5f, 0xf78772a3, +0xa2932c81, 0x77f2759d, 0x930dc8f1, 0x95ce14a5, 0x134363ee, 0x61ee143f, +0xf0034b35, 0xfdc75fce, 0x3be2dcf3, 0xff3a07eb, 0xdc43f0eb, 0x23ba73bb, +0x45f7649e, 0xcacc297c, 0xa3dd98db, 0x058d46a9, 0x6bcfc154, 0x7be8e1e6, +0x618a4754, 0x8d193c46, 0xba39e1ce, 0xc3b85cfc, 0xd80d853b, 0x38d6440d, +0x9c1a6185, 0x90c9dfcb, 0x01c6e841, 0xeedfe6ac, 0x7b61a3ae, 0xad3924dd, +0x514bc1a8, 0x6ed60d7e, 0x1b74b79b, 0xea295947, 0x5f9a5d33, 0xcd24311f, +0x0bd3d10c, 0xd214ecfe, 0xcb37035d, 0x8ce95168, 0x7020cb52, 0xe432107d, +0x042d63ac, 0x6201c8dd, 0x4bea65a2, 0x1a3cf453, 0x5b8e868d, 0x3d653bb0, +0x24c1967e, 0x37f183a9, 0x3700e703, 0xb168a02b, 0x2592cd82, 0x5c0cdb66, +0x70b64ebd, 0xf8dcb9a7, 0x634a8335, 0x06642398, 0xfe2f4497, 0x2e775256, +0x30f85cbe, 0xf4d2fdaf, 0x46ca6c9d, 0x022097d6, 0xcedeb03e, 0x3d8353bc, +0x0d005559, 0xd457eb38, 0xb1931313, 0xd4f12824, 0xcc444f1b, 0xfaa42c5f, +0x4fde636c, 0x10ea20a0, 0xe6da8721, 0xbfde5b08, 0x9c3739da, 0xb6dc015a, +0x427db088, 0xdfdb8e6f, 0x756be6c1, 0x21f5297b, 0x06135665, 0xc1602b7d, +0x049536c5, 0xbbb3b801, 0x0cdb0c19, 0x7b2ad622, 0xfee8218f, 0xc5c7f123, +0x8abd3301, 0xa15b534d, 0x29dd2053, 0xd409abf9, 0x3ef19d6b, 0x70a3cbc2, +0x7a51423a, 0x4505b2ad, 0xdc74c75e, 0x068751a9, 0xb0b56437, 0x14a10371, +0x76af806f, 0xa8a47e19, 0x7c97a26e, 0x7998a3d6, 0xdc1ad1e2, 0xb532a301, +0xca8a3e7d, 0xd0aef374, 0x204990c0, 0xc7011aec, 0xa69151ea, 0x53390026, +0x7bf0d762, 0x735c2202, 0x64159e54, 0x5a3b1a56, 0x9ef1def2, 0x0ab8a961, +0x587b0886, 0xb8cc5975, 0x2a5a0f23, 0x069d05be, 0x9cc3c207, 0x40ef1a02, +0x4fae3f5b, 0x1f127aae, 0xd4e6d411, 0x17ac43ef, 0xe4bf891b, 0xfbb21765, +0x2c560c7e, 0x8561988c, 0x73a01032, 0x0cfef73a, 0x694c4991, 0x885d7a3f, +0x4218d1ff, 0xc2efaffb, 0xaf9d9715, 0xf76de6b2, 0xcce8e8ff, 0x370e3800, +0x493675eb, 0xd8fbcbda, 0xa5b382c2, 0x86c8f1ea, 0x3d724ea4, 0xb067034c, +0x6491d87e, 0x1a745ce4, 0xbb27180b, 0x1a2f0acc, 0xac4b7b3b, 0xe324578b, +0xc87928df, 0x9c1de566, 0x0ce2a17d, 0xaf2e13ce, 0x146a8659, 0x8727f6ae, +0xe2df7d03, 0x1a8e4cb4, 0xfa590976, 0x13a7c236, 0xc07489d0, 0xbe905e17, +0xafeb3d4b, 0x201e73f2, 0x5bdca12e, 0x3e15a852, 0xbcfc3271, 0x5d398410, +0x6bfacc15, 0x011fc61f, 0x43e43fd7, 0x0640717c, 0x96bfb3ff, 0x158eac19, +0x3b852e91, 0x74f9ceda, 0xcac71326, 0xfc0e312a, 0x20e8137b, 0xa1162611, +0x239ac7fe, 0xb9d00f8a, 0xea0b5241, 0x019f0d25, 0xc5153264, 0xb48a5547, +0xe54e339f, 0x17a6cca5, 0x5065af0d, 0x5ce648b9, 0xb457b342, 0xc1cb3f0e, +0x28d03c8b, 0x5144ed7a, 0xdb80779f, 0x53ce1b87, 0xbc071147, 0xbfe46c11, +0x7296785e, 0x83e4a63e, 0xc58982e9, 0x9538c5b9, 0xf14abaaa, 0xd915124c, +0x73540cd6, 0xe333696b, 0x58f9e00a, 0xd4dad10f, 0xc0de1420, 0x355e2bdc, +0xb2faa8fd, 0xbe6a12f1, 0x45d415cc, 0x47f5aed9, 0x4754e770, 0x2bb07385, +0x41374352, 0xf80beb47, 0xef02f35c, 0xc9c1b86e, 0x94b5785b, 0xba33123f, +0x7e39f0c9, 0x028a9286, 0x7d52c9f1, 0x06f04da6, 0xbc6a68d1, 0xfc70bace, +0x95b6a129, 0xfff224bb, 0x701ef3fb, 0x3309286f, 0x544ae8c1, 0xdca62c4a, +0xf8862ee2, 0xf9e3cd29, 0x2c07cce2, 0x8d93652a, 0xf47e4611, 0x4635f586, +0x1c03e0f4, 0x819724c7, 0x96b2a3f0, 0xeeb1ad95, 0xff08e517, 0xbd4ba6ed, +0x49ddb12d, 0x365734b5, 0x5edf7459, 0x2ee117a9, 0x067b9462, 0xb703685f, +0x72499876, 0xb71dd772, 0x5d02b478, 0x4b6fed1d, 0x2df232c1, 0x733eaaff, +0x1618c029, 0x0adda94b, 0xd271e560, 0xd120ba30, 0x6dfecc73, 0x4e751af7, +0x0e7a6b63, 0xa1ad5dad, 0xb41201b1, 0x245f3e86, 0x3a5114ac, 0x5cd532c9, +0x27ff9c3a, 0x7a847392, 0x6b2d0514, 0x82d0cc45, 0x0c779ba2, 0x8690f3ef, +0x0c058474, 0x47a8ee9a, 0xac4c2475, 0x496b67a5, 0x033f43f5, 0x6e6ce3c9, +0x644f5163, 0x8f4b92d9, 0x217798ee, 0x4b8e5368, 0xce751989, 0x7ab05365, +0xe227cee3, 0xf35e2851, 0x6f304c87, 0x1389c81d, 0x6ebc3989, 0x2b32ac3e, +0x59c5d111, 0xa8e8699c, 0xa5b15150, 0x8cacebe8, 0x40c2ede6, 0x71ea78c9, +0xa9e40f49, 0x58309eab, 0xd2eb879c, 0x54b9086f, 0xce206d93, 0x47e7087a, +0xac1e1dd9, 0x1d3a7bf1, 0x07d21fd4, 0x3a84a2a7, 0xe3ce33de, 0x55c94ddf, +0xd827c1ea, 0x3b4dcf5a, 0x7d5fbeb6, 0xd71ccdae, 0x516a9035, 0x33b3bee0, +0x61201364, 0xcf344f8d, 0x8c887934, 0x1998c127, 0xe24f1190, 0x75e8ea20, +0x8a379eda, 0x8a894b14, 0xa3d7c264, 0xa62b0119, 0x87f4d316, 0xecdc5f2b, +0xbc2424ec, 0x71169a71, 0x61aa2d5d, 0xb5f5f160, 0xc15a4969, 0xc5419315, +0xbf84483e, 0x2b1687c1, 0xd1aa06df, 0x22d5befd, 0x8b09b15f, 0xa88ffc01, +0xbb33617f, 0xb01a3e2e, 0x912a939c, 0xc649d802, 0xeba14b11, 0x3c902b57, +0xbcf35a8d, 0x45a964a1, 0x0c9416ef, 0xec9ae2d1, 0xc5e56fee, 0x88bfa336, +0x653cfe85, 0x92d21037, 0x96bc60e4, 0xf5317c1e, 0x5e7118f4, 0x91a9e04b, +0x43f0cfec, 0xb763dc3c, 0xad6bac40, 0x21685f69, 0x5e4d6f70, 0xd8d5a4c4, +0x15ed5efe, 0x3bbefa38, 0xbf7cf912, 0xb1427ffd, 0x652235ff, 0x927a7046, +0x8806625f, 0x56967b59, 0x6ec3dba6, 0x0ee31e0f, 0x25111a51, 0x091c763e, +0x1867732b, 0x28365594, 0xa589c975, 0x585b906f, 0xb85c4f20, 0xced98475, +0x52f29d16, 0x1ca0de39, 0xede3c4c8, 0xda808e91, 0x449784ab, 0xe3caf590, +0xabf23939, 0xe97f259a, 0x0a879f81, 0x5c6268ac, 0x0a4f1b62, 0xfe15825e, +0xbea5e6f9, 0x0d2db98f, 0xeec8cd64, 0x1747ddf5, 0x8b4f161c, 0xd236c9f6, +0x1a5b5a2f, 0x08918e6d, 0xdde43ad9, 0x8c316d0f, 0x7f1b3342, 0x3e385cee, +0x55ff6f87, 0x89bbe534, 0x67c23afe, 0x250dc97a, 0x06b3b332, 0x61a03930, +0x2ca17159, 0x3260c4f6, 0x3c810bde, 0x28d94372, 0xa9701f5a, 0x3880475b, +0x352ff389, 0x2bf653e0, 0xa4a6cc35, 0x4d211cfc, 0xeda0b37d, 0x14c0e6d8, +0xe457aaea, 0x1c6ed15f, 0x155d3b90, 0x7f43aed4, 0x289a98f9, 0x42246910, +0x62d047f5, 0x75714462, 0xe2c2a4df, 0x9dfc0f13, 0x03361f4c, 0xc55b5ca7, +0x9dd89720, 0x17376073, 0xfb6ee849, 0x280e697e, 0x57b24f3a, 0x2089b956, +0xa8baa602, 0x5b7ab88c, 0xd69f8b56, 0x64f80d52, 0x223427b9, 0xfd393c95, +0xb074 + +e = +18444 + +k = +6144 + +ncb = +18528 + +rv_index = +0 + +code_block_mode = +1 + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e18448_crc24a.data b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e18448_crc24a.data new file mode 100644 index 00000000..345d1410 --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e18448_crc24a.data @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_ENC + +input0 = +0x11d2bcac, 0x7715ae4d, 0xc9f4dbc6, 0x2fda3c1a, 0x09349b16, 0x2cd3c189, +0xe5650429, 0xc73c42c1, 0xe7336bb7, 0xbb7de593, 0x83f986aa, 0xc0ade12e, +0x6730b095, 0x78f7c059, 0xaa907199, 0x75c323d1, 0x7061f9ba, 0x97e067bf, +0xc155cd55, 0x6a4cb08d, 0x4260e2fa, 0xff35e496, 0x37f251d3, 0x02fd9f9e, +0x6f53345c, 0x790cda6d, 0x3b8549e4, 0x56d0c6ea, 0x70a38006, 0xfff18223, +0x6f2035b5, 0x6cf508d3, 0x98a0a308, 0x432353fc, 0x0eb818eb, 0xdd9a128b, +0xf92431b2, 0xad788286, 0xda07d5de, 0x44b4b3a8, 0xbe62fd67, 0x61a4eb54, +0x90d7ac9f, 0xc4d9a930, 0xfddc3c24, 0xf5e3b1c8, 0x38143538, 0xcb1d3062, +0xcae36df3, 0x50a73437, 0x542dcab7, 0x875973a0, 0x34690309, 0x49cb1ddb, +0xf8a62b92, 0x82af6103, 0xc9f8c928, 0x3835b822, 0x16b044a6, 0xdae89096, +0xa5be47d4, 0xaf61189e, 0x5cd70faf, 0x037331cf, 0xef7fa5f6, 0xb2f2b41a, +0xa6f222c6, 0xdb60fe4c, 0x2a857a9a, 0x0b821f9d, 0x348afd17, 0x7eecbaeb, +0x92bb9509, 0x8a3cec24, 0xd02549a2, 0x155ffa81, 0x2b7feac6, 0x3ee461e7, +0xc981f936, 0x89b544c7, 0x9a431e36, 0x62511734, 0x769f9647, 0x211a747e, +0x567abef4, 0xad87e2b4, 0xa3e0c3bf, 0x6d325dd5, 0xf561cc46, 0x39925735, +0x3d8abbfd, 0xc3724c88, 0x8bdf03c9, 0x1b02a12a, 0x4f233a0c, 0x9ca9444a, +0xc5d1f7e6, 0x4d995f37, 0xd9aefb32, 0xd0465248, 0x0f3a3b21, 0x62ea8c0c, +0x91f8d54e, 0x5cf75514, 0x14618a01, 0x8fe9b87e, 0xf2b424f9, 0x49724ce2, +0xa1464587, 0x5e00dc83, 0x59475455, 0x444119b1, 0x4fb9f036, 0x65fcbc1a, +0x1c63a990, 0x767a6114, 0xb0ede06a, 0xcfb91ae3, 0x7874af5f, 0xf78772a3, +0xa2932c81, 0x77f2759d, 0x930dc8f1, 0x95ce14a5, 0x134363ee, 0x61ee143f, +0xf0034b35, 0xfdc75fce, 0x3be2dcf3, 0xff3a07eb, 0xdc43f0eb, 0x23ba73bb, +0x45f7649e, 0xcacc297c, 0xa3dd98db, 0x058d46a9, 0x6bcfc154, 0x7be8e1e6, +0x618a4754, 0x8d193c46, 0xba39e1ce, 0xc3b85cfc, 0xd80d853b, 0x38d6440d, +0x9c1a6185, 0x90c9dfcb, 0x01c6e841, 0xeedfe6ac, 0x7b61a3ae, 0xad3924dd, +0x514bc1a8, 0x6ed60d7e, 0x1b74b79b, 0xea295947, 0x5f9a5d33, 0xcd24311f, +0x0bd3d10c, 0xd214ecfe, 0xcb37035d, 0x8ce95168, 0x7020cb52, 0xe432107d, +0x042d63ac, 0x6201c8dd, 0x4bea65a2, 0x1a3cf453, 0x5b8e868d, 0x3d653bb0, +0x24c1967e, 0x37f183a9, 0x3700e703, 0xb168a02b, 0x2592cd82, 0x5c0cdb66, +0x70b64ebd, 0xf8dcb9a7, 0x634a8335, 0x06642398, 0xfe2f4497, 0x2e775256, +0x30f85cbe, 0xf4d2fdaf, 0x46ca6c9d, 0x022097d6, 0xcedeb03e, 0x3d8353bc, +0x0d005559, 0xd457eb38, 0xb1931313, 0xd4f12824, 0xcc444f1b, 0x5f + +output0 = +0x11d2bcac, 0x7715ae4d, 0xc9f4dbc6, 0x2fda3c1a, 0x09349b16, 0x2cd3c189, +0xe5650429, 0xc73c42c1, 0xe7336bb7, 0xbb7de593, 0x83f986aa, 0xc0ade12e, +0x6730b095, 0x78f7c059, 0xaa907199, 0x75c323d1, 0x7061f9ba, 0x97e067bf, +0xc155cd55, 0x6a4cb08d, 0x4260e2fa, 0xff35e496, 0x37f251d3, 0x02fd9f9e, +0x6f53345c, 0x790cda6d, 0x3b8549e4, 0x56d0c6ea, 0x70a38006, 0xfff18223, +0x6f2035b5, 0x6cf508d3, 0x98a0a308, 0x432353fc, 0x0eb818eb, 0xdd9a128b, +0xf92431b2, 0xad788286, 0xda07d5de, 0x44b4b3a8, 0xbe62fd67, 0x61a4eb54, +0x90d7ac9f, 0xc4d9a930, 0xfddc3c24, 0xf5e3b1c8, 0x38143538, 0xcb1d3062, +0xcae36df3, 0x50a73437, 0x542dcab7, 0x875973a0, 0x34690309, 0x49cb1ddb, +0xf8a62b92, 0x82af6103, 0xc9f8c928, 0x3835b822, 0x16b044a6, 0xdae89096, +0xa5be47d4, 0xaf61189e, 0x5cd70faf, 0x037331cf, 0xef7fa5f6, 0xb2f2b41a, +0xa6f222c6, 0xdb60fe4c, 0x2a857a9a, 0x0b821f9d, 0x348afd17, 0x7eecbaeb, +0x92bb9509, 0x8a3cec24, 0xd02549a2, 0x155ffa81, 0x2b7feac6, 0x3ee461e7, +0xc981f936, 0x89b544c7, 0x9a431e36, 0x62511734, 0x769f9647, 0x211a747e, +0x567abef4, 0xad87e2b4, 0xa3e0c3bf, 0x6d325dd5, 0xf561cc46, 0x39925735, +0x3d8abbfd, 0xc3724c88, 0x8bdf03c9, 0x1b02a12a, 0x4f233a0c, 0x9ca9444a, +0xc5d1f7e6, 0x4d995f37, 0xd9aefb32, 0xd0465248, 0x0f3a3b21, 0x62ea8c0c, +0x91f8d54e, 0x5cf75514, 0x14618a01, 0x8fe9b87e, 0xf2b424f9, 0x49724ce2, +0xa1464587, 0x5e00dc83, 0x59475455, 0x444119b1, 0x4fb9f036, 0x65fcbc1a, +0x1c63a990, 0x767a6114, 0xb0ede06a, 0xcfb91ae3, 0x7874af5f, 0xf78772a3, +0xa2932c81, 0x77f2759d, 0x930dc8f1, 0x95ce14a5, 0x134363ee, 0x61ee143f, +0xf0034b35, 0xfdc75fce, 0x3be2dcf3, 0xff3a07eb, 0xdc43f0eb, 0x23ba73bb, +0x45f7649e, 0xcacc297c, 0xa3dd98db, 0x058d46a9, 0x6bcfc154, 0x7be8e1e6, +0x618a4754, 0x8d193c46, 0xba39e1ce, 0xc3b85cfc, 0xd80d853b, 0x38d6440d, +0x9c1a6185, 0x90c9dfcb, 0x01c6e841, 0xeedfe6ac, 0x7b61a3ae, 0xad3924dd, +0x514bc1a8, 0x6ed60d7e, 0x1b74b79b, 0xea295947, 0x5f9a5d33, 0xcd24311f, +0x0bd3d10c, 0xd214ecfe, 0xcb37035d, 0x8ce95168, 0x7020cb52, 0xe432107d, +0x042d63ac, 0x6201c8dd, 0x4bea65a2, 0x1a3cf453, 0x5b8e868d, 0x3d653bb0, +0x24c1967e, 0x37f183a9, 0x3700e703, 0xb168a02b, 0x2592cd82, 0x5c0cdb66, +0x70b64ebd, 0xf8dcb9a7, 0x634a8335, 0x06642398, 0xfe2f4497, 0x2e775256, +0x30f85cbe, 0xf4d2fdaf, 0x46ca6c9d, 0x022097d6, 0xcedeb03e, 0x3d8353bc, +0x0d005559, 0xd457eb38, 0xb1931313, 0xd4f12824, 0xcc444f1b, 0xa533d85f, +0x4fde63ac, 0x10ea20a0, 0xe6da8721, 0xbfde5b08, 0x9c3739da, 0xb6dc015a, +0x427db088, 0xdfdb8e6f, 0x756be6c1, 0x21f5297b, 0x06135665, 0xc1602b7d, +0x049536c5, 0xbbb3b801, 0x0cdb0c19, 0x7b2ad622, 0xfee8218f, 0xc5c7f123, +0x8abd3301, 0xa15b534d, 0x29dd2053, 0xd409abf9, 0x3ef19d6b, 0x70a3cbc2, +0x7a51423a, 0x4505b2ad, 0xdc74c75e, 0x068751a9, 0xb0b56437, 0x14a10371, +0x76af806f, 0xa8a47e19, 0x7c97a26e, 0x7998a3d6, 0xdc1ad1e2, 0xb532a301, +0xca8a3e7d, 0xd0aef374, 0x204990c0, 0xc7011aec, 0xa69151ea, 0x53390026, +0x7bf0d762, 0x735c2202, 0x64159e54, 0x5a3b1a56, 0x9ef1def2, 0x0ab8a961, +0x587b0886, 0xb8cc5975, 0x2a5a0f23, 0x069d05be, 0x9cc3c207, 0x40ef1a02, +0x4fae3f5b, 0x1f127aae, 0xd4e6d411, 0x17ac43ef, 0xe4bf891b, 0xfbb21765, +0x2c560c7e, 0x8561988c, 0x73a01032, 0x0cfef73a, 0x694c4991, 0x885d7a3f, +0x4218d1ff, 0xc2efaffb, 0xaf9d9715, 0xf76de6b2, 0xcce8e8ff, 0x370e3800, +0x493675eb, 0xd8fbcbda, 0xa5b382c2, 0x86c8f1ea, 0x3d724ea4, 0xb067034c, +0x6491d87e, 0x1a745ce4, 0xbb27180b, 0x1a2f0acc, 0xac4b7b3b, 0xe324578b, +0xc87928df, 0x9c1de566, 0x0ce2a17d, 0xaf2e13ce, 0x146a8659, 0x8727f6ae, +0xe2df7d03, 0x1a8e4cb4, 0xfa590976, 0x13a7c236, 0xc07489d0, 0xbe905e17, +0xafeb3d4b, 0x201e73f2, 0x5bdca12e, 0x3e15a852, 0xbcfc3271, 0x5d398410, +0x6bfacc15, 0x011fc61f, 0x43e43fd7, 0x0640717c, 0x96bfb3ff, 0x158eac19, +0x3b852e91, 0x74f9ceda, 0xcac71326, 0xfc0e312a, 0x20e8137b, 0xa1162611, +0x239ac7fe, 0xb9d00f8a, 0xea0b5241, 0x019f0d25, 0xc5153264, 0xb48a5547, +0xe54e339f, 0x17a6cca5, 0x5065af0d, 0x5ce648b9, 0xb457b342, 0xc1cb3f0e, +0x28d03c8b, 0x5144ed7a, 0xdb80779f, 0x53ce1b87, 0xbc071147, 0xbfe46c11, +0x7296785e, 0x83e4a63e, 0xc58982e9, 0x9538c5b9, 0xf14abaaa, 0xd915124c, +0x73540cd6, 0xe333696b, 0x58f9e00a, 0xd4dad10f, 0xc0de1420, 0x355e2bdc, +0xb2faa8fd, 0xbe6a12f1, 0x45d415cc, 0x47f5aed9, 0x4754e770, 0x2bb07385, +0x41374352, 0xf80beb47, 0xef02f35c, 0xc9c1b86e, 0x94b5785b, 0xba33123f, +0x7e39f0c9, 0x028a9286, 0x7d52c9f1, 0x06f04da6, 0xbc6a68d1, 0xfc70bace, +0x95b6a129, 0xfff224bb, 0x701ef3fb, 0x3309286f, 0x544ae8c1, 0xdca62c4a, +0xf8862ee2, 0xf9e3cd29, 0x2c07cce2, 0x8d93652a, 0xf47e4611, 0x4635f586, +0x1c03e0f4, 0x819724c7, 0x96b2a3f0, 0xeeb1ad95, 0xff08e517, 0xbd4ba6ed, +0x49ddb12d, 0x365734b5, 0x5edf7459, 0x2ee117a9, 0x067b9462, 0xb703685f, +0x72499876, 0xb71dd772, 0x5d02b478, 0x4b6fed1d, 0x2df232c1, 0xb9dea0ff, +0x1618c046, 0x0adda94b, 0xd271e560, 0xd120ba30, 0x6dfecc73, 0x4e751af7, +0x0e7a6b63, 0xa1ad5dad, 0xb41201b1, 0x245f3e86, 0x3a5114ac, 0x5cd532c9, +0x27ff9c3a, 0x7a847392, 0x6b2d0514, 0x82d0cc45, 0x30779ba2, 0x3f0c8156, +0x9bce6106, 0x3511b2b4, 0x82dbef90, 0xac19def9, 0x5f11d43e, 0xa5899170, +0xdd137ff4, 0x188077ab, 0x53cec4c0, 0x6519988d, 0x2b07a0d5, 0x269ec4ae, +0x29c2bc5a, 0x4a0206c6, 0xf8fba9f5, 0x61309433, 0x402bf26c, 0xce401562, +0xf7eb46da, 0x4d9ad0c0, 0xf99fc69b, 0x1b670e56, 0x327bb1c8, 0x5f7db32c, +0x4c96b615, 0x041e0960, 0x190ef525, 0xede526f8, 0x59eb88e1, 0x355e5454, +0x8289d63c, 0xf848c2ad, 0x5bfc881f, 0xf161d01e, 0x5a921d49, 0xc202a8ad, +0xaa9e9dc4, 0x8211e14d, 0xea945bc4, 0xa5a59180, 0x7ffd5bd0, 0xd6c107bc, +0x3d0e84af, 0x04d13d34, 0xa21f54a3, 0xfcea787b, 0xbe61865b, 0xbe0d9899, +0x336bb04d, 0xf843ae66, 0x8d400981, 0x4359b845, 0xdbda44dd, 0x27392d92, +0x05780a7b, 0xe6dd7f03, 0x13137173, 0x9b623a85, 0x2428f035, 0x996f04de, +0x74613a87, 0x924aa956, 0x4661e3ad, 0x506ce2d3, 0xa59e7aba, 0x34fd455d, +0x70d613c6, 0x094610b9, 0x06e176ee, 0xb4f0842c, 0xc53680f4, 0xd9e2920b, +0xe0ddcd46, 0x8e4c1618, 0xb5c83878, 0x7b5107a3, 0xb75c33c0, 0xa62868d3, +0x804e47d9, 0xcefc87fc, 0xca5e125d, 0x3ed40ea7, 0xe72d3663, 0x06620539, +0x314993c2, 0x99f417d9, 0x4819151c, 0x7d46c8a2, 0x95a81dc9, 0x61898a53, +0x8226bb8c, 0x4907a616, 0x91eb32f7, 0x5430c6a1, 0x390ca234, 0x599f02ff, +0x315a4cc8, 0xc15d9e2b, 0x1c7a8788, 0x2074d5ea, 0xc063a30d, 0x5532e1f5, +0xd3820192, 0x916a7b03, 0x32422c07, 0x2ae2cc41, 0x96cb84c5, 0x2bab3d29, +0x0edc0add, 0x408e4981, 0x2606b671, 0x63dca006, 0x8f4261d9, 0xbfe4623b, +0x60174b80, 0x50230b0d, 0x9d4c7af3, 0x2edb3482, 0x24d8d087, 0x1b673b02, +0xe28b7132, 0xc6c8cb36, 0x5794e3f3, 0x808c3887, 0xf9f64a32, 0xfca10213, +0xff29e373, 0x54bf19a6, 0x16014860, 0x356d4398, 0xe8d0d630, 0x4c8100c0, +0x7b68a462, 0x6cc95c68, 0x3becad35, 0xb2c6b4c3, 0x740aef1c, 0x4f37f2d5, +0xc9d3c805, 0x6e4e533d, 0xf7647967, 0x91856de5, 0x87e7e428, 0xddf2fe07, +0x69016442, 0xe0132159, 0x1afae2a2, 0x63b6d719, 0x08d20a21, 0x48ee7113, +0x2fb2d853, 0xa532ffc8, 0x8296dee2, 0x0dfaf2fa, 0x060d531c, 0xa756d04c, +0x3efed03e, 0xbe9436db, 0x5b9e8a48, 0x0a37ea61, 0x718f4362, 0xebcc9742, +0x78aa2e7c, 0x4b19f7b8, 0x308b9af0, 0x915247e9, 0xc079aa48, 0x5230e578, +0x862d6de7, 0xbe0801d0, 0x8ab11c9d, 0xaf1d7feb, 0xe9d15530, 0x44651202, +0xc006 + +e = +18448 + +k = +6144 + +ncb = +18528 + +rv_index = +0 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_CRC_24A_ATTACH + +expected_status = +OK \ No newline at end of file diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e32256_crc24b_rm.data b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e32256_crc24b_rm.data new file mode 100644 index 00000000..9ce68b72 --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c1_k6144_r0_e32256_crc24b_rm.data @@ -0,0 +1,180 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_ENC + +input0 = +0xe5e0a392, 0x6f5468f1, 0xc8f1c505, 0x4502caf2, 0x23df5578, 0xc2f29afe, 0x06838d34, 0x264eab9b, +0x1e97c133, 0xabad27aa, 0x684fa9b4, 0x0fed735e, 0x6ab6bcad, 0x9c3e401a, 0x66a63035, 0xa72f7f6b, +0x509d2953, 0xf03c574a, 0x48f12185, 0x27318395, 0x008fc1ce, 0x7efcf19d, 0x0120de12, 0x6ed28bff, +0x2044547a, 0x04d95124, 0x6baf2e1e, 0x60d93509, 0x299aa27f, 0x8b67d93a, 0xc33d9631, 0xe468dbe1, +0x1a82d0b6, 0x8ee2ff7b, 0x39298170, 0x33a90b6a, 0x273d17f7, 0xb4368615, 0x60c390b8, 0x24696570, +0xa1f4abcc, 0x08b68dd0, 0x91f8c883, 0xc9ceed85, 0x6467da9b, 0x48dfae24, 0xf4ff4992, 0x8c8200cf, +0xaac02792, 0xf506f71d, 0x096df98a, 0x3074be38, 0x9b675626, 0x4ccace33, 0x50f0f60d, 0xf4b59cc4, +0x748a8799, 0x9032c151, 0x1fb945f6, 0x01bd098a, 0x90a7907b, 0xe086172c, 0x7ed3dc8d, 0xea0dd126, +0xf9b35bc1, 0x8ad4e233, 0x37b5e1a1, 0x95bf8c3a, 0x9d288b43, 0x23a860f1, 0x4e993fc3, 0x3c774bd4, +0xe411be48, 0x22ab80c7, 0x9dd09a47, 0xc9e6988e, 0xb4cbf0b2, 0xb9e2a389, 0xf1494438, 0x2c87638b, +0x892468ec, 0x7915d1aa, 0x4d9cb9c4, 0x643a388e, 0x48c4358f, 0x937ad5a1, 0xdae53508, 0xa1305607, +0xbfb63a5c, 0xb4bb9324, 0x71daab0b, 0xa582346b, 0x0e5402e1, 0x75af83e7, 0x2b241b17, 0xfa17aca1, +0x9637cb2e, 0xc61fe511, 0x093934f7, 0x4b703b44, 0x992ab229, 0x4ddc99b5, 0xe2be6b60, 0xa2217861, +0x8f621931, 0x272c0ec6, 0x23a4d5b2, 0xc6f4e002, 0xb76397c0, 0xc00e3c01, 0xb89d29b3, 0x5e9bc984, +0x00391d56, 0x55fae02f, 0x4bc4bc9d, 0x9a147b22, 0x0fb492ed, 0x753dc9a6, 0x9f493eec, 0x6e135dc2, +0x0eeca515, 0x1c1512f9, 0xb88075e3, 0x326a97f0, 0xc755faf5, 0x06b7961b, 0xbd50fd69, 0x5f071c92, +0x2806c099, 0xe364f4ab, 0x2c4a114a, 0x7655b100, 0xf5e66750, 0xf9212705, 0x245e3b8d, 0xb0f1fb0d, +0xd7be9558, 0x27030e24, 0x9ea9b62e, 0xbde9be45, 0xce0adae6, 0xc28780a4, 0x1664ccde, 0xa7a62e3f, +0xa4f8ec00, 0x964e813c, 0x108c95e0, 0x097fe10b, 0x08da3505, 0xde413158, 0x046a31c0, 0xd9cdd804, +0xdfbdff9a, 0xfd77493f, 0xdffb2388, 0xe2ffa111, 0x29eb6ac7, 0xf9e1ac68, 0xc2ed68c7, 0x34d65405, +0x25f187ba, 0x3df23a2c, 0x96a2a5a9, 0x2bef4b26, 0x28d1c41d, 0xf380c0f1, 0xc4d5456e, 0x6b699c28, +0x0aafd6b3, 0xf88f3078, 0x114438d0, 0x93836544, 0xebd5767d, 0x9185b5c2, 0x593e4fe7, 0x29d3348c, +0x881f16c4, 0x77e33d03, 0x6794bbf5, 0x6b6c38f3, 0xa860673a, 0xb1d2d152, 0x08f9f310, 0x82c37500, +0xce0ae34c, 0xd5478e86, 0xcfe1adfc, 0xfaf06d37, 0xf7e21bfc, 0x3b077287, 0x45b33c08, 0x41 + +output0 = +0x25832957, 0x1235fe50, 0xa3b0b09b, 0x0bb7c797, 0xce815b03, 0x7b702251, 0x2c25fc60, 0x23ac55e0, +0x8bea35cb, 0xe33b833c, 0xec1d38a4, 0xb59acb59, 0x0f1fa8b7, 0xcb77fce6, 0x6564fc78, 0x9e6c39cc, +0x94f1bed1, 0xe02684ad, 0x074045a4, 0x4dc9b344, 0xd187bed8, 0x2a129ecb, 0xf6c5bd4d, 0xf98d5c26, +0x78addaae, 0xbe87f7d2, 0xf42a7df6, 0xa4683306, 0xbfa9b509, 0x3a67ddfe, 0x013d68d5, 0xb2a1fb8a, +0x0dfc5da7, 0x792f6b24, 0xaa220ec2, 0x6cb0a7a4, 0xd6e5ea4a, 0x1cd6156c, 0x9bac3cad, 0x926a3950, +0x658815b0, 0x2a2e4349, 0x9684d70f, 0x67a62f63, 0x7a8aae48, 0x8aa046e5, 0x3cad66d5, 0xe7e269b8, +0x95a9a8ca, 0x426f46ae, 0xf780d265, 0x2e0d244d, 0x5480bc07, 0x66554568, 0x938aeda7, 0x1144006c, +0x450320b3, 0xce9dfce9, 0x59a277a8, 0xde38cc1a, 0x1cd2bd60, 0x0079ca20, 0x58dd4633, 0x55398bbd, +0x5fcf982f, 0xdbbd3e30, 0xdcaab5e4, 0xc368c59f, 0xfba8e7d2, 0x9902b581, 0x2ec0a87f, 0x0ebea4ee, +0x13bb2fd2, 0xbc1df5be, 0xe34baa5e, 0x4a66fc5d, 0x7fc9d926, 0x1d2dc178, 0xe780752f, 0x160e82fd, +0x2e6205ba, 0x3d2fd1d7, 0x22b03620, 0x1c1ce57b, 0x5a8f8e09, 0xc9522523, 0x712e5cc0, 0xa27b10ad, +0x52d2df01, 0x3fe886d6, 0x29f37f12, 0x479e3783, 0xa37a445c, 0x338f9df1, 0x74dca176, 0x11acbbc4, +0xac50920a, 0xe9375593, 0xc483a084, 0xed281b58, 0x21ea3cd3, 0x5cf2e76a, 0x50c5e5f4, 0x68d14253, +0x374cea8e, 0xf3533631, 0xb8f1bece, 0x3a188078, 0xb15325de, 0x989b4e70, 0x40f142f9, 0x4a55cc30, +0x72a442b2, 0xe583a6c7, 0x4cb90a7d, 0xe19187c0, 0x899ea4e0, 0x42598555, 0x7d012357, 0x3e9cd281, +0x75094cbb, 0x80afef50, 0x8a834f90, 0xa6182605, 0xdfb7c49e, 0xde57caf6, 0x62720457, 0x5a40705e, +0x2728f823, 0xde22d20c, 0x8389268a, 0x2aad9f81, 0xa376314c, 0xd9288769, 0xa7a1cc5d, 0x1dcdb0ed, +0x0b23e2d8, 0x024e9165, 0x0e375ea5, 0x27c561c7, 0x48687976, 0xe97021bc, 0xbf82aea2, 0x6cb87bd3, +0xaa3f7ed5, 0x1e5c3aaf, 0xccad60b5, 0xb672d5da, 0xde0ff047, 0xf7cd4216, 0x45220a3b, 0x060d16c7, +0xb6dc8617, 0xc10d9dc7, 0x3d229a69, 0x8deea79a, 0xda6883e1, 0x610f41f9, 0xfa0f57f0, 0x6f97b547, +0xf00577ee, 0xe1be3d28, 0x809810e0, 0x7d4b8416, 0xdcc82557, 0xce6a01b1, 0xbc4cb663, 0x8bc88eac, +0xab9e918d, 0x6ec29c41, 0xbd0e745e, 0xa97bc23e, 0x1e1370c2, 0xc2744139, 0xf5d2f67b, 0x9d5ae9dd, +0x1040a10a, 0x1b54944c, 0x9a6810b1, 0xc7079d11, 0x8c89d60d, 0xc9523c76, 0x304b7adc, 0x2fd915eb, +0x076192e8, 0x8cbd3bc1, 0x90440a9c, 0x81961a26, 0x6a53f026, 0xd2b07ce5, 0xc72e767a, 0x9d47c3d9, +0xab2b7cb3, 0xf4b3aeec, 0x18441c12, 0x56b3007f, 0x81c3cfff, 0x128b72ed, 0x214dee41, 0xc735baeb, +0x89cf54e7, 0xcdfce851, 0x77cc0a00, 0x57cc6e0a, 0x7ffc8dad, 0xb036445b, 0x0699a9f7, 0x8781d7b5, +0xdd0f9d01, 0x43f01de8, 0x6a2e7dee, 0xf802d6b3, 0x10094fc1, 0x954cf6ad, 0xd7aee91a, 0x1e3369c8, +0x2acd8c15, 0x3303a61d, 0x79344e4d, 0x1253cd91, 0x34c34c77, 0xbd075dce, 0xe0fdd1cc, 0xf9015c56, +0x931b9096, 0x604f4bbf, 0x3b1d991a, 0x24007891, 0x7c0c1709, 0xe0d43230, 0x11821166, 0xf21e2558, +0x2c37009d, 0x98718786, 0xa9a112e6, 0xd7e20b10, 0xfe917f72, 0xb76101ff, 0x42a8d327, 0x9039f71b, +0x6d1628cb, 0xfd01251c, 0x6f0572a2, 0xb36a7e7a, 0x2b1a4556, 0xe679b027, 0x313573d0, 0xb5f7c20c, +0xc8d1530a, 0x338e756e, 0xe6befc47, 0xfe12dccd, 0x6ad1e5b7, 0x40b078ff, 0x96aaacba, 0xedfdeb8f, +0x8da3673a, 0xc965882e, 0xa4dec6e9, 0x7bace08f, 0xf144852a, 0xd5d787d0, 0x204d32e6, 0x733098aa, +0x12f3a53a, 0xe060086d, 0x3ab1739a, 0xd391e860, 0xc23d9687, 0x0d3a84e0, 0x855ea4cf, 0x0d0ea5b7, +0xfd082430, 0x70df7cb4, 0xe712f026, 0x84ce32ac, 0x67b04602, 0xac0b1faa, 0xed119dcd, 0x184afb0b, +0x9abbefd2, 0x41b3e2f9, 0xe18d5703, 0x6e5f4de7, 0xb47380a2, 0x239a3714, 0x810bb522, 0x5ad95c77, +0xe4b37e0c, 0x1cc200a6, 0x404c4390, 0x90533af1, 0xb49f2403, 0x1dada65f, 0x96f86d1b, 0x58f679a9, +0x1fe3c66d, 0xcc5eb28d, 0x504b7aff, 0x7c18a397, 0x47ad4d5f, 0x348f12c5, 0x80bac6e0, 0x47bb3c74, +0xccf16f02, 0xa932a5af, 0x51ebb452, 0x25808fcc, 0x47d14a43, 0xf0c2a0af, 0x30f938b5, 0x428420c7, +0x61a66d9c, 0x082caa4f, 0xd9c74e3d, 0xb4d161b9, 0xb409f331, 0x14245a44, 0xddb67f1e, 0x1018c62c, +0xc1e58e6d, 0x4ba86bd6, 0xf4a282aa, 0x4a85f90d, 0xbf0efb7b, 0x03706f08, 0xbe06a5f2, 0xe37f45f1, +0xcbc4001c, 0x14decb05, 0xf3c6050f, 0xd1df33d7, 0xdab729c7, 0x8898dad3, 0x4a44a313, 0x30bee51e, +0xd67b07ef, 0xf233eb19, 0x4c52bca3, 0x5d824a4b, 0x275faf2b, 0x428f8a95, 0x03f70c8b, 0x609e4036, +0x59ce2147, 0xc7eecc6b, 0xa22d21c2, 0xc5764d9a, 0xb4524bc6, 0x6d9bdfc9, 0x0918b4f7, 0x62bb8022, +0x7a98bdcc, 0xc963131d, 0x949036b8, 0x649ae51d, 0x93038d33, 0xe26cb45e, 0x4813516a, 0xc06c759c, +0xfa6a9793, 0xdd59c686, 0x37b4b5b0, 0x66a02fac, 0xde159ecf, 0x90cdb66e, 0x1c9f7f94, 0x3a31f69e, +0xcd93bc45, 0x32744ee0, 0x0f6ae2b1, 0x3bc9e963, 0x8cddbae4, 0x8f9517ff, 0x7d304ec1, 0x344e0357, +0x7cd2ed30, 0x1f9fa3a4, 0x9dbeb67b, 0x59253b51, 0x41b6911a, 0x12b574af, 0x62a777da, 0x9a6c834a, +0xaa16081d, 0x3f105c37, 0x6eac5dc0, 0x76e56fd1, 0x67bffbfb, 0xce069b80, 0x8fe970b1, 0x61945ee0, +0xaf9d44c2, 0xf2cd16cc, 0x07a768c5, 0xd6eecaa6, 0x6e8ce063, 0x4b30b443, 0x360822c9, 0xa6442b33, +0xf02d09a0, 0x22056421, 0xc552aa94, 0x1346ea1a, 0xab6c54a1, 0x35a69bdf, 0xfc6095ea, 0xbde0d55a, +0x1b433ec0, 0x137483a0, 0x8c1cb29d, 0xf2bc4e17, 0x45df6882, 0xbd702bc9, 0xabbead61, 0xc97ef61d, +0x01b68438, 0xbdd10231, 0x2e046323, 0x80e619bd, 0xa5126a2a, 0x74106e0d, 0x9e57b499, 0xcb275127, +0x9e01f21e, 0xbaf4a744, 0x38604f7e, 0xa85493d6, 0x9a4b4541, 0x3f397269, 0xdde71734, 0x157dfe98, +0x40334ad9, 0xc889f52f, 0xbddf4d23, 0xd2200018, 0x2a4b7180, 0x8a1a357f, 0xcd6af7e4, 0x1426b52b, +0x2a45481f, 0xfc38394c, 0x05123e4e, 0xa0215834, 0xe2c2306e, 0x4ee9fe1a, 0x6bbbf30c, 0x19a39072, +0x846b0bdf, 0xde4ef7f9, 0x47f1d67c, 0xf507e9e8, 0x14c1f617, 0xaa756ccf, 0xececc02a, 0x183cd616, +0x9d4bac81, 0xd959d1ea, 0xefb81e2b, 0xd8ca27fc, 0x5c7e9eb4, 0x64d5d550, 0x32ed3c41, 0x8f62e8a8, +0x5e2b65c5, 0x0a7e5e1b, 0x88abb657, 0x781ee74a, 0x840c27fd, 0xec83df4c, 0x163acaf0, 0xb2649960, +0x2a706f9a, 0x5226c811, 0x5bcb501c, 0xb87caa99, 0xecb4a37e, 0x8927c424, 0xfea9ac62, 0x17da41d1, +0x571c5987, 0x4217ea55, 0x5043b664, 0x18fb3ece, 0xfb2d13b9, 0xb6599d4b, 0xbb8a536d, 0xa719683f, +0x81cc2d0f, 0xc888f1d3, 0xccb0a428, 0xd2cdff48, 0x0b3440e6, 0x45700619, 0xcea80987, 0xa5ad1cc3, +0xd29deaf7, 0x4a8c95af, 0x9ac63a98, 0xf46d26b0, 0xf9a09352, 0x5f8b3fb7, 0x367c8da1, 0x3244b6a2, +0x9634b973, 0x27292ae5, 0xc4f7ae33, 0x9b73a4f7, 0x086a8bb4, 0x616cd17b, 0x74fdb210, 0x5f483d60, +0xf7666630, 0x107cd026, 0xfa2a4a07, 0x171b319e, 0x509d50c8, 0xf2d15061, 0x981d5759, 0xf23bc5c6, +0xe9befe22, 0x1e784009, 0x7cda3751, 0x1c5b2b69, 0x3ba09115, 0x756f1ef5, 0x2b47353b, 0x553a33d9, +0x1f029262, 0x6393d0ee, 0x97e01180, 0x0c53980c, 0x5da79e34, 0x1e0b53c4, 0x06a9f2ba, 0x622181b0, +0x4898ac50, 0x874c780e, 0xf8cc5072, 0x4a5a63ca, 0xf6dea6a2, 0xc3134004, 0x4505bae6, 0xc3f4b638, +0x0dda8e07, 0x0c4135bf, 0x61e80f25, 0x4b31a76c, 0xd91f0d09, 0x4ba0d577, 0xc95d0086, 0x21a61725, +0x16862c1f, 0x9588795d, 0x6ecde7c4, 0x353288cd, 0x94e4647e, 0xeceaba51, 0x7c602ff8, 0xbba48cef, +0x6556c6bf, 0xde912190, 0x86602d55, 0xf549554a, 0xd4d52081, 0x9728c5db, 0x7d09e129, 0x068008ab, +0x353a6ea4, 0x90b44e4e, 0x5c97a08a, 0x6293b3c7, 0xa3e4114b, 0x7ce44646, 0x8f2b0269, 0x291ca1bb, +0x98721528, 0xe30f5532, 0x0bbb2951, 0x7b7c390a, 0xb835b070, 0x2712e51c, 0xc20fb607, 0x5a05ce52, +0x5eb33cc2, 0x33c8b3a8, 0x81433abe, 0xb99cc5de, 0x817a5bab, 0xc76ffef0, 0xc68fb77c, 0x96c35c46, +0xef1bedc9, 0x42d84a19, 0x54440a6e, 0x3c4b7400, 0xe88bdd94, 0xe1b91c7d, 0xdcdba422, 0xc865625f, +0xaaed9adf, 0x782f8dd7, 0xd267ef7b, 0x366340af, 0x5a9b408a, 0xd6edff9b, 0x8356ad73, 0xbaaf18d0, +0xdf752a1b, 0xb246d2c0, 0xe2209cf7, 0x7b4aaa2a, 0xaeaec406, 0x5dc1665d, 0xcad3ca61, 0x9603b5c9, +0x58012ba9, 0x32945486, 0x78fda0e2, 0xfa326649, 0xe88a7466, 0x6a54aea7, 0x6a56ad08, 0x9e86cbd3, +0x8aaa7c2e, 0x66e45a99, 0x285d26f4, 0x40d2740f, 0xc87be0d2, 0x55844605, 0xd87e6a56, 0x04c036a9, +0x00321b41, 0xc99f5e34, 0x7a87eadc, 0xc3ac9125, 0xdd0be68d, 0xa70cc221, 0x6d340390, 0xb3d88bd5, +0x8cf95295, 0xeb03f3f5, 0x5a4bbedd, 0x56fcc9ad, 0x7a2e3d8c, 0x501bb88f, 0x8cfa9729, 0x4beaee02, +0xfb22ede0, 0x51ef3bb1, 0xa4eac5db, 0xc6df35be, 0x9c6da264, 0x128cf797, 0x58f7d2d1, 0x20d87f0e, +0x56a06be1, 0x127ded22, 0x6b03d2f3, 0x51be2702, 0xe898c0c1, 0x5532a2f5, 0xc2059c2c, 0x07d11ae7, +0xfd1d20ba, 0x6e682d25, 0xff27f183, 0x79339832, 0x47c475e4, 0xd8193faa, 0x1d6a37f3, 0xba4b4cc7, +0x25a910c1, 0x5335c90a, 0x084a987e, 0xb281453c, 0xce33dd8e, 0x7fae16a2, 0x5c4ecf25, 0x2d340555, +0xa4ee8816, 0x651373c3, 0xefeb3c3f, 0x0188871b, 0x55e2ad83, 0xe904173b, 0x2f948fb9, 0xc50c0314, +0x2a24ab54, 0x687a2c47, 0xabd0573e, 0x7908cc94, 0x490a1e1e, 0x555895e8, 0x30722594, 0x291dd817, +0xc0b4ebc3, 0xfa0e5597, 0xf80409f8, 0x6152a038, 0x4bec698a, 0xa56cff7d, 0x4770e57d, 0x04e72526, +0x823fa205, 0x22cd7082, 0x68a2e82d, 0xfa193898, 0x17c3a4d2, 0x7298366a, 0xcadc958d, 0x0cdb7e1a, +0x228eddd1, 0x1459b630, 0xe3552ae0, 0x1c76ec70, 0x96677752, 0x17c28b84, 0xe82a9a0e, 0xbb37fd2b, +0xe357cd86, 0xa5f3aafa, 0x0a56ebc1, 0x57adcddc, 0x007f642b, 0x2c64e1fd, 0xa2b073df, 0x60715c24, +0x6d7861d0, 0xd0796ccb, 0xa29916dc, 0x7eaad923, 0x3618dee8, 0x1094af8d, 0x70051ff6, 0x597ba4ff, +0x70e7fe76, 0xdb83025f, 0x09011eee, 0x44680188, 0x5c72d5b7, 0x1610cb8d, 0x643be6ac, 0xecc8cacb, +0x19d9b888, 0xcc19b4ea, 0x40e7e526, 0x27ecd3eb, 0x01279cba, 0x1794e331, 0x6dbf274c, 0x95de5d2f, +0x14aad0a9, 0x45c90401, 0x0611bb41, 0xd019a189, 0x68dd707c, 0xc563c798, 0xa4c79d2c, 0x5db10eb3, +0x2689fe92, 0xbb137c10, 0xa4c0c9d8, 0xa9610249, 0x056f1268, 0xcb57ae36, 0x62a7270d, 0x349c7dec, +0xc237db79, 0xebcabeba, 0xc421413f, 0x0bf08741, 0xfcfc6f35, 0x28d71e38, 0xe41e24b1, 0xa3bb1ed2, +0x4c757e5c, 0x8f1e95f8, 0xac00d0cc, 0xeca670c7, 0xdfd87ac5, 0x43b4f5c7, 0x997a0f6b, 0x785d6b90, +0xd0197018, 0xdf81defd, 0xd2e73e04, 0x603dabe6, 0xf0148c2f, 0x64df0a91, 0x9aae51c9, 0x93867ced, +0xcc58e131, 0x60daa1d2, 0xe3d43433, 0xd51c9947, 0xcc742731, 0xd0e54c33, 0x1fcddc7b, 0xc06505de, +0x0169991f, 0xb4f43bb9, 0x91a901f6, 0x8017b9d3, 0x70914002, 0x2d03c3c7, 0x1861064e, 0x51821521, +0x03d029ef, 0x7768c872, 0x2a618e19, 0xbe00911a, 0xf927772d, 0x16f0ef1f, 0x3a7d721b, 0x73bf2184, +0x81b20c99, 0x50c2d166, 0x2027da1f, 0xe6a7f756, 0x516435ab, 0x077bb2a2, 0x33076d9e, 0x2fcc1053, +0x3da5507b, 0x58e7861c, 0xcb7f34e3, 0xc1dd6cee, 0x5d7eeb2f, 0x8bf7af16, 0xcaaa0b04, 0xbffe68a9, +0x7aa6d3de, 0x86e8d238, 0x6d9c9e5c, 0x0afe48ea, 0x54a8b2c7, 0x7d081d4f, 0x24635e7d, 0x83a90ad2, +0x5faa3307, 0x86d02631, 0x3ba7090e, 0x890ea613, 0x6379381d, 0x43082edc, 0x45fadca0, 0x507a5be8, +0x4002d3e0, 0xcd47db8f, 0x016f02f7, 0x2cc37a2e, 0x6b2440e8, 0xf0a17a06, 0xd1d9ccba, 0xb4bfd01e, +0xfb2e8da1, 0x2b9eafb9, 0x78351034, 0xd5741ede, 0x0728eaf6, 0x7943413b, 0x502b32a2, 0xcd7517b8, +0xebc7a095, 0x0c604a3e, 0x3404c921, 0xa5130fc4, 0x49320039, 0x6afa45fb, 0xdfb6d1d1, 0x9f976a89, +0x6edc8665, 0x25dbf831, 0xa4f7cfec, 0x317a09b5, 0xdaf4c587, 0x28517cd4, 0x6b0c4ef3, 0xcb4307a8, +0xff2670b4, 0x53faca1c, 0x4e2b952a, 0xf8c81cb5, 0xad345402, 0x0cfa7a14, 0x8f530b2f, 0x08720c93, +0xdac62944, 0xa2fa1466, 0xecd483c0, 0x1d969b7d, 0x301f431b, 0xa245449b, 0xfbe74141, 0x61ccd26d, +0xeed80681, 0xba661d5c, 0x2aa8ba84, 0x98df402f, 0xb0bfa754, 0xf786f0eb, 0x502a3f00, 0x5714ef6b, +0x0cc031fe, 0xbd5cb04c, 0x5cf040e1, 0x3d733d6f, 0x9b721cfd, 0xa93dad7d, 0x343a8188, 0x5beea144, +0x77f00ee3, 0xb39e61bd, 0xc53b2a3f, 0xa8b4c424, 0xf5bad225, 0xa85879f2, 0xcfb028f4, 0x09643370, +0x1c7204e6, 0xcebc96e5, 0x12227cec, 0xd7a429da, 0xb5645c6c, 0xf99d4c2b, 0x417bdfb6, 0x0b289280, +0xd9cb2cb6, 0x36d1a187, 0x69839b3c, 0x59de4109, 0xd03843a6, 0x46eb3539, 0x11a526ce, 0x56c78934, +0x763909cc, 0x656ca8af, 0x5b0bdb9d, 0xfac27a43, 0xe1f96c06, 0x6cebe65d, 0xf94709d9, 0x63efc9f1, +0xc95ba413, 0xe704de3c, 0x261e2b43, 0x9c3ef6a0, 0xad4bbe93, 0x79f1cfd8, 0xe314fc58, 0x3470d507, +0xdd0e43e3, 0x394aca27, 0x6bbbf7f1, 0xb213d5e9, 0x1ba99155, 0x4bf71a64, 0x7aa72d51, 0x36a82476, +0x81d0a1c9, 0xc175a36a, 0xda05fc03, 0xfe16edc6, 0xbbbf6f57, 0xb00978f6, 0x0e17eb6c, 0xe905fe98, +0x49241c46, 0x6cc1fcda, 0x8a562cdf, 0xae6c7a70, 0x083e66ed, 0x433be4c6, 0x2092bc04, 0xb4326383, +0x92006a4a, 0x401602df, 0xa54a2952, 0xa4ae512c, 0x46153a61, 0xbaf9bdca, 0x56a95e63, 0x5eadc50f, +0xe403dc0b, 0x3708ba31, 0x21db3941, 0xeb74c1c8, 0x8d2628cf, 0xb7925cf4, 0xdb1ad60b, 0x67dfb1ea, +0x4b8893ec, 0x2d101360, 0x3036d21b, 0x9ed1eb42, 0xa1a60268, 0xe1d6502a, 0x459b4907, 0x1275e279, +0x20efb17c, 0x7f4ae419, 0xf6e4a74b, 0x35698d03, 0x5414844a, 0x2397a6b9, 0x7e41f393, 0xe78fd97d + +e = +32256 + +k = +6144 + +ncb = +18528 + +rv_index = +0 + +code_block_mode = +1 + +op_flags = +RTE_BBDEV_TURBO_RATE_MATCH, RTE_BBDEV_TURBO_CRC_24B_ATTACH + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_enc_c2_k5952_r0_e17868_crc24b.data b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c2_k5952_r0_e17868_crc24b.data new file mode 100644 index 00000000..13006501 --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c2_k5952_r0_e17868_crc24b.data @@ -0,0 +1,300 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_ENC + +input0 = +0x11d2bcac, 0x7715ae4d, 0xc9f4dbc6, 0x2fda3c1a, 0x09349b16, 0x2cd3c189, +0xe5650429, 0xc73c42c1, 0xe7336bb7, 0xbb7de593, 0x83f986aa, 0xc0ade12e, +0x6730b095, 0x78f7c059, 0xaa907199, 0x75c323d1, 0x7061f9ba, 0x97e067bf, +0xc155cd55, 0x6a4cb08d, 0x4260e2fa, 0xff35e496, 0x37f251d3, 0x02fd9f9e, +0x6f53345c, 0x790cda6d, 0x3b8549e4, 0x56d0c6ea, 0x70a38006, 0xfff18223, +0x6f2035b5, 0x6cf508d3, 0x98a0a308, 0x432353fc, 0x0eb818eb, 0xdd9a128b, +0xf92431b2, 0xad788286, 0xda07d5de, 0x44b4b3a8, 0xbe62fd67, 0x61a4eb54, +0x90d7ac9f, 0xc4d9a930, 0xfddc3c24, 0xf5e3b1c8, 0x38143538, 0xcb1d3062, +0xcae36df3, 0x50a73437, 0x542dcab7, 0x875973a0, 0x34690309, 0x49cb1ddb, +0xf8a62b92, 0x82af6103, 0xc9f8c928, 0x3835b822, 0x16b044a6, 0xdae89096, +0xa5be47d4, 0xaf61189e, 0x5cd70faf, 0x037331cf, 0xef7fa5f6, 0xb2f2b41a, +0xa6f222c6, 0xdb60fe4c, 0x2a857a9a, 0x0b821f9d, 0x348afd17, 0x7eecbaeb, +0x92bb9509, 0x8a3cec24, 0xd02549a2, 0x155ffa81, 0x2b7feac6, 0x3ee461e7, +0xc981f936, 0x89b544c7, 0x9a431e36, 0x62511734, 0x769f9647, 0x211a747e, +0x567abef4, 0xad87e2b4, 0xa3e0c3bf, 0x6d325dd5, 0xf561cc46, 0x39925735, +0x3d8abbfd, 0xc3724c88, 0x8bdf03c9, 0x1b02a12a, 0x4f233a0c, 0x9ca9444a, +0xc5d1f7e6, 0x4d995f37, 0xd9aefb32, 0xd0465248, 0x0f3a3b21, 0x62ea8c0c, +0x91f8d54e, 0x5cf75514, 0x14618a01, 0x8fe9b87e, 0xf2b424f9, 0x49724ce2, +0xa1464587, 0x5e00dc83, 0x59475455, 0x444119b1, 0x4fb9f036, 0x65fcbc1a, +0x1c63a990, 0x767a6114, 0xb0ede06a, 0xcfb91ae3, 0x7874af5f, 0xf78772a3, +0xa2932c81, 0x77f2759d, 0x930dc8f1, 0x95ce14a5, 0x134363ee, 0x61ee143f, +0xf0034b35, 0xfdc75fce, 0x3be2dcf3, 0xff3a07eb, 0xdc43f0eb, 0x23ba73bb, +0x45f7649e, 0xcacc297c, 0xa3dd98db, 0x058d46a9, 0x6bcfc154, 0x7be8e1e6, +0x618a4754, 0x8d193c46, 0xba39e1ce, 0xc3b85cfc, 0xd80d853b, 0x38d6440d, +0x9c1a6185, 0x90c9dfcb, 0x01c6e841, 0xeedfe6ac, 0x7b61a3ae, 0xad3924dd, +0x514bc1a8, 0x6ed60d7e, 0x1b74b79b, 0xea295947, 0x5f9a5d33, 0xcd24311f, +0x0bd3d10c, 0xd214ecfe, 0xcb37035d, 0x8ce95168, 0x7020cb52, 0xe432107d, +0x042d63ac, 0x6201c8dd, 0x4bea65a2, 0x1a3cf453, 0x5b8e868d, 0x3d653bb0, +0x24c1967e, 0x37f183a9, 0x3700e703, 0xb168a02b, 0x2592cd82, 0x5c0cdb66, +0x70b64ebd, 0xf8dcb9a7, 0x634a8335, 0x06642398, 0xfe2f4497, 0x2e775256, +0x30f85cbe, 0xf4d2fdaf, 0x46ca6c9d, 0x022097d6, 0xcedeb03e, 0xaf9437bc, +0x341f8045, 0x93f6ff86, 0x2e019ec9, 0xc80116e8, 0xf984ff34, 0xe4f1be20, +0xfe455510, 0xc561cdba, 0x2d6f560f, 0xb239fd9d, 0xd254e343, 0x8090448a, +0xa724c4f1, 0xa00fe4f0, 0xf0379ac6, 0x0c61b383, 0x44b2228e, 0x9d223bc0, +0xe7cfbc38, 0x62ae6a40, 0x6f8d445c, 0xc6045eaf, 0x8507df18, 0xe64c057e, +0x4e589113, 0x3ba3bf55, 0x6a702b19, 0xea23af7d, 0xa1b98022, 0x7096c45c, +0xace3b8c1, 0x47ed7bb4, 0x198a2b8f, 0x586b57db, 0xbf9d24d4, 0x96054c66, +0x13f26306, 0x2a43524b, 0x1ecd419b, 0xb3a24852, 0x4ed7765f, 0x4c99fbe3, +0x6c3b9ebb, 0x1ef7bfeb, 0x0daf1256, 0xe9c90c7f, 0xbe067ed7, 0x469102ba, +0xedccc098, 0x96691fb1, 0x2a9b850f, 0x58aba5f2, 0x0bcad31d, 0x90b1f25b, +0x78a55fe4, 0xdd41e626, 0xdcba3e83, 0x2209d8a9, 0x6ee2f76f, 0x55643570, +0xd181a0ef, 0xd1b2790a, 0xc7793587, 0x3aebe10c, 0xbbde5fe0, 0x5c308948, +0x01439ece, 0x5319fbef, 0x10b0de1b, 0x6595f4f3, 0xa3e6a7c9, 0xf97b2e10, +0x4513cf3d, 0x8ded3394, 0x0949a772, 0x934f2269, 0x2b636628, 0x92101c68, +0x804dfd94, 0x178315be, 0x384adf65, 0xe64d11d3, 0xc44881c0, 0x7a8e7886, +0xb721c0c8, 0x6d0d0fc0, 0x97f0cc5a, 0x3c67c31f, 0x4906dd09, 0x6202c01a, +0xad8d6c5b, 0x53158056, 0x13704ed5, 0xa1dfeaaa, 0xb35e4b40, 0xd9c3e6fa, +0x0c9650ce, 0xbdfe5c22, 0x874bedb3, 0xbd07d0bd, 0x4eef4a4f, 0x2970d932, +0xfdd003ac, 0x609fbea4, 0xad817794, 0xb4ee9f96, 0x559b3faa, 0x1edcf35f, +0x8342e9fb, 0x69a73981, 0x5fc86a07, 0x929b45aa, 0x5c32e847, 0x43421484, +0x9962abf5, 0x8cf07ae4, 0xba7b556d, 0x2da42ac1, 0xac0eef6f, 0xdc118ef8, +0x6061597a, 0xc5f7a007, 0xb9b92c51, 0x87049008, 0x72c9da8a, 0x54dc48c6, +0xb4c4b392, 0x50204c8d, 0x00b2ef59, 0xcdac687c, 0xb2158dc3, 0x4f1c65e3, +0x8af22286, 0x4444ee49, 0x2d8921ce, 0x3c0a8d11, 0xb97325da, 0x056aec7b, +0xa96ca31b, 0x71a5febb, 0xa3166c47, 0x5e920dcd, 0xa58b2e61, 0x4d3b392f, +0x1b9b177b, 0x058b0dfa, 0x4bdc16ab, 0x8a4eeac3, 0xfbf2d2fc, 0x636f04a6, +0x75a14bda, 0xa28f1947, 0xf2f248e4, 0x934e7dd6, 0xeb4401cb, 0x37f693a2, +0xd1cb7e16, 0xbd5d9abb, 0x6f613f63, 0x9bfb5fd5, 0x6b70c5ea, 0xbdcc6c7a, +0x41efaee1, 0xf45d3965, 0x0499b12b, 0x7820873a, 0xed53961f, 0x303c4f04, +0x2e4bd363, 0x1bcdd589, 0xf7317de0, 0x7ba3d53a, 0x61fb36a8, 0x23e55b43, +0x0ef838e9, 0x7b9d91a3, 0x3cd7d835, 0x5f28e517, 0xa100fad7, 0xff39800d, +0xd7894433, 0xb61a62cd, 0xde618b70, 0x8560d770, 0xed02 + +output0 = +0x11d2bcac, 0x7715ae4d, 0xc9f4dbc6, 0x2fda3c1a, 0x09349b16, 0x2cd3c189, +0xe5650429, 0xc73c42c1, 0xe7336bb7, 0xbb7de593, 0x83f986aa, 0xc0ade12e, +0x6730b095, 0x78f7c059, 0xaa907199, 0x75c323d1, 0x7061f9ba, 0x97e067bf, +0xc155cd55, 0x6a4cb08d, 0x4260e2fa, 0xff35e496, 0x37f251d3, 0x02fd9f9e, +0x6f53345c, 0x790cda6d, 0x3b8549e4, 0x56d0c6ea, 0x70a38006, 0xfff18223, +0x6f2035b5, 0x6cf508d3, 0x98a0a308, 0x432353fc, 0x0eb818eb, 0xdd9a128b, +0xf92431b2, 0xad788286, 0xda07d5de, 0x44b4b3a8, 0xbe62fd67, 0x61a4eb54, +0x90d7ac9f, 0xc4d9a930, 0xfddc3c24, 0xf5e3b1c8, 0x38143538, 0xcb1d3062, +0xcae36df3, 0x50a73437, 0x542dcab7, 0x875973a0, 0x34690309, 0x49cb1ddb, +0xf8a62b92, 0x82af6103, 0xc9f8c928, 0x3835b822, 0x16b044a6, 0xdae89096, +0xa5be47d4, 0xaf61189e, 0x5cd70faf, 0x037331cf, 0xef7fa5f6, 0xb2f2b41a, +0xa6f222c6, 0xdb60fe4c, 0x2a857a9a, 0x0b821f9d, 0x348afd17, 0x7eecbaeb, +0x92bb9509, 0x8a3cec24, 0xd02549a2, 0x155ffa81, 0x2b7feac6, 0x3ee461e7, +0xc981f936, 0x89b544c7, 0x9a431e36, 0x62511734, 0x769f9647, 0x211a747e, +0x567abef4, 0xad87e2b4, 0xa3e0c3bf, 0x6d325dd5, 0xf561cc46, 0x39925735, +0x3d8abbfd, 0xc3724c88, 0x8bdf03c9, 0x1b02a12a, 0x4f233a0c, 0x9ca9444a, +0xc5d1f7e6, 0x4d995f37, 0xd9aefb32, 0xd0465248, 0x0f3a3b21, 0x62ea8c0c, +0x91f8d54e, 0x5cf75514, 0x14618a01, 0x8fe9b87e, 0xf2b424f9, 0x49724ce2, +0xa1464587, 0x5e00dc83, 0x59475455, 0x444119b1, 0x4fb9f036, 0x65fcbc1a, +0x1c63a990, 0x767a6114, 0xb0ede06a, 0xcfb91ae3, 0x7874af5f, 0xf78772a3, +0xa2932c81, 0x77f2759d, 0x930dc8f1, 0x95ce14a5, 0x134363ee, 0x61ee143f, +0xf0034b35, 0xfdc75fce, 0x3be2dcf3, 0xff3a07eb, 0xdc43f0eb, 0x23ba73bb, +0x45f7649e, 0xcacc297c, 0xa3dd98db, 0x058d46a9, 0x6bcfc154, 0x7be8e1e6, +0x618a4754, 0x8d193c46, 0xba39e1ce, 0xc3b85cfc, 0xd80d853b, 0x38d6440d, +0x9c1a6185, 0x90c9dfcb, 0x01c6e841, 0xeedfe6ac, 0x7b61a3ae, 0xad3924dd, +0x514bc1a8, 0x6ed60d7e, 0x1b74b79b, 0xea295947, 0x5f9a5d33, 0xcd24311f, +0x0bd3d10c, 0xd214ecfe, 0xcb37035d, 0x8ce95168, 0x7020cb52, 0xe432107d, +0x042d63ac, 0x6201c8dd, 0x4bea65a2, 0x1a3cf453, 0x5b8e868d, 0x3d653bb0, +0x24c1967e, 0x37f183a9, 0x3700e703, 0xb168a02b, 0x2592cd82, 0x5c0cdb66, +0x70b64ebd, 0xf8dcb9a7, 0x634a8335, 0x06642398, 0xfe2f4497, 0x2e775256, +0x30f85cbe, 0xf4d2fdaf, 0x46ca6c9d, 0x022097d6, 0xcedeb03e, 0xb4ca20bc, +0x4fde636c, 0x10ea20a0, 0xe6da8721, 0xbfde5b08, 0x9c3739da, 0xb6dc015a, +0x427db088, 0xdfdb8e6f, 0x756be6c1, 0x21f5297b, 0x06135665, 0xc1602b7d, +0x049536c5, 0xbbb3b801, 0x0cdb0c19, 0x7b2ad622, 0xfee8218f, 0xc5c7f123, +0x8abd3301, 0xa15b534d, 0x29dd2053, 0xd409abf9, 0x3ef19d6b, 0x70a3cbc2, +0x7a51423a, 0x4505b2ad, 0xdc74c75e, 0x068751a9, 0xb0b56437, 0x14a10371, +0x76af806f, 0xa8a47e19, 0x7c97a26e, 0x7998a3d6, 0xdc1ad1e2, 0xb532a301, +0xca8a3e7d, 0xd0aef374, 0x204990c0, 0xc7011aec, 0xa69151ea, 0x53390026, +0x7bf0d762, 0x735c2202, 0x64159e54, 0x5a3b1a56, 0x9ef1def2, 0x0ab8a961, +0x587b0886, 0xb8cc5975, 0x2a5a0f23, 0x069d05be, 0x9cc3c207, 0x40ef1a02, +0x4fae3f5b, 0x1f127aae, 0xd4e6d411, 0x17ac43ef, 0xe4bf891b, 0xfbb21765, +0x2c560c7e, 0x8561988c, 0x73a01032, 0x0cfef73a, 0x694c4991, 0x885d7a3f, +0x4218d1ff, 0xc2efaffb, 0xaf9d9715, 0xf76de6b2, 0xcce8e8ff, 0x370e3800, +0x493675eb, 0xd8fbcbda, 0xa5b382c2, 0x86c8f1ea, 0x3d724ea4, 0xb067034c, +0x6491d87e, 0x1a745ce4, 0xbb27180b, 0x1a2f0acc, 0xac4b7b3b, 0xe324578b, +0xc87928df, 0x9c1de566, 0x0ce2a17d, 0xaf2e13ce, 0x146a8659, 0x8727f6ae, +0xe2df7d03, 0x1a8e4cb4, 0xfa590976, 0x13a7c236, 0xc07489d0, 0xbe905e17, +0xafeb3d4b, 0x201e73f2, 0x5bdca12e, 0x3e15a852, 0xbcfc3271, 0x5d398410, +0x6bfacc15, 0x011fc61f, 0x43e43fd7, 0x0640717c, 0x96bfb3ff, 0x158eac19, +0x3b852e91, 0x74f9ceda, 0xcac71326, 0xfc0e312a, 0x20e8137b, 0xa1162611, +0x239ac7fe, 0xb9d00f8a, 0xea0b5241, 0x019f0d25, 0xc5153264, 0xb48a5547, +0xe54e339f, 0x17a6cca5, 0x5065af0d, 0x5ce648b9, 0xb457b342, 0xc1cb3f0e, +0x28d03c8b, 0x5144ed7a, 0xdb80779f, 0x53ce1b87, 0xbc071147, 0xbfe46c11, +0x7296785e, 0x83e4a63e, 0xc58982e9, 0x9538c5b9, 0xf14abaaa, 0xd915124c, +0x73540cd6, 0xe333696b, 0x58f9e00a, 0xd4dad10f, 0xc0de1420, 0x355e2bdc, +0xb2faa8fd, 0xbe6a12f1, 0x45d415cc, 0x47f5aed9, 0x4754e770, 0x2bb07385, +0x41374352, 0xf80beb47, 0xef02f35c, 0xc9c1b86e, 0x94b5785b, 0xba33123f, +0x7e39f0c9, 0x028a9286, 0x7d52c9f1, 0x06f04da6, 0xbc6a68d1, 0xfc70bace, +0x95b6a129, 0xfff224bb, 0x701ef3fb, 0x3309286f, 0x544ae8c1, 0xdca62c4a, +0xf8862ee2, 0xf9e3cd29, 0x2c07cce2, 0x8d93652a, 0xf47e4611, 0x4635f586, +0x1c03e0f4, 0x819724c7, 0x96b2a3f0, 0xeeb1ad95, 0xff08e517, 0xbd4ba6ed, +0x49ddb12d, 0x365734b5, 0x5edf7459, 0x2ee117a9, 0x067b9462, 0x549f6d5f, +0xe1dd8309, 0x12bb8e0a, 0x4382fe0f, 0x57fce11b, 0x1bc1c809, 0xd2741876, +0xcffd36b9, 0x45a64a67, 0xb77955d5, 0xb5825f24, 0x86eef2a8, 0xb66cac3b, +0xfec661f8, 0x531d5963, 0xa0a2f109, 0xe1795b68, 0x6bfd44e0, 0x0849af41, +0xf56d7d73, 0x3eab025f, 0xd109b015, 0xec24d23b, 0x8b3f603c, 0x648f421f, +0xc833f32a, 0x53cbda14, 0xb9b3fee4, 0xcf9ac8ab, 0x300f0548, 0xfe0bc595, +0x4f437a1a, 0xdacfae1f, 0x284e4a6d, 0x57815e28, 0x15d32b39, 0x68f85b97, +0x18b21602, 0x941259ef, 0x68598d90, 0x6dfd81db, 0xa8fc9a55, 0x60fef2b3, +0xcf07961b, 0x11b96588, 0xd9928a88, 0x2bbc72fd, 0x025f81b4, 0x44794d6d, +0x6a7420aa, 0xa73d4d89, 0x25b523ff, 0x521dbea3, 0xdacde374, 0xbf41776f, +0x9c1e3cb6, 0xb7b3499b, 0x8e154ed3, 0x1ac8f8c8, 0x4f2cf203, 0xa293c979, +0x2792a2e1, 0x329ca1dd, 0x01e1aa4a, 0x01a78d3e, 0x100c74ed, 0xb0db13c7, +0x99d53541, 0xd435a3b9, 0x898bd713, 0x069287bd, 0xe3175d24, 0xb09dc7fb, +0x7340d7d3, 0x81369cc3, 0x8f16a7a8, 0x43963d57, 0x58e0f6a3, 0x39803b6c, +0xa419a5ac, 0xf45fd6fa, 0xec389d5b, 0x636ce34f, 0x71405df6, 0xfc504724, +0xa3a3b192, 0x19df1379, 0xaf028f42, 0x3e1838e8, 0x6b7a64a9, 0xaa90ffee, +0x238ee398, 0x67eae4bd, 0x0a549a15, 0x125873d3, 0x00ece005, 0x9f7b2b8f, +0x7571f73c, 0x98940ab4, 0x192ed328, 0x8fae66da, 0x9063b323, 0x2542f666, +0x07ac7c8e, 0xce92857a, 0x3ded4b69, 0x50204cb4, 0x2a81cd17, 0xfd6ac536, +0xafb46424, 0xe802147b, 0x30a37858, 0x90697235, 0x1606e767, 0x26c1048e, +0x3a20df7b, 0xe127de97, 0x481abd6c, 0xc3a61d15, 0xbb2e929c, 0xd2ec0d65, +0xc0e53693, 0x500c2e9a, 0x316d0f4b, 0x18354452, 0xb2dd7728, 0x5a874b3e, +0x6dcf6f9c, 0x9d7c15ce, 0x1b25566d, 0xe13d6e7a, 0x9d9d06ae, 0x4d0e7d23, +0x33100e24, 0x7c20e0f8, 0x2cfc95bf, 0xb8e702a0, 0x5e60215b, 0xa516093e, +0x4cf3bed7, 0x4a0d6f56, 0xaf4901cc, 0xaeb7137e, 0x07d3c34d, 0x38641bc6, +0xf7366ca3, 0x6e01bd0d, 0x86818282, 0xf0f8712a, 0x431f57ad, 0x46b832c3, +0xec1893b8, 0xfa43c94f, 0x8b4c382a, 0x84cc0cec, 0x3f1016f3, 0xdbc7b88d, +0xa3c34e52, 0x50691492, 0x8b10e709, 0x5c5db670, 0xcda5888b, 0xa1ef2917, +0x1d4689ba, 0xc6470dcc, 0x7010373d, 0x4f27b426, 0x06b0de9d, 0xba83e005, +0x97cd5aad, 0xc1d0f41a, 0x01bae35a, 0xb30c805e, 0x8d7982eb, 0xe1a7b51d, +0x88a9f4d2, 0x7a6742a8, 0x0d9cb569, 0x06e9bc87, 0xf8c93b93, 0x89f3e80a, +0x96b971de, 0xcd548ef1, 0x5fbeb12a, 0x39c798e3, 0x90183352, 0x9b2476a7, +0x9437b0ae, 0x1f8045af, 0xf6ff8634, 0x019ec993, 0x0116e82e, 0x84ff34c8, +0xf1be20f9, 0x455510e4, 0x61cdbafe, 0x6f560fc5, 0x39fd9d2d, 0x54e343b2, +0x90448ad2, 0x24c4f180, 0x0fe4f0a7, 0x379ac6a0, 0x61b383f0, 0xb2228e0c, +0x223bc044, 0xcfbc389d, 0xae6a40e7, 0x8d445c62, 0x045eaf6f, 0x07df18c6, +0x4c057e85, 0x589113e6, 0xa3bf554e, 0x702b193b, 0x23af7d6a, 0xb98022ea, +0x96c45ca1, 0xe3b8c170, 0xed7bb4ac, 0x8a2b8f47, 0x6b57db19, 0x9d24d458, +0x054c66bf, 0xf2630696, 0x43524b13, 0xcd419b2a, 0xa248521e, 0xd7765fb3, +0x99fbe34e, 0x3b9ebb4c, 0xf7bfeb6c, 0xaf12561e, 0xc90c7f0d, 0x067ed7e9, +0x9102babe, 0xccc09846, 0x691fb1ed, 0x9b850f96, 0xaba5f22a, 0xcad31d58, +0xb1f25b0b, 0xa55fe490, 0x41e62678, 0xba3e83dd, 0x09d8a9dc, 0xe2f76f22, +0x6435706e, 0x81a0ef55, 0xb2790ad1, 0x793587d1, 0xebe10cc7, 0xde5fe03a, +0x308948bb, 0x439ece5c, 0x19fbef01, 0xb0de1b53, 0x95f4f310, 0xe6a7c965, +0x7b2e10a3, 0x13cf3df9, 0xed339445, 0x49a7728d, 0x4f226909, 0x63662893, +0x101c682b, 0x4dfd9492, 0x8315be80, 0x4adf6517, 0x4d11d338, 0x4881c0e6, +0x8e7886c4, 0x21c0c87a, 0x0d0fc0b7, 0xf0cc5a6d, 0x67c31f97, 0x06dd093c, +0x02c01a49, 0x8d6c5b62, 0x158056ad, 0x704ed553, 0xdfeaaa13, 0x5e4b40a1, +0xc3e6fab3, 0x9650ced9, 0xfe5c220c, 0x4bedb3bd, 0x07d0bd87, 0xef4a4fbd, +0x70d9324e, 0xd003ac29, 0x9fbea4fd, 0x81779460, 0xee9f96ad, 0x9b3faab4, +0xdcf35f55, 0x42e9fb1e, 0xa7398183, 0xc86a0769, 0x9b45aa5f, 0x32e84792, +0x4214845c, 0x62abf543, 0xf07ae499, 0x7b556d8c, 0xa42ac1ba, 0x0eef6f2d, +0x118ef8ac, 0x61597adc, 0xf7a00760, 0xb92c51c5, 0x049008b9, 0xc9da8a87, +0xdc48c672, 0xc4b39254, 0x204c8db4, 0xb2ef5950, 0xac687c00, 0x158dc3cd, +0x1c65e3b2, 0xf222864f, 0x44ee498a, 0x8921ce44, 0x0a8d112d, 0x7325da3c, +0x6aec7bb9, 0x6ca31b05, 0xa5febba9, 0x166c4771, 0x920dcda3, 0x8b2e615e, +0x3b392fa5, 0x9b177b4d, 0x8b0dfa1b, 0xdc16ab05, 0x4eeac34b, 0xf2d2fc8a, +0x6f04a6fb, 0xa14bda63, 0x8f194775, 0xf248e4a2, 0x4e7dd6f2, 0x4401cb93, +0xf693a2eb, 0xcb7e1637, 0x5d9abbd1, 0x613f63bd, 0xfb5fd56f, 0x70c5ea9b, +0xcc6c7a6b, 0xefaee1bd, 0x5d396541, 0x99b12bf4, 0x20873a04, 0x53961f78, +0x3c4f04ed, 0x4bd36330, 0xcdd5892e, 0x317de01b, 0xa3d53af7, 0xfb36a87b, +0xe55b4361, 0xf838e923, 0x9d91a30e, 0xd7d8357b, 0x28e5173c, 0x00fad75f, +0x39800da1, 0x894433ff, 0x1a62cdd7, 0x618b70b6, 0x60d770de, 0xbaed0285, +0x7bd24f54, 0x96a33975, 0x6884bd60, 0xeb75cf39, 0x19dd058a, 0xd4b8f064, +0x045178ca, 0xd9e3c1de, 0x91b494a6, 0x3316ac81, 0xb926c7d8, 0xc190f058, +0xce70a42c, 0x54f3b3c4, 0x79b20d90, 0x75177ad7, 0x9f010f51, 0x889a4fd5, +0x8fbbc8f0, 0xf1340040, 0xc22acb69, 0xe1473a0a, 0x52ddd7e3, 0x4cd54494, +0x4eb7a9ed, 0xd35e6a65, 0x3236b14c, 0x89786899, 0x3dc59704, 0x51d674be, +0xbcf13659, 0x6ef6266e, 0xf50ef497, 0x141354bc, 0xc8a42e76, 0x59b14233, +0x8b6cad7d, 0x061d8efe, 0xaca42513, 0x161cee21, 0xfeac5471, 0x4c499d31, +0xac2892cb, 0x9bacf4c5, 0xfff36588, 0x898f7a06, 0xad87be87, 0xebe7cb95, +0xa52b5fdf, 0x9f1f3945, 0xad435317, 0x1b316bcc, 0x8e6c0fd3, 0x767981d6, +0x1f86412b, 0x32b457f5, 0x30ab14db, 0x7120e8ae, 0x3972f57e, 0x2268af9d, +0xf45e5738, 0xc48bd523, 0x76726cae, 0x10bcdaa2, 0x49dcbe96, 0x8e28c545, +0x25fd1cc8, 0xc272f106, 0x1a34497e, 0x5f850d98, 0x92e324e4, 0x7e59f811, +0x0bba7ff0, 0x86b3b23f, 0x29558cee, 0xcbd55b0d, 0x15c10dc2, 0xd6617dd5, +0x41168898, 0x920132eb, 0xd10f51c6, 0xdc6eba51, 0xa07f89cc, 0xfcea8627, +0x44a52afa, 0x03d4230b, 0xdbb998d8, 0xf30d2a06, 0x08d35aeb, 0x9597f9bc, +0x720d2689, 0xf64314dd, 0xcadd04c5, 0x9cab9213, 0x22205cca, 0xd6cc196e, +0xa820e3c8, 0x7c262f92, 0xaf481f6b, 0xcecc0fe6, 0x2ec0b33a, 0x20aebc0a, +0xb5699b4b, 0xe0a559ba, 0xb9cd5224, 0x9f7ceb48, 0x35807e3c, 0x30be0794, +0x05463d41, 0x0e10046f, 0x776e71d6, 0xc6bac22d, 0xfee7f911, 0x3bc76715, +0x5024f666, 0x88a07482, 0x7dc55ed7, 0x3ee1d3b6, 0xec419af3, 0x4cd0e39f, +0x988f3802, 0x23baa080, 0xef9ce568, 0xf46ff60f, 0x2efcb093, 0x23b5b12a, +0xd5fb68c7, 0x58d45968, 0x6dce76cd, 0xf82c5a91, 0xfe6a2bf7, 0x0303b383, +0xf1b1196a, 0xa18d6d69, 0xb0025e33, 0x3f51e6c7, 0x0263bac1, 0x87743b29, +0x8f62b573, 0x7a80ad05, 0x95869387, 0x33e6a0f5, 0x50950449, 0x5da456ad, +0xb2ab1617, 0x7b345bf9, 0x7f923fd1, 0x4ca337fe, 0x95ccf61b, 0xfd8221ca, +0x0f19808f, 0x80029266, 0x915a3c47, 0x415045d7, 0x92f91b36, 0xd23576e5, +0x74a5652e, 0xe3d7288c, 0xba8213f7, 0x3a0ecd53, 0xa34fa01a, 0x62b4db3b, +0x3a96a793, 0xcef7b0da, 0xb64b8d17, 0x8716c3ed, 0xbd0ab055, 0xa6896492, +0x1250e75b, 0xb2bce1a7, 0x4100ca8f, 0x2e05625f, 0x8eb5bc83, 0x3dac8027, +0x62b3376d, 0xd6b2cc64, 0x4c9502dc, 0x6c1cfcbe, 0x380c1dce, 0x2b238ea8, +0x7051d59b, 0x580b122d, 0x6174b6e5, 0x4353d726, 0xba555765, 0xb645490f, +0x58427e5f, 0x06fc96e1, 0xae8f4528, 0xab0fee5e, 0x5de960b3, 0x63e79c19, +0x8552eeb8, 0x9344db26, 0x3326d17c, 0xa4174885, 0x6fe45472, 0x7bd20a9e, +0xa5e4aba0, 0x77b509af, 0x22f64240, 0x40bc5144, 0x57ca1b2d, 0xc1260090, +0x008997aa, 0xe8029716, 0x6efb01f3, 0xc73e1366, 0x9de737ff, 0x1b5fabe8, +0x957bb682, 0x5e27f5a5, 0x7777491a, 0x9b6fc6c1, 0xb0802205, 0x16e94d46, +0x58079aee, 0x2b24c4c4, 0xfffb2e7b, 0x7c5560e1, 0xe1ad3009, 0x6a0c0027, +0x09aea741, 0x9af375a6, 0x7106b7bf, 0x1cdcff69, 0xa007f9c2, 0x6f7729c0, +0xeac4297f, 0x8622ede7, 0x89346bfa, 0xc24674b5, 0xd1eabf00, 0xfa6b2d39, +0x04b0c153, 0x6adf49dc, 0x48ea0937, 0xae771ea6, 0xd4208255, 0x4052586d, +0x9c6aa640, 0x9adbb652, 0x017d8661, 0xb20855f0, 0x1e2ed7b6, 0x70af7f25, +0x354df0de, 0xf2fdfde7, 0x0844478c, 0x63993fb2, 0xc49e71ae, 0x1e15ae81, +0x932ad241, 0xba83cf8f, 0xc7e227f3, 0xd1171a85, 0x63ea34a0, 0xe74f042b, +0xd88173d9, 0xba7d8443, 0x9d47c6c4, 0xbe282a93, 0x6142fbac, 0x69307e44, +0xbb86a7f1, 0x9fa3418f, 0x3a3eadce, 0x9804167b, 0x2d5eb47a, 0x5aa2372d, +0x66eb8e1c, 0xf56da1fd, 0x3753eee9, 0xac78c709, 0x787ae648, 0x42d4fa7f, +0x5a036af5, 0xfc6dfa54, 0x269f1195, 0xef1c698d, 0x2932b2d6, 0xba10b26c, +0xf9b58396, 0x8f04a38a, 0x31c1264d, 0xf11c5ac6, 0x693d8ef6, 0x25b1d015, +0x01c06e8f, 0x9c250a6f, 0x4b8fffcd, 0x1eff0743, 0x5a4974ec, 0x9d2d24c1, +0x5562b2ae, 0x2a5b2deb, 0xd4a32596, 0x8ec43665, 0xde3ae654, 0x99e979d0, +0xb6250c7c, 0x478a2891, 0xe1d6c035, 0x1dc453e8, 0xc7490c8d, 0xd4e85297, +0x8fa94952, 0xc863cde8, 0x724dbbc8, 0x89ba9607, 0x0144ba1b, 0x3fed833c, +0xd16278d4, 0x0080f227, 0xfbe241c5, 0x61892e7f, 0x503dced0, 0xa54d6ea5, +0xcb5024a3, 0x591e11d1, 0xabeb85db, 0xc8314e58, 0xfb7792cb, 0xc70e5ada, +0xcc07e739, 0xdb3ad4d6, 0x7a2d24e8, 0x2a0e216a, 0xa04aa774, 0x1762ac59, +0xed53f5f9, 0x261a791e, 0x7a4102ae, 0x4650dcc9, 0xffa48ffa, 0x35a8f4e1, +0x98e0f041, 0x3e3a9a83, 0xfb43c8e9, 0x7e0e6a1e, 0x06f718e5, 0x6812c114, +0xaf21e4a1, 0x10ce04ca, 0x6e465a05, 0x250758f2, 0xfdf02f68, 0x426ec74c, +0x51e44567, 0x17e637cc, 0x9da657e4, 0x39ecef6e, 0xd31c4423, 0x12bc06b4, +0x35af82e0, 0x76379d09, 0x7fb7aa1e, 0x0b73e469, 0x7b3697a4, 0x6bbbca38, +0x018d2e07, 0x8d74a290, 0x02e00314, 0xdc9d1f4f, 0x62cbdd73, 0xd1173c86, +0x62c1d5f2, 0xdfb16100, 0x37cc28e8, 0x43954e64, 0xeaa699d9, 0x0cd0a997, +0xa0f26045 + +ea = +17868 + +eb = +17868 + +cab = +2 + +c = +2 + +c_neg = +0 + +k_pos = +5952 + +k_neg = +5888 + +ncb_pos = +17952 + +ncb_neg = +17760 + +r = +0 + +rv_index = +0 + +code_block_mode = +0 + +op_flags = +RTE_BBDEV_TURBO_CRC_24B_ATTACH + +expected_status = +OK diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_enc_c3_k4800_r2_e14412_crc24b.data b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c3_k4800_r2_e14412_crc24b.data new file mode 100644 index 00000000..9e17429d --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c3_k4800_r2_e14412_crc24b.data @@ -0,0 +1,153 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_ENC + +input0 = +0xb0fe86aa, 0xffbf2968, 0x4f55a631, 0xf2152dbc, 0x195a49a5, 0xa4aa7c6c, +0x48a7be38, 0xd32c84f2, 0xb040da60, 0xb39eb07c, 0x3443aad6, 0x19f52e13, +0xb2914341, 0x0d5dfdb3, 0x1ba8aec0, 0x62d0c7be, 0x1d55475c, 0x10f7dd79, +0x967e5333, 0xc0d524c1, 0xdca8da87, 0x1146ecf7, 0xfa09a85c, 0x83be800f, +0x11b814cc, 0x362a4704, 0x4f22e7ce, 0x9985a0e2, 0x9de4be72, 0x488a45b3, +0x6996ade2, 0xdb18c09a, 0x487d62a3, 0x94f82d13, 0xa0743f83, 0x75d2f5dd, +0x0e470346, 0x6e2ec2e5, 0x1a67fe5f, 0xc369330d, 0x860e8766, 0x42c110f6, +0x804a9985, 0x55db02b4, 0xb899fc5c, 0x57811d20, 0xda4f1f9d, 0x2a2efc30, +0x953e7487, 0x4b0c3065, 0x247d5d47, 0xdde382e7, 0x1d6d2ad7, 0x4358ac24, +0x2d0fd97f, 0x1483821e, 0xf6c74ee5, 0x23b5c2f4, 0x95d4a5fc, 0x486f370b, +0x8dbae43d, 0x248f17d5, 0x2df76099, 0xd19ec872, 0xf592aa0e, 0x8de04c3c, +0x3ada94fb, 0x9dd001a3, 0x2b4db6d1, 0x791cf495, 0x57dc764b, 0x21789f1f, +0x0d38adf5, 0xd8cefdc2, 0xed98f8bf, 0x024bc4ed, 0x4f9d3062, 0x332d27c5, +0xde70ad4e, 0xd85872d2, 0x5d0e29c0, 0xa4fd1887, 0x3fa67ba1, 0x46b9b008, +0x055bbf61, 0xf78e59f0, 0x287d2e3b, 0x5b473fe7, 0xd7ad6e04, 0x9cdf4986, +0x830432ae, 0xffded1ba, 0x35c00f77, 0xcf0a6648, 0x196c03aa, 0xba4e2dd3, +0xdaf70fe4, 0x883de2cd, 0xedc3b52e, 0x14e638c1, 0xe607103a, 0xc4cfb700, +0x4ed9843d, 0xc442a0cb, 0xc0d3bba2, 0x36789ba8, 0x9c9b7a24, 0xb943b6a4, +0x8f2c6879, 0x0fb16a0c, 0x2677bada, 0xd14f2f7e, 0x662ec398, 0x1e37b7ae, +0x8ad0312d, 0x21adb6a1, 0x072a11fd, 0x7ae828d1, 0xb510655f, 0x09f7b477, +0x10a175fc, 0xb7086f51, 0x24546af3, 0xfae81b50, 0xcb328832, 0x63693615, +0xa37c3678, 0x15fb6364, 0xb5f75a78, 0xfa9aa5f6, 0x03804a0e, 0x1d245890, +0x06d6cdcd, 0x7485a3c7, 0x35a008ae, 0x1abeaf1c, 0xd9c55966, 0x2b694666, +0x5341f2df, 0xcb647280, 0xfb88d81e, 0xf99a44d0, 0xb0585534, 0x0877b530, +0x752ab39a, 0xd493d8bc, 0xfaa6a57a, 0xccb7c008, 0x9c35994b, 0xb8 + +output0 = +0xb0fe86aa, 0xffbf2968, 0x4f55a631, 0xf2152dbc, 0x195a49a5, 0xa4aa7c6c, +0x48a7be38, 0xd32c84f2, 0xb040da60, 0xb39eb07c, 0x3443aad6, 0x19f52e13, +0xb2914341, 0x0d5dfdb3, 0x1ba8aec0, 0x62d0c7be, 0x1d55475c, 0x10f7dd79, +0x967e5333, 0xc0d524c1, 0xdca8da87, 0x1146ecf7, 0xfa09a85c, 0x83be800f, +0x11b814cc, 0x362a4704, 0x4f22e7ce, 0x9985a0e2, 0x9de4be72, 0x488a45b3, +0x6996ade2, 0xdb18c09a, 0x487d62a3, 0x94f82d13, 0xa0743f83, 0x75d2f5dd, +0x0e470346, 0x6e2ec2e5, 0x1a67fe5f, 0xc369330d, 0x860e8766, 0x42c110f6, +0x804a9985, 0x55db02b4, 0xb899fc5c, 0x57811d20, 0xda4f1f9d, 0x2a2efc30, +0x953e7487, 0x4b0c3065, 0x247d5d47, 0xdde382e7, 0x1d6d2ad7, 0x4358ac24, +0x2d0fd97f, 0x1483821e, 0xf6c74ee5, 0x23b5c2f4, 0x95d4a5fc, 0x486f370b, +0x8dbae43d, 0x248f17d5, 0x2df76099, 0xd19ec872, 0xf592aa0e, 0x8de04c3c, +0x3ada94fb, 0x9dd001a3, 0x2b4db6d1, 0x791cf495, 0x57dc764b, 0x21789f1f, +0x0d38adf5, 0xd8cefdc2, 0xed98f8bf, 0x024bc4ed, 0x4f9d3062, 0x332d27c5, +0xde70ad4e, 0xd85872d2, 0x5d0e29c0, 0xa4fd1887, 0x3fa67ba1, 0x46b9b008, +0x055bbf61, 0xf78e59f0, 0x287d2e3b, 0x5b473fe7, 0xd7ad6e04, 0x9cdf4986, +0x830432ae, 0xffded1ba, 0x35c00f77, 0xcf0a6648, 0x196c03aa, 0xba4e2dd3, +0xdaf70fe4, 0x883de2cd, 0xedc3b52e, 0x14e638c1, 0xe607103a, 0xc4cfb700, +0x4ed9843d, 0xc442a0cb, 0xc0d3bba2, 0x36789ba8, 0x9c9b7a24, 0xb943b6a4, +0x8f2c6879, 0x0fb16a0c, 0x2677bada, 0xd14f2f7e, 0x662ec398, 0x1e37b7ae, +0x8ad0312d, 0x21adb6a1, 0x072a11fd, 0x7ae828d1, 0xb510655f, 0x09f7b477, +0x10a175fc, 0xb7086f51, 0x24546af3, 0xfae81b50, 0xcb328832, 0x63693615, +0xa37c3678, 0x15fb6364, 0xb5f75a78, 0xfa9aa5f6, 0x03804a0e, 0x1d245890, +0x06d6cdcd, 0x7485a3c7, 0x35a008ae, 0x1abeaf1c, 0xd9c55966, 0x2b694666, +0x5341f2df, 0xcb647280, 0xfb88d81e, 0xf99a44d0, 0xb0585534, 0x0877b530, +0x752ab39a, 0xd493d8bc, 0xfaa6a57a, 0xccb7c008, 0x9c35994b, 0x47ee26b8, +0x94f624ac, 0xa3bd4876, 0xf7d4854e, 0x8871d433, 0x9321d942, 0x7b626be8, +0x72c934b0, 0x3b7af8a4, 0x5102c29f, 0x710e4dbc, 0x99708292, 0x1458c4c1, +0x61026bc5, 0xe776e388, 0x4a0222b3, 0x760e5aaf, 0x662f3583, 0x5ab1005b, +0xe527ef70, 0x4170d611, 0x307bebc4, 0xfdd00caf, 0xbaae1044, 0xcab4d459, +0x38281dcf, 0x90580c89, 0x49cf5986, 0xa27da769, 0xceced49b, 0x5ea37953, +0x8a7e6c1c, 0x1e01b4e2, 0xe08026ae, 0x3754534a, 0x903b0ecf, 0x65f97a55, +0x90798ed9, 0x9d1133bc, 0xe356a39f, 0xe47acbce, 0x01ccf326, 0x1954fd3d, +0x240e69f8, 0x1da20bb4, 0xe1ab1684, 0x44c65d48, 0xd265e6c2, 0x51d4ef07, +0x000970ef, 0xfeb939f4, 0x5dcc0132, 0x2bd27ae5, 0xba5dbd25, 0xa9d190e4, +0x61556bec, 0x7fd6caba, 0x7fe312cb, 0xdd319413, 0x92a5dbbf, 0x17e61915, +0x56067b67, 0x3dc348c0, 0x58c17fe0, 0xbe6bffcc, 0xd379026c, 0xc4174780, +0xcce7f026, 0xb74b7eb0, 0xe4d5f625, 0x6bc16d3b, 0xff3e300e, 0x83f0d55a, +0xf2e537df, 0x75c18f78, 0xa5458d1a, 0x47c778b0, 0x92eb8716, 0xcb5816fd, +0xc2cc7079, 0xa7f1dc75, 0xed9e5ffa, 0xb7d6747f, 0xa2dc6907, 0x99b4d187, +0x69f3138a, 0xbd285a1f, 0xb8ee18e0, 0x734b77a5, 0xc0700f69, 0x6c72a77a, +0x76609ba9, 0xcfae9b73, 0x2cd329f0, 0x0d45aa12, 0x419fbcd7, 0x03e00f00, +0x0effee99, 0x7f879eb4, 0x29a4c8df, 0x4432400a, 0x22b9cf55, 0xa1c4c645, +0x200251b0, 0xff293906, 0xe11288e2, 0xde1e8ec7, 0x675752d1, 0x9630743e, +0xd848c67e, 0x1eedfda9, 0x58b954fc, 0xa0dc7f1c, 0x0aea313e, 0x062b9449, +0x0f17e57f, 0x96def6ec, 0x2ba45f14, 0xd9a5f8c5, 0x231483c8, 0x29b8783e, +0xa6d24594, 0x1a62ffcd, 0x54c87a7f, 0x6fa9ca9a, 0xfff0f890, 0x51d6ae46, +0xa96e82b0, 0x68cd9808, 0x56214df5, 0x2169defa, 0x72c1ecce, 0x2448e3c9, +0x8eb2b3b3, 0x62c89c11, 0xbba20ee2, 0xea33f53a, 0x8b96a6b8, 0x9d33d551, +0x363bd14e, 0x5fdd2b5f, 0xf8187546, 0xb692beab, 0x8df6b4c0, 0x753e2302, +0xb90f7a37, 0x2b6bac6a, 0x13a07bc5, 0xb67f6d2b, 0x47b21569, 0xc2ab143f, +0xf86ce30a, 0xde2dab70, 0x6f258860, 0x2878735c, 0x2aaeac20, 0xda80fb3e, +0xe7a8ccf6, 0xbf011844, 0x40a610f8, 0x82f3fdf7, 0xadc4ec2c, 0xb8551030, +0x76004380, 0xa384ff18, 0x080cf587, 0x6c4991d3, 0x2daea9aa, 0xa92d0c4f, +0x4a9d117e, 0x2761f025, 0x96b40443, 0x57a82be4, 0x62374c44, 0x55dc64ae, +0x28aa9f0c, 0x03a3b963, 0x41dbaa26, 0xb3c23735, 0x971cbd31, 0x939a9f80, +0x76439fdc, 0xa9df79d6, 0x926ae3e2, 0x5ee75745, 0xf2396e52, 0xe18bd816, +0x3f9834b9, 0x816a07bc, 0x8f873310, 0x45cf9b96, 0x0ce634dd, 0xe64a16d9, +0x2733775a, 0x2b648c7e, 0xe600ee8e, 0xd99d8ae3, 0x10dadf2a, 0x904d3f87, +0x3963e9e7, 0x47fcce89, 0xc256c898, 0x7db6cb66, 0xe1611a8b, 0xed81b10d, +0x75eff974, 0x8a0a3d67, 0xa44311ff, 0x6f876783, 0x43dc93ce, 0x88616a33, +0xa8706e8f, 0x33a2cbed, 0x3a6d20c3, 0x55175086, 0x39680945, 0x2d779a7b, +0x0818a4ce, 0x55558918, 0xf7c08d35, 0x980a3038, 0x9cf068db, 0x3385d333, +0xd81b33fb, 0x188018d5, 0x47c57bd3, 0x9ec2324f, 0x6901cd77, 0xc3bac44f, +0x4d96aba8, 0x9094da5b, 0xa67a1353, 0x1fdfc4db, 0xa2fbeefa, 0xab4828e3, +0x37d1db45, 0x0d33b3e9, 0x1ad72bb9, 0x7257bf9c, 0x2ec35167, 0xa4488b7f, +0xf9dae588, 0x1038905a, 0x88ddf410, 0xaac11693, 0x24ac025d, 0x56cefbb5, +0x6afe7f59, 0xc7f989e8, 0x15872570, 0x1bf16cdb, 0xfe9c93ce, 0x1fc9a076, +0x85d37185, 0x1078cd31, 0xe1cd0327, 0x6d5315bc, 0x298cd836, 0xc8e21f06, +0xe561c32d, 0x8ec404b4, 0x4d39bfbb, 0x24ede8c8, 0x451d6034, 0x3bafeea2, +0x202f0ccf, 0x1fad37ce, 0xf04b5693, 0xeee57cd9, 0x5ef70007, 0x018e8a4f, +0xfa61c9a9, 0x09989fcf, 0xe66b558b, 0x966efd48, 0x7525021d, 0xe7978b5e, +0x7eb1d6dc, 0xa10c5e5b, 0xb7815e69, 0x7d486cfb, 0xcffdeb2a, 0x7375cb32, +0x599008dc, 0xff91c796, 0x560ed4ad, 0x14e9a876, 0xfccf6a66, 0xa58be028, +0xea9d408b, 0x3afc373b, 0xee008458, 0x19b6042e, 0x84806314, 0x431a4ba4, +0x009ad6a1, 0xd7c62bf4, 0x1bebecba, 0x5c662f69, 0x83bfdea1, 0x45872a9a, +0x00c9 + + +ea = +14412 + +eb = +14412 + +cab = +4 + +c = +4 + +c_neg = +0 + +k_pos = +4800 + +k_neg = +4736 + +ncb_pos = +14496 + +ncb_neg = +14304 + +r = +2 + +rv_index = +0 + +code_block_mode = +0 + +op_flags = +RTE_BBDEV_TURBO_CRC_24B_ATTACH + +expected_status = +OK \ No newline at end of file diff --git a/dpdk/app/test-bbdev/test_vectors/turbo_enc_c4_k4800_r2_e14412_crc24b.data b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c4_k4800_r2_e14412_crc24b.data new file mode 100644 index 00000000..ff95a3f9 --- /dev/null +++ b/dpdk/app/test-bbdev/test_vectors/turbo_enc_c4_k4800_r2_e14412_crc24b.data @@ -0,0 +1,252 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Intel Corporation + +op_type = +RTE_BBDEV_OP_TURBO_ENC + +input0 = +0xb0fe86aa, 0xffbf2968, 0x4f55a631, 0xf2152dbc, 0x195a49a5, 0xa4aa7c6c, +0x48a7be38, 0xd32c84f2, 0xb040da60, 0xb39eb07c, 0x3443aad6, 0x19f52e13, +0xb2914341, 0x0d5dfdb3, 0x1ba8aec0, 0x62d0c7be, 0x1d55475c, 0x10f7dd79, +0x967e5333, 0xc0d524c1, 0xdca8da87, 0x1146ecf7, 0xfa09a85c, 0x83be800f, +0x11b814cc, 0x362a4704, 0x4f22e7ce, 0x9985a0e2, 0x9de4be72, 0x488a45b3, +0x6996ade2, 0xdb18c09a, 0x487d62a3, 0x94f82d13, 0xa0743f83, 0x75d2f5dd, +0x0e470346, 0x6e2ec2e5, 0x1a67fe5f, 0xc369330d, 0x860e8766, 0x42c110f6, +0x804a9985, 0x55db02b4, 0xb899fc5c, 0x57811d20, 0xda4f1f9d, 0x2a2efc30, +0x953e7487, 0x4b0c3065, 0x247d5d47, 0xdde382e7, 0x1d6d2ad7, 0x4358ac24, +0x2d0fd97f, 0x1483821e, 0xf6c74ee5, 0x23b5c2f4, 0x95d4a5fc, 0x486f370b, +0x8dbae43d, 0x248f17d5, 0x2df76099, 0xd19ec872, 0xf592aa0e, 0x8de04c3c, +0x3ada94fb, 0x9dd001a3, 0x2b4db6d1, 0x791cf495, 0x57dc764b, 0x21789f1f, +0x0d38adf5, 0xd8cefdc2, 0xed98f8bf, 0x024bc4ed, 0x4f9d3062, 0x332d27c5, +0xde70ad4e, 0xd85872d2, 0x5d0e29c0, 0xa4fd1887, 0x3fa67ba1, 0x46b9b008, +0x055bbf61, 0xf78e59f0, 0x287d2e3b, 0x5b473fe7, 0xd7ad6e04, 0x9cdf4986, +0x830432ae, 0xffded1ba, 0x35c00f77, 0xcf0a6648, 0x196c03aa, 0xba4e2dd3, +0xdaf70fe4, 0x883de2cd, 0xedc3b52e, 0x14e638c1, 0xe607103a, 0xc4cfb700, +0x4ed9843d, 0xc442a0cb, 0xc0d3bba2, 0x36789ba8, 0x9c9b7a24, 0xb943b6a4, +0x8f2c6879, 0x0fb16a0c, 0x2677bada, 0xd14f2f7e, 0x662ec398, 0x1e37b7ae, +0x8ad0312d, 0x21adb6a1, 0x072a11fd, 0x7ae828d1, 0xb510655f, 0x09f7b477, +0x10a175fc, 0xb7086f51, 0x24546af3, 0xfae81b50, 0xcb328832, 0x63693615, +0xa37c3678, 0x15fb6364, 0xb5f75a78, 0xfa9aa5f6, 0x03804a0e, 0x1d245890, +0x06d6cdcd, 0x7485a3c7, 0x35a008ae, 0x1abeaf1c, 0xd9c55966, 0x2b694666, +0x5341f2df, 0xcb647280, 0xfb88d81e, 0xf99a44d0, 0xb0585534, 0x0877b530, +0x752ab39a, 0xd493d8bc, 0xfaa6a57a, 0xccb7c008, 0x9c35994b, 0x9cca47b8, +0xff47dda2, 0xa557841a, 0x7459fb6a, 0x93ec6495, 0x0ef7e6d2, 0xa8a6e6d4, +0x46364f91, 0xd52c5776, 0x4dfd0d25, 0xd458c769, 0x24864930, 0xf7789991, +0xfcc231f2, 0xc8e4cfd0, 0x50975038, 0x989398d1, 0x04504b14, 0x5e529570, +0xf8c17202, 0xaf459a8a, 0xac2fecb2, 0x0eb3a176, 0xf6bac24f, 0x6fbddade, +0x42456542, 0x83c10198, 0xbea4efaf, 0xff2f2966, 0x5edae82b, 0x09c27f13, +0xa094c3b4, 0xc53b810e, 0x7f1cc39d, 0x25731c8a, 0x8d34b6ee, 0x5fb78b80, +0x2980eef8, 0x7b76a9fb, 0x37bfacb9, 0x80f7a0ea, 0xd1e3e69d, 0xcda19b8b, +0x9af26246, 0xf55495cf, 0x6f45ad1b, 0xfe2cb329, 0xc33cb7e6, 0x67bbde4a, +0x32310c7b, 0x80e6b00a, 0xdd4004d2, 0x0e0eb8b6, 0x6f8d951f, 0xb196cb40, +0x6dbe29dd, 0xa138a375, 0xd4f5b058, 0x69ea638f, 0x50438a7a, 0xe5530eb3, +0x527cb218, 0xbfa67b9d, 0x7efb86df, 0x724ed65b, 0x38006782, 0x6a9924a4, +0x749955c2, 0x5cb82627, 0xa74ceecb, 0x89eb103d, 0x89a12973, 0x660613c4, +0x8bdf96ae, 0xf08afcce, 0xa4023081, 0x79c6786c, 0x8026bdf0, 0x621f29a1, +0xa8d8d3ec, 0x4ee5d537, 0xda83c99f, 0x9a69d0a8, 0xae027f9a, 0x639d9cc9, +0xe80e2be1, 0x89d78cfb, 0x862fabb7, 0xb248c138, 0x3fe35f29, 0xf8b74cc6, +0x27741dcf, 0xf9236485, 0x05225155, 0x4ed40ee6, 0xb30e4a95, 0x7d890ee0, +0x0a97eb0d, 0xe7877652, 0x3de6fa67, 0xa3a5961e, 0x1bdf2af5, 0x5a7dcbbe, +0x1641f1af, 0x4db5845b, 0x735761f2, 0x32a1431e, 0xab1e00b1, 0x495ddeb7, +0xb480f29c, 0x47d83714, 0x8ba04129, 0x8c96062e, 0x657e7f7f, 0x30dfba23, +0xfd4ee1bc, 0x9ffe10b1, 0xc917d2db, 0x18f0bd8d, 0x4b6c35d7, 0x54896222, +0x7ce0f6c6, 0xb7813f4a, 0x71c52b4b, 0xac65da3a, 0x9aef9b4e, 0x85f9657e, +0x62ba1965, 0xa1e39267, 0x40f66fb2, 0x10433b94, 0x851df333, 0xf0c6aa26, +0xe0e19368, 0x5a706636, 0xf5de1e8c, 0x5ebc735a, 0x0967ef39, 0xf76c1fb7, +0xe53f7f16, 0x51f2c4eb, 0x047dda3d, 0x73518b4a, 0x3ad90dd3, 0xaee51fc1, +0x78f01060, 0x98d1502c, 0x990d5499, 0x7a19453c, 0x104c + +output0 = +0xb0fe86aa, 0xffbf2968, 0x4f55a631, 0xf2152dbc, 0x195a49a5, 0xa4aa7c6c, +0x48a7be38, 0xd32c84f2, 0xb040da60, 0xb39eb07c, 0x3443aad6, 0x19f52e13, +0xb2914341, 0x0d5dfdb3, 0x1ba8aec0, 0x62d0c7be, 0x1d55475c, 0x10f7dd79, +0x967e5333, 0xc0d524c1, 0xdca8da87, 0x1146ecf7, 0xfa09a85c, 0x83be800f, +0x11b814cc, 0x362a4704, 0x4f22e7ce, 0x9985a0e2, 0x9de4be72, 0x488a45b3, +0x6996ade2, 0xdb18c09a, 0x487d62a3, 0x94f82d13, 0xa0743f83, 0x75d2f5dd, +0x0e470346, 0x6e2ec2e5, 0x1a67fe5f, 0xc369330d, 0x860e8766, 0x42c110f6, +0x804a9985, 0x55db02b4, 0xb899fc5c, 0x57811d20, 0xda4f1f9d, 0x2a2efc30, +0x953e7487, 0x4b0c3065, 0x247d5d47, 0xdde382e7, 0x1d6d2ad7, 0x4358ac24, +0x2d0fd97f, 0x1483821e, 0xf6c74ee5, 0x23b5c2f4, 0x95d4a5fc, 0x486f370b, +0x8dbae43d, 0x248f17d5, 0x2df76099, 0xd19ec872, 0xf592aa0e, 0x8de04c3c, +0x3ada94fb, 0x9dd001a3, 0x2b4db6d1, 0x791cf495, 0x57dc764b, 0x21789f1f, +0x0d38adf5, 0xd8cefdc2, 0xed98f8bf, 0x024bc4ed, 0x4f9d3062, 0x332d27c5, +0xde70ad4e, 0xd85872d2, 0x5d0e29c0, 0xa4fd1887, 0x3fa67ba1, 0x46b9b008, +0x055bbf61, 0xf78e59f0, 0x287d2e3b, 0x5b473fe7, 0xd7ad6e04, 0x9cdf4986, +0x830432ae, 0xffded1ba, 0x35c00f77, 0xcf0a6648, 0x196c03aa, 0xba4e2dd3, +0xdaf70fe4, 0x883de2cd, 0xedc3b52e, 0x14e638c1, 0xe607103a, 0xc4cfb700, +0x4ed9843d, 0xc442a0cb, 0xc0d3bba2, 0x36789ba8, 0x9c9b7a24, 0xb943b6a4, +0x8f2c6879, 0x0fb16a0c, 0x2677bada, 0xd14f2f7e, 0x662ec398, 0x1e37b7ae, +0x8ad0312d, 0x21adb6a1, 0x072a11fd, 0x7ae828d1, 0xb510655f, 0x09f7b477, +0x10a175fc, 0xb7086f51, 0x24546af3, 0xfae81b50, 0xcb328832, 0x63693615, +0xa37c3678, 0x15fb6364, 0xb5f75a78, 0xfa9aa5f6, 0x03804a0e, 0x1d245890, +0x06d6cdcd, 0x7485a3c7, 0x35a008ae, 0x1abeaf1c, 0xd9c55966, 0x2b694666, +0x5341f2df, 0xcb647280, 0xfb88d81e, 0xf99a44d0, 0xb0585534, 0x0877b530, +0x752ab39a, 0xd493d8bc, 0xfaa6a57a, 0xccb7c008, 0x9c35994b, 0x47ee26b8, +0x94f624ac, 0xa3bd4876, 0xf7d4854e, 0x8871d433, 0x9321d942, 0x7b626be8, +0x72c934b0, 0x3b7af8a4, 0x5102c29f, 0x710e4dbc, 0x99708292, 0x1458c4c1, +0x61026bc5, 0xe776e388, 0x4a0222b3, 0x760e5aaf, 0x662f3583, 0x5ab1005b, +0xe527ef70, 0x4170d611, 0x307bebc4, 0xfdd00caf, 0xbaae1044, 0xcab4d459, +0x38281dcf, 0x90580c89, 0x49cf5986, 0xa27da769, 0xceced49b, 0x5ea37953, +0x8a7e6c1c, 0x1e01b4e2, 0xe08026ae, 0x3754534a, 0x903b0ecf, 0x65f97a55, +0x90798ed9, 0x9d1133bc, 0xe356a39f, 0xe47acbce, 0x01ccf326, 0x1954fd3d, +0x240e69f8, 0x1da20bb4, 0xe1ab1684, 0x44c65d48, 0xd265e6c2, 0x51d4ef07, +0x000970ef, 0xfeb939f4, 0x5dcc0132, 0x2bd27ae5, 0xba5dbd25, 0xa9d190e4, +0x61556bec, 0x7fd6caba, 0x7fe312cb, 0xdd319413, 0x92a5dbbf, 0x17e61915, +0x56067b67, 0x3dc348c0, 0x58c17fe0, 0xbe6bffcc, 0xd379026c, 0xc4174780, +0xcce7f026, 0xb74b7eb0, 0xe4d5f625, 0x6bc16d3b, 0xff3e300e, 0x83f0d55a, +0xf2e537df, 0x75c18f78, 0xa5458d1a, 0x47c778b0, 0x92eb8716, 0xcb5816fd, +0xc2cc7079, 0xa7f1dc75, 0xed9e5ffa, 0xb7d6747f, 0xa2dc6907, 0x99b4d187, +0x69f3138a, 0xbd285a1f, 0xb8ee18e0, 0x734b77a5, 0xc0700f69, 0x6c72a77a, +0x76609ba9, 0xcfae9b73, 0x2cd329f0, 0x0d45aa12, 0x419fbcd7, 0x03e00f00, +0x0effee99, 0x7f879eb4, 0x29a4c8df, 0x4432400a, 0x22b9cf55, 0xa1c4c645, +0x200251b0, 0xff293906, 0xe11288e2, 0xde1e8ec7, 0x675752d1, 0x9630743e, +0xd848c67e, 0x1eedfda9, 0x58b954fc, 0xa0dc7f1c, 0x0aea313e, 0x062b9449, +0x0f17e57f, 0x96def6ec, 0x2ba45f14, 0xd9a5f8c5, 0x231483c8, 0x29b8783e, +0xa6d24594, 0x1a62ffcd, 0x54c87a7f, 0x6fa9ca9a, 0xfff0f890, 0x51d6ae46, +0xa96e82b0, 0x68cd9808, 0x56214df5, 0x2169defa, 0x72c1ecce, 0x2448e3c9, +0x8eb2b3b3, 0x62c89c11, 0xbba20ee2, 0xea33f53a, 0x8b96a6b8, 0x9d33d551, +0x363bd14e, 0x5fdd2b5f, 0xf8187546, 0xb692beab, 0x8df6b4c0, 0x753e2302, +0xb90f7a37, 0x2b6bac6a, 0x13a07bc5, 0xb67f6d2b, 0x47b21569, 0xc2ab143f, +0xf86ce30a, 0xde2dab70, 0x6f258860, 0x2878735c, 0x2aaeac20, 0xda80fb3e, +0xe7a8ccf6, 0xbf011844, 0x40a610f8, 0x82f3fdf7, 0xadc4ec2c, 0xb8551030, +0x76004380, 0xa384ff18, 0x080cf587, 0x6c4991d3, 0x2daea9aa, 0xa92d0c4f, +0x4a9d117e, 0x2761f025, 0x96b40443, 0x57a82be4, 0x62374c44, 0x55dc64ae, +0x28aa9f0c, 0x03a3b963, 0x41dbaa26, 0xb3c23735, 0x971cbd31, 0x939a9f80, +0x76439fdc, 0xa9df79d6, 0x926ae3e2, 0x5ee75745, 0xf2396e52, 0xe18bd816, +0x3f9834b9, 0x816a07bc, 0x8f873310, 0x45cf9b96, 0x0ce634dd, 0xe64a16d9, +0x2733775a, 0x2b648c7e, 0xe600ee8e, 0xd99d8ae3, 0x10dadf2a, 0x904d3f87, +0x3963e9e7, 0x47fcce89, 0xc256c898, 0x7db6cb66, 0xe1611a8b, 0xed81b10d, +0x75eff974, 0x8a0a3d67, 0xa44311ff, 0x6f876783, 0x43dc93ce, 0x88616a33, +0xa8706e8f, 0x33a2cbed, 0x3a6d20c3, 0x55175086, 0x39680945, 0x2d779a7b, +0x0818a4ce, 0x55558918, 0xf7c08d35, 0x980a3038, 0x9cf068db, 0x3385d333, +0xd81b33fb, 0x188018d5, 0x47c57bd3, 0x9ec2324f, 0x6901cd77, 0xc3bac44f, +0x4d96aba8, 0x9094da5b, 0xa67a1353, 0x1fdfc4db, 0xa2fbeefa, 0xab4828e3, +0x37d1db45, 0x0d33b3e9, 0x1ad72bb9, 0x7257bf9c, 0x2ec35167, 0xa4488b7f, +0xf9dae588, 0x1038905a, 0x88ddf410, 0xaac11693, 0x24ac025d, 0x56cefbb5, +0x6afe7f59, 0xc7f989e8, 0x15872570, 0x1bf16cdb, 0xfe9c93ce, 0x1fc9a076, +0x85d37185, 0x1078cd31, 0xe1cd0327, 0x6d5315bc, 0x298cd836, 0xc8e21f06, +0xe561c32d, 0x8ec404b4, 0x4d39bfbb, 0x24ede8c8, 0x451d6034, 0x3bafeea2, +0x202f0ccf, 0x1fad37ce, 0xf04b5693, 0xeee57cd9, 0x5ef70007, 0x018e8a4f, +0xfa61c9a9, 0x09989fcf, 0xe66b558b, 0x966efd48, 0x7525021d, 0xe7978b5e, +0x7eb1d6dc, 0xa10c5e5b, 0xb7815e69, 0x7d486cfb, 0xcffdeb2a, 0x7375cb32, +0x599008dc, 0xff91c796, 0x560ed4ad, 0x14e9a876, 0xfccf6a66, 0xa58be028, +0xea9d408b, 0x3afc373b, 0xee008458, 0x19b6042e, 0x84806314, 0x431a4ba4, +0x009ad6a1, 0xd7c62bf4, 0x1bebecba, 0x5c662f69, 0x83bfdea1, 0x45872a9a, +0xca4700c9, 0x47dda29c, 0x57841aff, 0x59fb6aa5, 0xec649574, 0xf7e6d293, +0xa6e6d40e, 0x364f91a8, 0x2c577646, 0xfd0d25d5, 0x58c7694d, 0x864930d4, +0x78999124, 0xc231f2f7, 0xe4cfd0fc, 0x975038c8, 0x9398d150, 0x504b1498, +0x52957004, 0xc172025e, 0x459a8af8, 0x2fecb2af, 0xb3a176ac, 0xbac24f0e, +0xbddadef6, 0x4565426f, 0xc1019842, 0xa4efaf83, 0x2f2966be, 0xdae82bff, +0xc27f135e, 0x94c3b409, 0x3b810ea0, 0x1cc39dc5, 0x731c8a7f, 0x34b6ee25, +0xb78b808d, 0x80eef85f, 0x76a9fb29, 0xbfacb97b, 0xf7a0ea37, 0xe3e69d80, +0xa19b8bd1, 0xf26246cd, 0x5495cf9a, 0x45ad1bf5, 0x2cb3296f, 0x3cb7e6fe, +0xbbde4ac3, 0x310c7b67, 0xe6b00a32, 0x4004d280, 0x0eb8b6dd, 0x8d951f0e, +0x96cb406f, 0xbe29ddb1, 0x38a3756d, 0xf5b058a1, 0xea638fd4, 0x438a7a69, +0x530eb350, 0x7cb218e5, 0xa67b9d52, 0xfb86dfbf, 0x4ed65b7e, 0x00678272, +0x9924a438, 0x9955c26a, 0xb8262774, 0x4ceecb5c, 0xeb103da7, 0xa1297389, +0x0613c489, 0xdf96ae66, 0x8afcce8b, 0x023081f0, 0xc6786ca4, 0x26bdf079, +0x1f29a180, 0xd8d3ec62, 0xe5d537a8, 0x83c99f4e, 0x69d0a8da, 0x027f9a9a, +0x9d9cc9ae, 0x0e2be163, 0xd78cfbe8, 0x2fabb789, 0x48c13886, 0xe35f29b2, +0xb74cc63f, 0x741dcff8, 0x23648527, 0x225155f9, 0xd40ee605, 0x0e4a954e, +0x890ee0b3, 0x97eb0d7d, 0x8776520a, 0xe6fa67e7, 0xa5961e3d, 0xdf2af5a3, +0x7dcbbe1b, 0x41f1af5a, 0xb5845b16, 0x5761f24d, 0xa1431e73, 0x1e00b132, +0x5ddeb7ab, 0x80f29c49, 0xd83714b4, 0xa0412947, 0x96062e8b, 0x7e7f7f8c, +0xdfba2365, 0x4ee1bc30, 0xfe10b1fd, 0x17d2db9f, 0xf0bd8dc9, 0x6c35d718, +0x8962224b, 0xe0f6c654, 0x813f4a7c, 0xc52b4bb7, 0x65da3a71, 0xef9b4eac, +0xf9657e9a, 0xba196585, 0xe3926762, 0xf66fb2a1, 0x433b9440, 0x1df33310, +0xc6aa2685, 0xe19368f0, 0x706636e0, 0xde1e8c5a, 0xbc735af5, 0x67ef395e, +0x6c1fb709, 0x3f7f16f7, 0xf2c4ebe5, 0x7dda3d51, 0x518b4a04, 0xd90dd373, +0xe51fc13a, 0xf01060ae, 0xd1502c78, 0x0d549998, 0x19453c99, 0x52104c7a, +0xc477d443, 0xce6910c0, 0x0dc163b4, 0x1dc4dab2, 0x5c4dbc0c, 0x6145ced9, +0xf02e295b, 0x60196b15, 0x2aa68b89, 0x9ff0dd3b, 0xe4b874ce, 0x9175ec6c, +0x9b629234, 0x644ce81d, 0x359fac36, 0x5211675f, 0xd24c092f, 0x1de9385e, +0x1352b7e5, 0x8d650e83, 0x7057d366, 0x43371dd2, 0x61678e6e, 0xd436279b, +0x267562b6, 0xabaf1706, 0x814bd74e, 0x3269cbda, 0x0a34b3cd, 0xa74c3d1a, +0xd3b098c8, 0x02030412, 0xd75d7207, 0x519d1b3d, 0x1958436b, 0x69ba4221, +0x81b6b4cf, 0xb83234e4, 0x7e652d03, 0x63bcf36e, 0xefecb5f7, 0x60550e08, +0x394963ce, 0xfd6f2b38, 0x1342c68b, 0xbc39f1ca, 0x21bdd863, 0x59ade0af, +0x2d0c793a, 0xa74702d4, 0xc00885f9, 0xc73c27af, 0x6566ea9e, 0xa31e0f7c, +0x499f1706, 0xe19617ad, 0x0e19900a, 0x0a8d3669, 0xcc482af9, 0x5eb35096, +0x269b51c1, 0x80d1145b, 0x4be232ff, 0x3d31fc83, 0x89127a6c, 0x90af3379, +0x7726d002, 0x35f15151, 0x8393c3b5, 0x27a9ada8, 0x25940510, 0x05c49bf1, +0xc7c1b886, 0xdb00826f, 0xf658f61f, 0xe5d77d98, 0xa637b6fb, 0x2f515fa5, +0xb1f80c38, 0xe082d248, 0x4220acd0, 0x06360060, 0xcf42c277, 0xf5972529, +0xf7e274fb, 0xfe41cc28, 0xde661de0, 0xa157bd26, 0x8e1f4788, 0x35c4111a, +0x11a7360e, 0x751188e8, 0x544d9fc3, 0x33d853cd, 0x754542f7, 0x05d9979b, +0x73e59071, 0x4909bb7e, 0x64ae94be, 0x0eb4c8e9, 0xacb903fe, 0xc716288e, +0x1e914aa8, 0x19d127bd, 0x913a6dd4, 0x6af354d9, 0x72c29a95, 0x2fa731a5, +0x9f206402, 0xa44abe92, 0xeb090ab4, 0x85b7584c, 0xf27cd398, 0x3b828e38, +0x0cb8dd56, 0x37657f3c, 0x78fa0f2a, 0x3df7a0cb, 0x2621740f, 0x4d92422f, +0x334a8fa2, 0x124f947d, 0x31be3505, 0xf6a1e561, 0x5d0c4087, 0x62971b45, +0x312472a7, 0x933af4e3, 0x45c28196, 0x8ddd4f00, 0xa5ec20fb, 0x9acde751, +0x23ac64e8, 0xbea00461, 0xf9d65eac, 0xe21db306, 0xcacb4f76, 0x83950ca3, +0x66069329, 0xf7d72838, 0x5d13a747, 0x5c9ca583, 0x1d9d225e, 0xb2c705fe, +0xd0fac625, 0x0a73b38e, 0xec8d6692, 0x0bb64587, 0xf147c00c, 0xcdd6020c, +0x1608cede, 0xb531a423, 0x1c8b1b08, 0xdd74f03e, 0x6d3076fb, 0x2dcaaa44, +0xe869582b, 0x0f19cde9, 0x44d15927, 0xee751f17, 0x655f24c4, 0x7508164e, +0xab414b0d, 0x381d3525, 0x7a18cb74, 0xc91e435a, 0xb3397aa0, 0xa5567595, +0x1da36e52, 0xb43ba598, 0x3ef0d4fd, 0x11d348c4, 0xfcb4cbfa, 0xa3dee1a7, +0x8c6be341, 0xefafff59, 0xeaa95a58, 0x01ba74bb, 0x5b1d3193, 0x2ac942bb, +0xbb5816ca, 0xe79720dd, 0x1683bbad, 0xea2f3992, 0xa89cdd39, 0x3a389386, +0xc54d671f, 0xe8564365, 0x373196e0, 0xc08acfd2, 0x32f556b2, 0x15340220, +0x817d0d1b, 0x4b9afaa3, 0xd3bbf932, 0x6c9ddf75, 0x0fca9bea, 0x2cd2d913, +0x026e3647, 0x6e769b35, 0xdab34e6e, 0xc9b75a74, 0xfdc90304, 0xc6442959, +0xa8f44fb4, 0x73fb26de, 0x833d2d60, 0x8cf7a461, 0xdc5c4bf9, 0x1bce371f, +0xd1d6b743, 0xc1124cae, 0xf4d33161, 0x696956a5, 0xa5ce9c72, 0x5f84e109, +0xa0dae0ff, 0xdfdf169e, 0xb734307c, 0x25843b5d, 0x0b710ae4, 0xafd25b2b, +0xc013b89b, 0x5246e064, 0xeb28ab92, 0x4f92747c, 0x2f3c0c8b, 0x3268720d, +0xef638533, 0xd0fc40ab, 0x5d29c943, 0x3fd9b311, 0x6833bf43, 0xa188ad03, +0xacba3ad2, 0x9696f4da, 0x07b1b2fa, 0xb8f925f6, 0x082573f4, 0x506b5c0e, +0xe290e707, 0x80933f06, 0x38dcad25, 0x276ab82f, 0x788f3a2c, 0x4b0e14bc, +0xc1b38b43, 0x962a0efe, 0x77f19522, 0xd5bd951b, 0x90415ed2, 0x67a6a806, +0x82d0503f, 0x814e505f, 0x6448341a, 0x2c88ba72, 0x1f783411, 0x5dcfc5d9, +0x15cea1e2, 0x351ebdaa, 0xff083e5e, 0x173297d6, 0xbdadb9f2, 0xe82ebe50, +0x33fff936, 0xaed0e402, 0xcd08e297, 0xeefee410, 0x023aedc0, 0x493fdd4c, +0xd4454937, 0xa4c21893, 0xf9740292, 0x14308130, 0x2aa05568, 0xbd88a714, +0x3f264976, 0x62203300, 0x5d9aa0bf, 0x3b9be4b0, 0xcb1c6dff, 0xe7f5ded8, +0xb6ce0ec7, 0xdc1f094b, 0x0f93747c, 0x1768e49a, 0x1d7ebe93, 0x25d53887, +0x144a1fe2, 0x32bc280c, 0x1d0b7884, 0x106b8928, 0x5aa38780, 0x87ca0d93, +0x11a81f71, 0xe0957877, 0xb21fb0f7, 0x1e5a7ac0, 0x09db53a1, 0x5210dd59, +0xa0566364 + +ea = +14412 + +eb = +14412 + +cab = +4 + +c = +4 + +c_neg = +0 + +k_pos = +4800 + +k_neg = +4736 + +ncb_pos = +14496 + +ncb_neg = +14304 + +r = +2 + +rv_index = +0 + +code_block_mode = +0 + +op_flags = +RTE_BBDEV_TURBO_CRC_24B_ATTACH + +expected_status = +OK \ No newline at end of file diff --git a/dpdk/app/test-bbdev/turbo_dec_default.data b/dpdk/app/test-bbdev/turbo_dec_default.data new file mode 120000 index 00000000..371cbc69 --- /dev/null +++ b/dpdk/app/test-bbdev/turbo_dec_default.data @@ -0,0 +1 @@ +test_vectors/turbo_dec_c1_k6144_r0_e10376_crc24b_sbd_negllr_high_snr.data \ No newline at end of file diff --git a/dpdk/app/test-bbdev/turbo_enc_default.data b/dpdk/app/test-bbdev/turbo_enc_default.data new file mode 120000 index 00000000..5587f9cc --- /dev/null +++ b/dpdk/app/test-bbdev/turbo_enc_default.data @@ -0,0 +1 @@ +test_vectors/turbo_enc_c1_k6144_r0_e32256_crc24b_rm.data \ No newline at end of file diff --git a/dpdk/app/test-crypto-perf/Makefile b/dpdk/app/test-crypto-perf/Makefile index c75d7ed1..78135f38 100644 --- a/dpdk/app/test-crypto-perf/Makefile +++ b/dpdk/app/test-crypto-perf/Makefile @@ -1,32 +1,5 @@ -# BSD LICENSE -# -# Copyright(c) 2016-2017 Intel Corporation. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2016-2017 Intel Corporation include $(RTE_SDK)/mk/rte.vars.mk @@ -34,6 +7,8 @@ include $(RTE_SDK)/mk/rte.vars.mk APP = dpdk-test-crypto-perf CFLAGS += $(WERROR_FLAGS) +CFLAGS += -DALLOW_EXPERIMENTAL_API +CFLAGS += -O3 # all source are stored in SRCS-y SRCS-y := main.c diff --git a/dpdk/app/test-crypto-perf/cperf.h b/dpdk/app/test-crypto-perf/cperf.h index c9f7f817..db58228d 100644 --- a/dpdk/app/test-crypto-perf/cperf.h +++ b/dpdk/app/test-crypto-perf/cperf.h @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #ifndef _CPERF_ diff --git a/dpdk/app/test-crypto-perf/cperf_ops.c b/dpdk/app/test-crypto-perf/cperf_ops.c index 0fa1c412..44808f50 100644 --- a/dpdk/app/test-crypto-perf/cperf_ops.c +++ b/dpdk/app/test-crypto-perf/cperf_ops.c @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #include @@ -41,7 +13,7 @@ cperf_set_ops_null_cipher(struct rte_crypto_op **ops, uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, const struct cperf_options *options, const struct cperf_test_vector *test_vector __rte_unused, - uint16_t iv_offset __rte_unused) + uint16_t iv_offset __rte_unused, uint32_t *imix_idx) { uint16_t i; @@ -62,7 +34,12 @@ cperf_set_ops_null_cipher(struct rte_crypto_op **ops, dst_buf_offset); /* cipher parameters */ - sym_op->cipher.data.length = options->test_buffer_size; + if (options->imix_distribution_count) { + sym_op->cipher.data.length = + options->imix_buffer_sizes[*imix_idx]; + *imix_idx = (*imix_idx + 1) % options->pool_sz; + } else + sym_op->cipher.data.length = options->test_buffer_size; sym_op->cipher.data.offset = 0; } @@ -75,7 +52,7 @@ cperf_set_ops_null_auth(struct rte_crypto_op **ops, uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, const struct cperf_options *options, const struct cperf_test_vector *test_vector __rte_unused, - uint16_t iv_offset __rte_unused) + uint16_t iv_offset __rte_unused, uint32_t *imix_idx) { uint16_t i; @@ -96,7 +73,12 @@ cperf_set_ops_null_auth(struct rte_crypto_op **ops, dst_buf_offset); /* auth parameters */ - sym_op->auth.data.length = options->test_buffer_size; + if (options->imix_distribution_count) { + sym_op->auth.data.length = + options->imix_buffer_sizes[*imix_idx]; + *imix_idx = (*imix_idx + 1) % options->pool_sz; + } else + sym_op->auth.data.length = options->test_buffer_size; sym_op->auth.data.offset = 0; } @@ -109,7 +91,7 @@ cperf_set_ops_cipher(struct rte_crypto_op **ops, uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, const struct cperf_options *options, const struct cperf_test_vector *test_vector, - uint16_t iv_offset) + uint16_t iv_offset, uint32_t *imix_idx) { uint16_t i; @@ -130,12 +112,17 @@ cperf_set_ops_cipher(struct rte_crypto_op **ops, dst_buf_offset); /* cipher parameters */ + if (options->imix_distribution_count) { + sym_op->cipher.data.length = + options->imix_buffer_sizes[*imix_idx]; + *imix_idx = (*imix_idx + 1) % options->pool_sz; + } else + sym_op->cipher.data.length = options->test_buffer_size; + if (options->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || options->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 || options->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3) - sym_op->cipher.data.length = options->test_buffer_size << 3; - else - sym_op->cipher.data.length = options->test_buffer_size; + sym_op->cipher.data.length <<= 3; sym_op->cipher.data.offset = 0; } @@ -160,7 +147,7 @@ cperf_set_ops_auth(struct rte_crypto_op **ops, uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, const struct cperf_options *options, const struct cperf_test_vector *test_vector, - uint16_t iv_offset) + uint16_t iv_offset, uint32_t *imix_idx) { uint16_t i; @@ -225,12 +212,17 @@ cperf_set_ops_auth(struct rte_crypto_op **ops, } + if (options->imix_distribution_count) { + sym_op->auth.data.length = + options->imix_buffer_sizes[*imix_idx]; + *imix_idx = (*imix_idx + 1) % options->pool_sz; + } else + sym_op->auth.data.length = options->test_buffer_size; + if (options->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || options->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9 || options->auth_algo == RTE_CRYPTO_AUTH_ZUC_EIA3) - sym_op->auth.data.length = options->test_buffer_size << 3; - else - sym_op->auth.data.length = options->test_buffer_size; + sym_op->auth.data.length <<= 3; sym_op->auth.data.offset = 0; } @@ -255,7 +247,7 @@ cperf_set_ops_cipher_auth(struct rte_crypto_op **ops, uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, const struct cperf_options *options, const struct cperf_test_vector *test_vector, - uint16_t iv_offset) + uint16_t iv_offset, uint32_t *imix_idx) { uint16_t i; @@ -276,12 +268,17 @@ cperf_set_ops_cipher_auth(struct rte_crypto_op **ops, dst_buf_offset); /* cipher parameters */ + if (options->imix_distribution_count) { + sym_op->cipher.data.length = + options->imix_buffer_sizes[*imix_idx]; + *imix_idx = (*imix_idx + 1) % options->pool_sz; + } else + sym_op->cipher.data.length = options->test_buffer_size; + if (options->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 || options->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 || options->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3) - sym_op->cipher.data.length = options->test_buffer_size << 3; - else - sym_op->cipher.data.length = options->test_buffer_size; + sym_op->cipher.data.length <<= 3; sym_op->cipher.data.offset = 0; @@ -321,12 +318,17 @@ cperf_set_ops_cipher_auth(struct rte_crypto_op **ops, rte_pktmbuf_iova_offset(buf, offset); } + if (options->imix_distribution_count) { + sym_op->auth.data.length = + options->imix_buffer_sizes[*imix_idx]; + *imix_idx = (*imix_idx + 1) % options->pool_sz; + } else + sym_op->auth.data.length = options->test_buffer_size; + if (options->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 || options->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9 || options->auth_algo == RTE_CRYPTO_AUTH_ZUC_EIA3) - sym_op->auth.data.length = options->test_buffer_size << 3; - else - sym_op->auth.data.length = options->test_buffer_size; + sym_op->auth.data.length <<= 3; sym_op->auth.data.offset = 0; } @@ -360,7 +362,7 @@ cperf_set_ops_aead(struct rte_crypto_op **ops, uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, const struct cperf_options *options, const struct cperf_test_vector *test_vector, - uint16_t iv_offset) + uint16_t iv_offset, uint32_t *imix_idx) { uint16_t i; /* AAD is placed after the IV */ @@ -384,7 +386,12 @@ cperf_set_ops_aead(struct rte_crypto_op **ops, dst_buf_offset); /* AEAD parameters */ - sym_op->aead.data.length = options->test_buffer_size; + if (options->imix_distribution_count) { + sym_op->aead.data.length = + options->imix_buffer_sizes[*imix_idx]; + *imix_idx = (*imix_idx + 1) % options->pool_sz; + } else + sym_op->aead.data.length = options->test_buffer_size; sym_op->aead.data.offset = 0; sym_op->aead.aad.data = rte_crypto_op_ctod_offset(ops[i], diff --git a/dpdk/app/test-crypto-perf/cperf_ops.h b/dpdk/app/test-crypto-perf/cperf_ops.h index 94951cc3..29e109f2 100644 --- a/dpdk/app/test-crypto-perf/cperf_ops.h +++ b/dpdk/app/test-crypto-perf/cperf_ops.h @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #ifndef _CPERF_OPS_ @@ -51,7 +23,7 @@ typedef int (*cperf_populate_ops_t)(struct rte_crypto_op **ops, uint16_t nb_ops, struct rte_cryptodev_sym_session *sess, const struct cperf_options *options, const struct cperf_test_vector *test_vector, - uint16_t iv_offset); + uint16_t iv_offset, uint32_t *imix_idx); struct cperf_op_fns { cperf_sessions_create_t sess_create; diff --git a/dpdk/app/test-crypto-perf/cperf_options.h b/dpdk/app/test-crypto-perf/cperf_options.h index da4fb47c..f5bf03c8 100644 --- a/dpdk/app/test-crypto-perf/cperf_options.h +++ b/dpdk/app/test-crypto-perf/cperf_options.h @@ -1,8 +1,12 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ #ifndef _CPERF_OPTIONS_ #define _CPERF_OPTIONS_ #include +#include #define CPERF_PTEST_TYPE ("ptest") #define CPERF_SILENT ("silent") @@ -13,6 +17,7 @@ #define CPERF_BUFFER_SIZE ("buffer-sz") #define CPERF_SEGMENT_SIZE ("segment-sz") #define CPERF_DESC_NB ("desc-nb") +#define CPERF_IMIX ("imix") #define CPERF_DEVTYPE ("devtype") #define CPERF_OPTYPE ("optype") @@ -71,8 +76,11 @@ struct cperf_options { uint32_t pool_sz; uint32_t total_ops; + uint32_t headroom_sz; + uint32_t tailroom_sz; uint32_t segment_sz; uint32_t test_buffer_size; + uint32_t *imix_buffer_sizes; uint32_t nb_descriptors; uint16_t nb_qps; @@ -102,7 +110,7 @@ struct cperf_options { uint16_t digest_sz; - char device_type[RTE_CRYPTODEV_NAME_LEN]; + char device_type[RTE_CRYPTODEV_NAME_MAX_LEN]; enum cperf_op_type op_type; char *test_file; @@ -122,6 +130,8 @@ struct cperf_options { /* pmd-cyclecount specific options */ uint32_t pmdcc_delay; + uint32_t imix_distribution_list[MAX_LIST]; + uint8_t imix_distribution_count; }; void diff --git a/dpdk/app/test-crypto-perf/cperf_options_parsing.c b/dpdk/app/test-crypto-perf/cperf_options_parsing.c index ad43e84c..7a5aa06a 100644 --- a/dpdk/app/test-crypto-perf/cperf_options_parsing.c +++ b/dpdk/app/test-crypto-perf/cperf_options_parsing.c @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #include @@ -57,6 +29,7 @@ usage(char *progname) " --total-ops N: set the number of total operations performed\n" " --burst-sz N: set the number of packets per burst\n" " --buffer-sz N: set the size of a single packet\n" + " --imix N: set the distribution of packet sizes\n" " --segment-sz N: set the size of the segment to use\n" " --desc-nb N: set number of descriptors for each crypto device\n" " --devtype TYPE: set crypto device type to use\n" @@ -243,6 +216,8 @@ parse_list(const char *arg, uint32_t *list, uint32_t *min, uint32_t *max) char *token; uint32_t number; uint8_t count = 0; + uint32_t temp_min; + uint32_t temp_max; char *copy_arg = strdup(arg); @@ -261,8 +236,8 @@ parse_list(const char *arg, uint32_t *list, uint32_t *min, uint32_t *max) goto err_list; list[count++] = number; - *min = number; - *max = number; + temp_min = number; + temp_max = number; } else goto err_list; @@ -283,14 +258,19 @@ parse_list(const char *arg, uint32_t *list, uint32_t *min, uint32_t *max) list[count++] = number; - if (number < *min) - *min = number; - if (number > *max) - *max = number; + if (number < temp_min) + temp_min = number; + if (number > temp_max) + temp_max = number; token = strtok(NULL, ","); } + if (min) + *min = temp_min; + if (max) + *max = temp_max; + free(copy_arg); return count; @@ -386,6 +366,29 @@ parse_segment_sz(struct cperf_options *opts, const char *arg) return 0; } +static int +parse_imix(struct cperf_options *opts, const char *arg) +{ + int ret; + + ret = parse_list(arg, opts->imix_distribution_list, + NULL, NULL); + if (ret < 0) { + RTE_LOG(ERR, USER1, "failed to parse imix distribution\n"); + return -1; + } + + opts->imix_distribution_count = ret; + + if (opts->imix_distribution_count <= 1) { + RTE_LOG(ERR, USER1, "imix distribution should have " + "at least two entries\n"); + return -1; + } + + return 0; +} + static int parse_desc_nb(struct cperf_options *opts, const char *arg) { @@ -722,6 +725,7 @@ static struct option lgopts[] = { { CPERF_SEGMENT_SIZE, required_argument, 0, 0 }, { CPERF_DESC_NB, required_argument, 0, 0 }, + { CPERF_IMIX, required_argument, 0, 0 }, { CPERF_DEVTYPE, required_argument, 0, 0 }, { CPERF_OPTYPE, required_argument, 0, 0 }, @@ -786,6 +790,7 @@ cperf_options_default(struct cperf_options *opts) */ opts->segment_sz = 0; + opts->imix_distribution_count = 0; strncpy(opts->device_type, "crypto_aesni_mb", sizeof(opts->device_type)); opts->nb_qps = 1; @@ -835,6 +840,7 @@ cperf_opts_parse_long(int opt_idx, struct cperf_options *opts) { CPERF_OPTYPE, parse_op_type }, { CPERF_SESSIONLESS, parse_sessionless }, { CPERF_OUT_OF_PLACE, parse_out_of_place }, + { CPERF_IMIX, parse_imix }, { CPERF_TEST_FILE, parse_test_file }, { CPERF_TEST_NAME, parse_test_name }, { CPERF_CIPHER_ALGO, parse_cipher_algo }, @@ -973,6 +979,14 @@ cperf_options_check(struct cperf_options *options) return -EINVAL; } + if ((options->imix_distribution_count != 0) && + (options->imix_distribution_count != + options->buffer_size_count)) { + RTE_LOG(ERR, USER1, "IMIX distribution must have the same " + "number of buffer sizes\n"); + return -EINVAL; + } + if (options->test == CPERF_TEST_TYPE_VERIFY && options->test_file == NULL) { RTE_LOG(ERR, USER1, "Define path to the file with test" @@ -1025,6 +1039,13 @@ cperf_options_check(struct cperf_options *options) return -EINVAL; } + if (options->test == CPERF_TEST_TYPE_VERIFY && + options->imix_distribution_count > 0) { + RTE_LOG(ERR, USER1, "IMIX is not allowed when " + "using the verify test.\n"); + return -EINVAL; + } + if (options->op_type == CPERF_CIPHER_THEN_AUTH) { if (options->cipher_op != RTE_CRYPTO_CIPHER_OP_ENCRYPT && options->auth_op != diff --git a/dpdk/app/test-crypto-perf/cperf_test_common.c b/dpdk/app/test-crypto-perf/cperf_test_common.c index 9b08b3f7..e803dc10 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_common.c +++ b/dpdk/app/test-crypto-perf/cperf_test_common.c @@ -1,36 +1,9 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation */ #include +#include #include "cperf_test_common.h" @@ -38,12 +11,15 @@ struct obj_params { uint32_t src_buf_offset; uint32_t dst_buf_offset; uint16_t segment_sz; + uint16_t headroom_sz; + uint16_t data_len; uint16_t segments_nb; }; static void fill_single_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp, - void *obj, uint32_t mbuf_offset, uint16_t segment_sz) + void *obj, uint32_t mbuf_offset, uint16_t segment_sz, + uint16_t headroom, uint16_t data_len) { uint32_t mbuf_hdr_size = sizeof(struct rte_mbuf); @@ -53,10 +29,10 @@ fill_single_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp, m->buf_iova = rte_mempool_virt2iova(obj) + mbuf_offset + mbuf_hdr_size; m->buf_len = segment_sz; - m->data_len = segment_sz; + m->data_len = data_len; - /* No headroom needed for the buffer */ - m->data_off = 0; + /* Use headroom specified for the buffer */ + m->data_off = headroom; /* init some constant fields */ m->pool = mp; @@ -69,7 +45,7 @@ fill_single_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp, static void fill_multi_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp, void *obj, uint32_t mbuf_offset, uint16_t segment_sz, - uint16_t segments_nb) + uint16_t headroom, uint16_t data_len, uint16_t segments_nb) { uint16_t mbuf_hdr_size = sizeof(struct rte_mbuf); uint16_t remaining_segments = segments_nb; @@ -84,10 +60,10 @@ fill_multi_seg_mbuf(struct rte_mbuf *m, struct rte_mempool *mp, m->buf_iova = next_seg_phys_addr; next_seg_phys_addr += mbuf_hdr_size + segment_sz; m->buf_len = segment_sz; - m->data_len = segment_sz; + m->data_len = data_len; - /* No headroom needed for the buffer */ - m->data_off = 0; + /* Use headroom specified for the buffer */ + m->data_off = headroom; /* init some constant fields */ m->pool = mp; @@ -126,10 +102,12 @@ mempool_obj_init(struct rte_mempool *mp, op->sym->m_src = m; if (params->segments_nb == 1) fill_single_seg_mbuf(m, mp, obj, params->src_buf_offset, - params->segment_sz); + params->segment_sz, params->headroom_sz, + params->data_len); else fill_multi_seg_mbuf(m, mp, obj, params->src_buf_offset, - params->segment_sz, params->segments_nb); + params->segment_sz, params->headroom_sz, + params->data_len, params->segments_nb); /* Set destination buffer */ @@ -137,7 +115,8 @@ mempool_obj_init(struct rte_mempool *mp, m = (struct rte_mbuf *) ((uint8_t *) obj + params->dst_buf_offset); fill_single_seg_mbuf(m, mp, obj, params->dst_buf_offset, - params->segment_sz); + params->segment_sz, params->headroom_sz, + params->data_len); op->sym->m_dst = m; } else op->sym->m_dst = NULL; @@ -152,6 +131,7 @@ cperf_alloc_common_memory(const struct cperf_options *options, uint32_t *dst_buf_offset, struct rte_mempool **pool) { + const char *mp_ops_name; char pool_name[32] = ""; int ret; @@ -198,6 +178,11 @@ cperf_alloc_common_memory(const struct cperf_options *options, struct obj_params params = { .segment_sz = options->segment_sz, + .headroom_sz = options->headroom_sz, + /* Data len = segment size - (headroom + tailroom) */ + .data_len = options->segment_sz - + options->headroom_sz - + options->tailroom_sz, .segments_nb = segments_nb, .src_buf_offset = crypto_op_total_size_padded, .dst_buf_offset = 0 @@ -221,8 +206,10 @@ cperf_alloc_common_memory(const struct cperf_options *options, return -1; } + mp_ops_name = rte_mbuf_best_mempool_ops(); + ret = rte_mempool_set_ops_byname(*pool, - RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL); + mp_ops_name, NULL); if (ret != 0) { RTE_LOG(ERR, USER1, "Error setting mempool handler for device %u\n", diff --git a/dpdk/app/test-crypto-perf/cperf_test_common.h b/dpdk/app/test-crypto-perf/cperf_test_common.h index 4cee7852..3ace0d2e 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_common.h +++ b/dpdk/app/test-crypto-perf/cperf_test_common.h @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation */ #ifndef _CPERF_TEST_COMMON_H_ diff --git a/dpdk/app/test-crypto-perf/cperf_test_latency.c b/dpdk/app/test-crypto-perf/cperf_test_latency.c index ca2a4ba6..c9c98dc5 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_latency.c +++ b/dpdk/app/test-crypto-perf/cperf_test_latency.c @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #include @@ -154,6 +126,7 @@ cperf_latency_test_runner(void *arg) struct cperf_latency_ctx *ctx = arg; uint16_t test_burst_size; uint8_t burst_size_idx = 0; + uint32_t imix_idx = 0; static int only_once; @@ -218,7 +191,7 @@ cperf_latency_test_runner(void *arg) burst_size) != 0) { RTE_LOG(ERR, USER1, "Failed to allocate more crypto operations " - "from the the crypto operation pool.\n" + "from the crypto operation pool.\n" "Consider increasing the pool size " "with --pool-sz\n"); return -1; @@ -228,7 +201,8 @@ cperf_latency_test_runner(void *arg) (ctx->populate_ops)(ops, ctx->src_buf_offset, ctx->dst_buf_offset, burst_size, ctx->sess, ctx->options, - ctx->test_vector, iv_offset); + ctx->test_vector, iv_offset, + &imix_idx); tsc_start = rte_rdtsc_precise(); diff --git a/dpdk/app/test-crypto-perf/cperf_test_latency.h b/dpdk/app/test-crypto-perf/cperf_test_latency.h index 1bbedb4e..d3fc3218 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_latency.h +++ b/dpdk/app/test-crypto-perf/cperf_test_latency.h @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #ifndef _CPERF_LATENCY_ diff --git a/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.c b/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.c index 9b41724a..c8d16db6 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.c +++ b/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.c @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation */ #include @@ -169,10 +141,11 @@ pmd_cyclecount_bench_ops(struct pmd_cyclecount_state *state, uint32_t cur_op, uint32_t iter_ops_needed = RTE_MIN(state->opts->nb_descriptors, iter_ops_left); uint32_t cur_iter_op; + uint32_t imix_idx = 0; for (cur_iter_op = 0; cur_iter_op < iter_ops_needed; cur_iter_op += test_burst_size) { - uint32_t burst_size = RTE_MIN(state->opts->total_ops - cur_op, + uint32_t burst_size = RTE_MIN(iter_ops_needed - cur_iter_op, test_burst_size); struct rte_crypto_op **ops = &state->ctx->ops[cur_iter_op]; @@ -181,7 +154,7 @@ pmd_cyclecount_bench_ops(struct pmd_cyclecount_state *state, uint32_t cur_op, burst_size) != 0) { RTE_LOG(ERR, USER1, "Failed to allocate more crypto operations " - "from the the crypto operation pool.\n" + "from the crypto operation pool.\n" "Consider increasing the pool size " "with --pool-sz\n"); return -1; @@ -193,7 +166,8 @@ pmd_cyclecount_bench_ops(struct pmd_cyclecount_state *state, uint32_t cur_op, state->ctx->dst_buf_offset, burst_size, state->ctx->sess, state->opts, - state->ctx->test_vector, iv_offset); + state->ctx->test_vector, iv_offset, + &imix_idx); #ifdef CPERF_LINEARIZATION_ENABLE /* Check if source mbufs require coalescing */ @@ -218,6 +192,7 @@ pmd_cyclecount_build_ops(struct pmd_cyclecount_state *state, uint32_t iter_ops_needed, uint16_t test_burst_size) { uint32_t cur_iter_op; + uint32_t imix_idx = 0; for (cur_iter_op = 0; cur_iter_op < iter_ops_needed; cur_iter_op += test_burst_size) { @@ -230,7 +205,7 @@ pmd_cyclecount_build_ops(struct pmd_cyclecount_state *state, burst_size) != 0) { RTE_LOG(ERR, USER1, "Failed to allocate more crypto operations " - "from the the crypto operation pool.\n" + "from the crypto operation pool.\n" "Consider increasing the pool size " "with --pool-sz\n"); return -1; @@ -242,7 +217,8 @@ pmd_cyclecount_build_ops(struct pmd_cyclecount_state *state, state->ctx->dst_buf_offset, burst_size, state->ctx->sess, state->opts, - state->ctx->test_vector, iv_offset); + state->ctx->test_vector, iv_offset, + &imix_idx); } return 0; } diff --git a/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.h b/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.h index 93f0eae0..beb44199 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.h +++ b/dpdk/app/test-crypto-perf/cperf_test_pmd_cyclecount.h @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Intel Corporation */ #ifndef _CPERF_TEST_PMD_CYCLECOUNT_H_ diff --git a/dpdk/app/test-crypto-perf/cperf_test_throughput.c b/dpdk/app/test-crypto-perf/cperf_test_throughput.c index b84dc630..8766d6e9 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_throughput.c +++ b/dpdk/app/test-crypto-perf/cperf_test_throughput.c @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #include @@ -120,6 +92,7 @@ cperf_throughput_test_runner(void *test_ctx) struct cperf_throughput_ctx *ctx = test_ctx; uint16_t test_burst_size; uint8_t burst_size_idx = 0; + uint32_t imix_idx = 0; static int only_once; @@ -182,7 +155,7 @@ cperf_throughput_test_runner(void *test_ctx) ops_needed) != 0) { RTE_LOG(ERR, USER1, "Failed to allocate more crypto operations " - "from the the crypto operation pool.\n" + "from the crypto operation pool.\n" "Consider increasing the pool size " "with --pool-sz\n"); return -1; @@ -193,7 +166,7 @@ cperf_throughput_test_runner(void *test_ctx) ctx->dst_buf_offset, ops_needed, ctx->sess, ctx->options, ctx->test_vector, - iv_offset); + iv_offset, &imix_idx); /** * When ops_needed is smaller than ops_enqd, the diff --git a/dpdk/app/test-crypto-perf/cperf_test_throughput.h b/dpdk/app/test-crypto-perf/cperf_test_throughput.h index 987d0c31..439ec8e5 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_throughput.h +++ b/dpdk/app/test-crypto-perf/cperf_test_throughput.h @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #ifndef _CPERF_THROUGHPUT_ diff --git a/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c index 56df4e78..92932a23 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c +++ b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.c @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #ifdef RTE_EXEC_ENV_BSDAPP #define _WITH_GETLINE diff --git a/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.h b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.h index e3df98bd..247b1422 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.h +++ b/dpdk/app/test-crypto-perf/cperf_test_vector_parsing.h @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #ifndef APP_CRYPTO_PERF_CPERF_TEST_VECTOR_PARSING_H_ diff --git a/dpdk/app/test-crypto-perf/cperf_test_vectors.c b/dpdk/app/test-crypto-perf/cperf_test_vectors.c index fa911ff6..1af95249 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_vectors.c +++ b/dpdk/app/test-crypto-perf/cperf_test_vectors.c @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #include @@ -447,13 +419,19 @@ cperf_test_vector_get_dummy(struct cperf_options *options) t_vec->cipher_key.length = 0; t_vec->ciphertext.data = plaintext; t_vec->cipher_key.data = NULL; - t_vec->cipher_iv.data = NULL; } else { t_vec->cipher_key.length = options->cipher_key_sz; t_vec->ciphertext.data = ciphertext; t_vec->cipher_key.data = cipher_key; - t_vec->cipher_iv.data = rte_malloc(NULL, options->cipher_iv_sz, - 16); + } + + /* Init IV data ptr */ + t_vec->cipher_iv.data = NULL; + + if (options->cipher_iv_sz != 0) { + /* Set IV parameters */ + t_vec->cipher_iv.data = rte_malloc(NULL, + options->cipher_iv_sz, 16); if (t_vec->cipher_iv.data == NULL) { rte_free(t_vec); return NULL; @@ -461,17 +439,7 @@ cperf_test_vector_get_dummy(struct cperf_options *options) memcpy(t_vec->cipher_iv.data, iv, options->cipher_iv_sz); } t_vec->ciphertext.length = options->max_buffer_size; - - /* Set IV parameters */ - t_vec->cipher_iv.data = rte_malloc(NULL, options->cipher_iv_sz, - 16); - if (options->cipher_iv_sz && t_vec->cipher_iv.data == NULL) { - rte_free(t_vec); - return NULL; - } - memcpy(t_vec->cipher_iv.data, iv, options->cipher_iv_sz); t_vec->cipher_iv.length = options->cipher_iv_sz; - t_vec->data.cipher_offset = 0; t_vec->data.cipher_length = options->max_buffer_size; diff --git a/dpdk/app/test-crypto-perf/cperf_test_vectors.h b/dpdk/app/test-crypto-perf/cperf_test_vectors.h index cb5d8284..6f10823e 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_vectors.h +++ b/dpdk/app/test-crypto-perf/cperf_test_vectors.h @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #ifndef _CPERF_TEST_VECTRORS_ diff --git a/dpdk/app/test-crypto-perf/cperf_test_verify.c b/dpdk/app/test-crypto-perf/cperf_test_verify.c index 6945c8b4..9134b921 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_verify.c +++ b/dpdk/app/test-crypto-perf/cperf_test_verify.c @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #include @@ -264,6 +236,7 @@ cperf_verify_test_runner(void *test_ctx) uint64_t i; uint16_t ops_unused = 0; + uint32_t imix_idx = 0; struct rte_crypto_op *ops[ctx->options->max_burst_size]; struct rte_crypto_op *ops_processed[ctx->options->max_burst_size]; @@ -307,7 +280,7 @@ cperf_verify_test_runner(void *test_ctx) ops_needed) != 0) { RTE_LOG(ERR, USER1, "Failed to allocate more crypto operations " - "from the the crypto operation pool.\n" + "from the crypto operation pool.\n" "Consider increasing the pool size " "with --pool-sz\n"); return -1; @@ -317,7 +290,7 @@ cperf_verify_test_runner(void *test_ctx) (ctx->populate_ops)(ops, ctx->src_buf_offset, ctx->dst_buf_offset, ops_needed, ctx->sess, ctx->options, - ctx->test_vector, iv_offset); + ctx->test_vector, iv_offset, &imix_idx); /* Populate the mbuf with the test vector, for verification */ diff --git a/dpdk/app/test-crypto-perf/cperf_test_verify.h b/dpdk/app/test-crypto-perf/cperf_test_verify.h index e67b48d3..9f70ad87 100644 --- a/dpdk/app/test-crypto-perf/cperf_test_verify.h +++ b/dpdk/app/test-crypto-perf/cperf_test_verify.h @@ -1,33 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #ifndef _CPERF_VERIFY_ diff --git a/dpdk/app/test-crypto-perf/main.c b/dpdk/app/test-crypto-perf/main.c index 13e01218..953e058c 100644 --- a/dpdk/app/test-crypto-perf/main.c +++ b/dpdk/app/test-crypto-perf/main.c @@ -1,38 +1,12 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #include #include +#include +#include #include #include #ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER @@ -47,8 +21,6 @@ #include "cperf_test_verify.h" #include "cperf_test_pmd_cyclecount.h" -#define NUM_SESSIONS 2048 -#define SESS_MEMPOOL_CACHE_SIZE 64 const char *cperf_test_type_strs[] = { [CPERF_TEST_TYPE_THROUGHPUT] = "throughput", @@ -93,6 +65,7 @@ cperf_initialize_cryptodev(struct cperf_options *opts, uint8_t *enabled_cdevs, struct rte_mempool *session_pool_socket[]) { uint8_t enabled_cdev_count = 0, nb_lcores, cdev_id; + uint32_t sessions_needed = 0; unsigned int i, j; int ret; @@ -123,7 +96,7 @@ cperf_initialize_cryptodev(struct cperf_options *opts, uint8_t *enabled_cdevs, uint32_t max_sess_size = 0, sess_size; for (cdev_id = 0; cdev_id < rte_cryptodev_count(); cdev_id++) { - sess_size = rte_cryptodev_get_private_session_size(cdev_id); + sess_size = rte_cryptodev_sym_get_private_session_size(cdev_id); if (sess_size > max_sess_size) max_sess_size = sess_size; } @@ -175,17 +148,62 @@ cperf_initialize_cryptodev(struct cperf_options *opts, uint8_t *enabled_cdevs, .nb_descriptors = opts->nb_descriptors }; + /** + * Device info specifies the min headroom and tailroom + * requirement for the crypto PMD. This need to be honoured + * by the application, while creating mbuf. + */ + if (opts->headroom_sz < cdev_info.min_mbuf_headroom_req) { + /* Update headroom */ + opts->headroom_sz = cdev_info.min_mbuf_headroom_req; + } + if (opts->tailroom_sz < cdev_info.min_mbuf_tailroom_req) { + /* Update tailroom */ + opts->tailroom_sz = cdev_info.min_mbuf_tailroom_req; + } + + /* Update segment size to include headroom & tailroom */ + opts->segment_sz += (opts->headroom_sz + opts->tailroom_sz); + + uint32_t dev_max_nb_sess = cdev_info.sym.max_nb_sessions; + /* + * Two sessions objects are required for each session + * (one for the header, one for the private data) + */ + if (!strcmp((const char *)opts->device_type, + "crypto_scheduler")) { +#ifdef RTE_LIBRTE_PMD_CRYPTO_SCHEDULER + uint32_t nb_slaves = + rte_cryptodev_scheduler_slaves_get(cdev_id, + NULL); + + sessions_needed = 2 * enabled_cdev_count * + opts->nb_qps * nb_slaves; +#endif + } else + sessions_needed = 2 * enabled_cdev_count * + opts->nb_qps; + + /* + * A single session is required per queue pair + * in each device + */ + if (dev_max_nb_sess != 0 && dev_max_nb_sess < opts->nb_qps) { + RTE_LOG(ERR, USER1, + "Device does not support at least " + "%u sessions\n", opts->nb_qps); + return -ENOTSUP; + } if (session_pool_socket[socket_id] == NULL) { char mp_name[RTE_MEMPOOL_NAMESIZE]; struct rte_mempool *sess_mp; snprintf(mp_name, RTE_MEMPOOL_NAMESIZE, "sess_mp_%u", socket_id); - sess_mp = rte_mempool_create(mp_name, - NUM_SESSIONS, + sessions_needed, max_sess_size, - SESS_MEMPOOL_CACHE_SIZE, + 0, 0, NULL, NULL, NULL, NULL, socket_id, 0); @@ -324,7 +342,9 @@ cperf_check_test_vector(struct cperf_options *opts, return -1; if (test_vec->ciphertext.length < opts->max_buffer_size) return -1; - if (test_vec->cipher_iv.data == NULL) + /* Cipher IV is only required for some algorithms */ + if (opts->cipher_iv_sz && + test_vec->cipher_iv.data == NULL) return -1; if (test_vec->cipher_iv.length != opts->cipher_iv_sz) return -1; @@ -339,7 +359,9 @@ cperf_check_test_vector(struct cperf_options *opts, return -1; if (test_vec->plaintext.length < opts->max_buffer_size) return -1; - if (test_vec->auth_key.data == NULL) + /* Auth key is only required for some algorithms */ + if (opts->auth_key_sz && + test_vec->auth_key.data == NULL) return -1; if (test_vec->auth_key.length != opts->auth_key_sz) return -1; @@ -403,6 +425,10 @@ cperf_check_test_vector(struct cperf_options *opts, return -1; if (test_vec->ciphertext.length < opts->max_buffer_size) return -1; + if (test_vec->aead_key.data == NULL) + return -1; + if (test_vec->aead_key.length != opts->aead_key_sz) + return -1; if (test_vec->aead_iv.data == NULL) return -1; if (test_vec->aead_iv.length != opts->aead_iv_sz) @@ -542,13 +568,45 @@ main(int argc, char **argv) i++; } - /* Get first size from range or list */ - if (opts.inc_buffer_size != 0) - opts.test_buffer_size = opts.min_buffer_size; - else - opts.test_buffer_size = opts.buffer_size_list[0]; + if (opts.imix_distribution_count != 0) { + uint8_t buffer_size_count = opts.buffer_size_count; + uint16_t distribution_total[buffer_size_count]; + uint32_t op_idx; + uint32_t test_average_size = 0; + const uint32_t *buffer_size_list = opts.buffer_size_list; + const uint32_t *imix_distribution_list = opts.imix_distribution_list; + + opts.imix_buffer_sizes = rte_malloc(NULL, + sizeof(uint32_t) * opts.pool_sz, + 0); + /* + * Calculate accumulated distribution of + * probabilities per packet size + */ + distribution_total[0] = imix_distribution_list[0]; + for (i = 1; i < buffer_size_count; i++) + distribution_total[i] = imix_distribution_list[i] + + distribution_total[i-1]; + + /* Calculate a random sequence of packet sizes, based on distribution */ + for (op_idx = 0; op_idx < opts.pool_sz; op_idx++) { + uint16_t random_number = rte_rand() % + distribution_total[buffer_size_count - 1]; + for (i = 0; i < buffer_size_count; i++) + if (random_number < distribution_total[i]) + break; + + opts.imix_buffer_sizes[op_idx] = buffer_size_list[i]; + } + + /* Calculate average buffer size for the IMIX distribution */ + for (i = 0; i < buffer_size_count; i++) + test_average_size += buffer_size_list[i] * + imix_distribution_list[i]; + + opts.test_buffer_size = test_average_size / + distribution_total[buffer_size_count - 1]; - while (opts.test_buffer_size <= opts.max_buffer_size) { i = 0; RTE_LCORE_FOREACH_SLAVE(lcore_id) { @@ -567,14 +625,43 @@ main(int argc, char **argv) rte_eal_wait_lcore(lcore_id); i++; } + } else { /* Get next size from range or list */ if (opts.inc_buffer_size != 0) - opts.test_buffer_size += opts.inc_buffer_size; - else { - if (++buffer_size_idx == opts.buffer_size_count) - break; - opts.test_buffer_size = opts.buffer_size_list[buffer_size_idx]; + opts.test_buffer_size = opts.min_buffer_size; + else + opts.test_buffer_size = opts.buffer_size_list[0]; + + while (opts.test_buffer_size <= opts.max_buffer_size) { + i = 0; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + + if (i == total_nb_qps) + break; + + rte_eal_remote_launch(cperf_testmap[opts.test].runner, + ctx[i], lcore_id); + i++; + } + i = 0; + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + + if (i == total_nb_qps) + break; + rte_eal_wait_lcore(lcore_id); + i++; + } + + /* Get next size from range or list */ + if (opts.inc_buffer_size != 0) + opts.test_buffer_size += opts.inc_buffer_size; + else { + if (++buffer_size_idx == opts.buffer_size_count) + break; + opts.test_buffer_size = + opts.buffer_size_list[buffer_size_idx]; + } } } @@ -603,8 +690,6 @@ err: if (i == total_nb_qps) break; - cdev_id = enabled_cdevs[i]; - if (ctx[i] && cperf_testmap[opts.test].destructor) cperf_testmap[opts.test].destructor(ctx[i]); i++; @@ -613,7 +698,7 @@ err: for (i = 0; i < nb_cryptodevs && i < RTE_CRYPTO_MAX_DEVS; i++) rte_cryptodev_stop(enabled_cdevs[i]); - + rte_free(opts.imix_buffer_sizes); free_test_vector(t_vec, &opts); printf("\n"); diff --git a/dpdk/app/test-crypto-perf/meson.build b/dpdk/app/test-crypto-perf/meson.build new file mode 100644 index 00000000..d735b186 --- /dev/null +++ b/dpdk/app/test-crypto-perf/meson.build @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2018 Intel Corporation + +allow_experimental_apis = true +sources = files('cperf_ops.c', + 'cperf_options_parsing.c', + 'cperf_test_common.c', + 'cperf_test_latency.c', + 'cperf_test_pmd_cyclecount.c', + 'cperf_test_throughput.c', + 'cperf_test_vector_parsing.c', + 'cperf_test_vectors.c', + 'cperf_test_verify.c', + 'main.c') +deps += ['cryptodev'] diff --git a/dpdk/app/test-eventdev/Makefile b/dpdk/app/test-eventdev/Makefile index dcb2ac47..cb659110 100644 --- a/dpdk/app/test-eventdev/Makefile +++ b/dpdk/app/test-eventdev/Makefile @@ -1,37 +1,12 @@ -# BSD LICENSE +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Cavium, Inc # -# Copyright(c) 2017 Cavium, Inc. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Cavium, Inc nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. include $(RTE_SDK)/mk/rte.vars.mk APP = dpdk-test-eventdev +CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) @@ -51,4 +26,8 @@ SRCS-y += test_perf_common.c SRCS-y += test_perf_queue.c SRCS-y += test_perf_atq.c +SRCS-y += test_pipeline_common.c +SRCS-y += test_pipeline_queue.c +SRCS-y += test_pipeline_atq.c + include $(RTE_SDK)/mk/rte.app.mk diff --git a/dpdk/app/test-eventdev/evt_common.h b/dpdk/app/test-eventdev/evt_common.h index 0fadab4a..21d3c103 100644 --- a/dpdk/app/test-eventdev/evt_common.h +++ b/dpdk/app/test-eventdev/evt_common.h @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #ifndef _EVT_COMMON_ @@ -94,41 +66,36 @@ evt_has_all_types_queue(uint8_t dev_id) } static inline int -evt_service_setup(uint8_t dev_id) +evt_service_setup(uint32_t service_id) { - uint32_t service_id; int32_t core_cnt; unsigned int lcore = 0; uint32_t core_array[RTE_MAX_LCORE]; uint8_t cnt; uint8_t min_cnt = UINT8_MAX; - if (evt_has_distributed_sched(dev_id)) - return 0; - if (!rte_service_lcore_count()) return -ENOENT; - if (!rte_event_dev_service_id_get(dev_id, &service_id)) { - core_cnt = rte_service_lcore_list(core_array, - RTE_MAX_LCORE); - if (core_cnt < 0) - return -ENOENT; - /* Get the core which has least number of services running. */ - while (core_cnt--) { - /* Reset default mapping */ - rte_service_map_lcore_set(service_id, - core_array[core_cnt], 0); - cnt = rte_service_lcore_count_services( - core_array[core_cnt]); - if (cnt < min_cnt) { - lcore = core_array[core_cnt]; - min_cnt = cnt; - } + core_cnt = rte_service_lcore_list(core_array, + RTE_MAX_LCORE); + if (core_cnt < 0) + return -ENOENT; + /* Get the core which has least number of services running. */ + while (core_cnt--) { + /* Reset default mapping */ + rte_service_map_lcore_set(service_id, + core_array[core_cnt], 0); + cnt = rte_service_lcore_count_services( + core_array[core_cnt]); + if (cnt < min_cnt) { + lcore = core_array[core_cnt]; + min_cnt = cnt; } - if (rte_service_map_lcore_set(service_id, lcore, 1)) - return -ENOENT; } + if (rte_service_map_lcore_set(service_id, lcore, 1)) + return -ENOENT; + return 0; } diff --git a/dpdk/app/test-eventdev/evt_main.c b/dpdk/app/test-eventdev/evt_main.c index 1c3a7fae..a8d304ba 100644 --- a/dpdk/app/test-eventdev/evt_main.c +++ b/dpdk/app/test-eventdev/evt_main.c @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #include @@ -48,26 +20,41 @@ struct evt_test *test; static void signal_handler(int signum) { - if (signum == SIGINT || signum == SIGTERM) { + int i; + static uint8_t once; + + if ((signum == SIGINT || signum == SIGTERM) && !once) { + once = true; printf("\nSignal %d received, preparing to exit...\n", signum); - /* request all lcores to exit from the main loop */ - *(int *)test->test_priv = true; - rte_wmb(); - rte_eal_mp_wait_lcore(); + if (test != NULL) { + /* request all lcores to exit from the main loop */ + *(int *)test->test_priv = true; + rte_wmb(); - if (test->ops.eventdev_destroy) - test->ops.eventdev_destroy(test, &opt); + if (test->ops.ethdev_destroy) + test->ops.ethdev_destroy(test, &opt); - if (test->ops.ethdev_destroy) - test->ops.ethdev_destroy(test, &opt); + rte_eal_mp_wait_lcore(); - if (test->ops.mempool_destroy) - test->ops.mempool_destroy(test, &opt); + if (test->ops.test_result) + test->ops.test_result(test, &opt); - if (test->ops.test_destroy) - test->ops.test_destroy(test, &opt); + if (opt.prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { + RTE_ETH_FOREACH_DEV(i) + rte_eth_dev_close(i); + } + + if (test->ops.eventdev_destroy) + test->ops.eventdev_destroy(test, &opt); + + if (test->ops.mempool_destroy) + test->ops.mempool_destroy(test, &opt); + + if (test->ops.test_destroy) + test->ops.test_destroy(test, &opt); + } /* exit with the expected status */ signal(signum, SIG_DFL); diff --git a/dpdk/app/test-eventdev/evt_options.c b/dpdk/app/test-eventdev/evt_options.c index e2187dfc..cfa43a16 100644 --- a/dpdk/app/test-eventdev/evt_options.c +++ b/dpdk/app/test-eventdev/evt_options.c @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #include @@ -55,6 +27,12 @@ evt_options_default(struct evt_options *opt) opt->pool_sz = 16 * 1024; opt->wkr_deq_dep = 16; opt->nb_pkts = (1ULL << 26); /* do ~64M packets */ + opt->nb_timers = 1E8; + opt->nb_timer_adptrs = 1; + opt->timer_tick_nsec = 1E3; /* 1000ns ~ 1us */ + opt->max_tmo_nsec = 1E5; /* 100000ns ~100us */ + opt->expiry_nsec = 1E4; /* 10000ns ~10us */ + opt->prod_type = EVT_PROD_TYPE_SYNT; } typedef int (*option_parser_t)(struct evt_options *opt, @@ -106,6 +84,29 @@ evt_parse_queue_priority(struct evt_options *opt, const char *arg __rte_unused) return 0; } +static int +evt_parse_eth_prod_type(struct evt_options *opt, const char *arg __rte_unused) +{ + opt->prod_type = EVT_PROD_TYPE_ETH_RX_ADPTR; + return 0; +} + +static int +evt_parse_timer_prod_type(struct evt_options *opt, const char *arg __rte_unused) +{ + opt->prod_type = EVT_PROD_TYPE_EVENT_TIMER_ADPTR; + return 0; +} + +static int +evt_parse_timer_prod_type_burst(struct evt_options *opt, + const char *arg __rte_unused) +{ + opt->prod_type = EVT_PROD_TYPE_EVENT_TIMER_ADPTR; + opt->timdev_use_burst = 1; + return 0; +} + static int evt_parse_test_name(struct evt_options *opt, const char *arg) { @@ -139,6 +140,56 @@ evt_parse_nb_pkts(struct evt_options *opt, const char *arg) return ret; } +static int +evt_parse_nb_timers(struct evt_options *opt, const char *arg) +{ + int ret; + + ret = parser_read_uint64(&(opt->nb_timers), arg); + + return ret; +} + +static int +evt_parse_timer_tick_nsec(struct evt_options *opt, const char *arg) +{ + int ret; + + ret = parser_read_uint64(&(opt->timer_tick_nsec), arg); + + return ret; +} + +static int +evt_parse_max_tmo_nsec(struct evt_options *opt, const char *arg) +{ + int ret; + + ret = parser_read_uint64(&(opt->max_tmo_nsec), arg); + + return ret; +} + +static int +evt_parse_expiry_nsec(struct evt_options *opt, const char *arg) +{ + int ret; + + ret = parser_read_uint64(&(opt->expiry_nsec), arg); + + return ret; +} + +static int +evt_parse_nb_timer_adptrs(struct evt_options *opt, const char *arg) +{ + int ret; + + ret = parser_read_uint8(&(opt->nb_timer_adptrs), arg); + + return ret; +} + static int evt_parse_pool_sz(struct evt_options *opt, const char *arg) { @@ -189,6 +240,17 @@ usage(char *program) "\t--worker_deq_depth : dequeue depth of the worker\n" "\t--fwd_latency : perform fwd_latency measurement\n" "\t--queue_priority : enable queue priority\n" + "\t--prod_type_ethdev : use ethernet device as producer.\n" + "\t--prod_type_timerdev : use event timer device as producer.\n" + "\t expity_nsec would be the timeout\n" + "\t in ns.\n" + "\t--prod_type_timerdev_burst : use timer device as producer\n" + "\t burst mode.\n" + "\t--nb_timers : number of timers to arm.\n" + "\t--nb_timer_adptrs : number of timer adapters to use.\n" + "\t--timer_tick_nsec : timer tick interval in ns.\n" + "\t--max_tmo_nsec : max timeout interval in ns.\n" + "\t--expiry_nsec : event timer expiry ns.\n" ); printf("available tests:\n"); evt_test_dump_names(); @@ -236,21 +298,29 @@ evt_parse_sched_type_list(struct evt_options *opt, const char *arg) } static struct option lgopts[] = { - { EVT_NB_FLOWS, 1, 0, 0 }, - { EVT_DEVICE, 1, 0, 0 }, - { EVT_VERBOSE, 1, 0, 0 }, - { EVT_TEST, 1, 0, 0 }, - { EVT_PROD_LCORES, 1, 0, 0 }, - { EVT_WORK_LCORES, 1, 0, 0 }, - { EVT_SOCKET_ID, 1, 0, 0 }, - { EVT_POOL_SZ, 1, 0, 0 }, - { EVT_NB_PKTS, 1, 0, 0 }, - { EVT_WKR_DEQ_DEP, 1, 0, 0 }, - { EVT_SCHED_TYPE_LIST, 1, 0, 0 }, - { EVT_FWD_LATENCY, 0, 0, 0 }, - { EVT_QUEUE_PRIORITY, 0, 0, 0 }, - { EVT_HELP, 0, 0, 0 }, - { NULL, 0, 0, 0 } + { EVT_NB_FLOWS, 1, 0, 0 }, + { EVT_DEVICE, 1, 0, 0 }, + { EVT_VERBOSE, 1, 0, 0 }, + { EVT_TEST, 1, 0, 0 }, + { EVT_PROD_LCORES, 1, 0, 0 }, + { EVT_WORK_LCORES, 1, 0, 0 }, + { EVT_SOCKET_ID, 1, 0, 0 }, + { EVT_POOL_SZ, 1, 0, 0 }, + { EVT_NB_PKTS, 1, 0, 0 }, + { EVT_WKR_DEQ_DEP, 1, 0, 0 }, + { EVT_SCHED_TYPE_LIST, 1, 0, 0 }, + { EVT_FWD_LATENCY, 0, 0, 0 }, + { EVT_QUEUE_PRIORITY, 0, 0, 0 }, + { EVT_PROD_ETHDEV, 0, 0, 0 }, + { EVT_PROD_TIMERDEV, 0, 0, 0 }, + { EVT_PROD_TIMERDEV_BURST, 0, 0, 0 }, + { EVT_NB_TIMERS, 1, 0, 0 }, + { EVT_NB_TIMER_ADPTRS, 1, 0, 0 }, + { EVT_TIMER_TICK_NSEC, 1, 0, 0 }, + { EVT_MAX_TMO_NSEC, 1, 0, 0 }, + { EVT_EXPIRY_NSEC, 1, 0, 0 }, + { EVT_HELP, 0, 0, 0 }, + { NULL, 0, 0, 0 } }; static int @@ -272,11 +342,19 @@ evt_opts_parse_long(int opt_idx, struct evt_options *opt) { EVT_SCHED_TYPE_LIST, evt_parse_sched_type_list}, { EVT_FWD_LATENCY, evt_parse_fwd_latency}, { EVT_QUEUE_PRIORITY, evt_parse_queue_priority}, + { EVT_PROD_ETHDEV, evt_parse_eth_prod_type}, + { EVT_PROD_TIMERDEV, evt_parse_timer_prod_type}, + { EVT_PROD_TIMERDEV_BURST, evt_parse_timer_prod_type_burst}, + { EVT_NB_TIMERS, evt_parse_nb_timers}, + { EVT_NB_TIMER_ADPTRS, evt_parse_nb_timer_adptrs}, + { EVT_TIMER_TICK_NSEC, evt_parse_timer_tick_nsec}, + { EVT_MAX_TMO_NSEC, evt_parse_max_tmo_nsec}, + { EVT_EXPIRY_NSEC, evt_parse_expiry_nsec}, }; for (i = 0; i < RTE_DIM(parsermap); i++) { if (strncmp(lgopts[opt_idx].name, parsermap[i].lgopt_name, - strlen(parsermap[i].lgopt_name)) == 0) + strlen(lgopts[opt_idx].name)) == 0) return parsermap[i].parser_fn(opt, optarg); } @@ -322,6 +400,7 @@ evt_options_dump(struct evt_options *opt) evt_dump("pool_sz", "%d", opt->pool_sz); evt_dump("master lcore", "%d", rte_get_master_lcore()); evt_dump("nb_pkts", "%"PRIu64, opt->nb_pkts); + evt_dump("nb_timers", "%"PRIu64, opt->nb_timers); evt_dump_begin("available lcores"); RTE_LCORE_FOREACH(lcore_id) printf("%d ", lcore_id); diff --git a/dpdk/app/test-eventdev/evt_options.h b/dpdk/app/test-eventdev/evt_options.h index a9a91252..f3de48a1 100644 --- a/dpdk/app/test-eventdev/evt_options.h +++ b/dpdk/app/test-eventdev/evt_options.h @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #ifndef _EVT_OPTIONS_ @@ -37,6 +9,7 @@ #include #include +#include #include #include @@ -58,8 +31,24 @@ #define EVT_SCHED_TYPE_LIST ("stlist") #define EVT_FWD_LATENCY ("fwd_latency") #define EVT_QUEUE_PRIORITY ("queue_priority") +#define EVT_PROD_ETHDEV ("prod_type_ethdev") +#define EVT_PROD_TIMERDEV ("prod_type_timerdev") +#define EVT_PROD_TIMERDEV_BURST ("prod_type_timerdev_burst") +#define EVT_NB_TIMERS ("nb_timers") +#define EVT_NB_TIMER_ADPTRS ("nb_timer_adptrs") +#define EVT_TIMER_TICK_NSEC ("timer_tick_nsec") +#define EVT_MAX_TMO_NSEC ("max_tmo_nsec") +#define EVT_EXPIRY_NSEC ("expiry_nsec") #define EVT_HELP ("help") +enum evt_prod_type { + EVT_PROD_TYPE_NONE, + EVT_PROD_TYPE_SYNT, /* Producer type Synthetic i.e. CPU. */ + EVT_PROD_TYPE_ETH_RX_ADPTR, /* Producer type Eth Rx Adapter. */ + EVT_PROD_TYPE_EVENT_TIMER_ADPTR, /* Producer type Timer Adapter. */ + EVT_PROD_TYPE_MAX, +}; + struct evt_options { #define EVT_TEST_NAME_MAX_LEN 32 char test_name[EVT_TEST_NAME_MAX_LEN]; @@ -72,10 +61,19 @@ struct evt_options { int nb_stages; int verbose_level; uint64_t nb_pkts; + uint8_t nb_timer_adptrs; + uint64_t nb_timers; + uint64_t timer_tick_nsec; + uint64_t optm_timer_tick_nsec; + uint64_t max_tmo_nsec; + uint64_t expiry_nsec; uint16_t wkr_deq_dep; uint8_t dev_id; uint32_t fwd_latency:1; uint32_t q_priority:1; + enum evt_prod_type prod_type; + uint8_t timdev_use_burst; + uint8_t timdev_cnt; }; void evt_options_default(struct evt_options *opt); @@ -266,4 +264,42 @@ evt_dump_sched_type_list(struct evt_options *opt) evt_dump_end; } +#define EVT_PROD_MAX_NAME_LEN 50 +static inline void +evt_dump_producer_type(struct evt_options *opt) +{ + char name[EVT_PROD_MAX_NAME_LEN]; + + switch (opt->prod_type) { + default: + case EVT_PROD_TYPE_SYNT: + snprintf(name, EVT_PROD_MAX_NAME_LEN, + "Synthetic producer lcores"); + break; + case EVT_PROD_TYPE_ETH_RX_ADPTR: + snprintf(name, EVT_PROD_MAX_NAME_LEN, + "Ethdev Rx Adapter producers"); + evt_dump("nb_ethdev", "%d", rte_eth_dev_count_avail()); + break; + case EVT_PROD_TYPE_EVENT_TIMER_ADPTR: + if (opt->timdev_use_burst) + snprintf(name, EVT_PROD_MAX_NAME_LEN, + "Event timer adapter burst mode producer"); + else + snprintf(name, EVT_PROD_MAX_NAME_LEN, + "Event timer adapter producer"); + evt_dump("nb_timer_adapters", "%d", opt->nb_timer_adptrs); + evt_dump("max_tmo_nsec", "%"PRIu64"", opt->max_tmo_nsec); + evt_dump("expiry_nsec", "%"PRIu64"", opt->expiry_nsec); + if (opt->optm_timer_tick_nsec) + evt_dump("optm_timer_tick_nsec", "%"PRIu64"", + opt->optm_timer_tick_nsec); + else + evt_dump("timer_tick_nsec", "%"PRIu64"", + opt->timer_tick_nsec); + break; + } + evt_dump("prod_type", "%s", name); +} + #endif /* _EVT_OPTIONS_ */ diff --git a/dpdk/app/test-eventdev/evt_test.c b/dpdk/app/test-eventdev/evt_test.c index 3a432233..72d6228b 100644 --- a/dpdk/app/test-eventdev/evt_test.c +++ b/dpdk/app/test-eventdev/evt_test.c @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc. 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #include diff --git a/dpdk/app/test-eventdev/evt_test.h b/dpdk/app/test-eventdev/evt_test.h index 17bdd165..f07d2c33 100644 --- a/dpdk/app/test-eventdev/evt_test.h +++ b/dpdk/app/test-eventdev/evt_test.h @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #ifndef _EVT_TEST_ @@ -105,8 +77,7 @@ void evt_test_dump_names(void); #define EVT_TEST_REGISTER(nm) \ static struct evt_test_entry _evt_test_entry_ ##nm; \ -RTE_INIT(evt_test_ ##nm); \ -static void evt_test_ ##nm(void) \ +RTE_INIT(evt_test_ ##nm) \ { \ _evt_test_entry_ ##nm.test.name = RTE_STR(nm);\ memcpy(&_evt_test_entry_ ##nm.test.ops, &nm, \ diff --git a/dpdk/app/test-eventdev/meson.build b/dpdk/app/test-eventdev/meson.build new file mode 100644 index 00000000..a81dcd13 --- /dev/null +++ b/dpdk/app/test-eventdev/meson.build @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2017 Cavium, Inc + +allow_experimental_apis = true +sources = files('evt_main.c', + 'evt_options.c', + 'evt_test.c', + 'parser.c', + 'test_order_common.c', + 'test_order_atq.c', + 'test_order_queue.c', + 'test_perf_common.c', + 'test_perf_atq.c', + 'test_perf_queue.c') +deps += 'eventdev' diff --git a/dpdk/app/test-eventdev/parser.h b/dpdk/app/test-eventdev/parser.h index 75a5a3b4..673ff22d 100644 --- a/dpdk/app/test-eventdev/parser.h +++ b/dpdk/app/test-eventdev/parser.h @@ -1,34 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2016 Intel Corporation */ #ifndef __INCLUDE_PARSER_H__ diff --git a/dpdk/app/test-eventdev/test_order_atq.c b/dpdk/app/test-eventdev/test_order_atq.c index 4ee0dea8..4a8546e7 100644 --- a/dpdk/app/test-eventdev/test_order_atq.c +++ b/dpdk/app/test-eventdev/test_order_atq.c @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #include @@ -35,7 +7,7 @@ #include "test_order_common.h" -/* See http://dpdk.org/doc/guides/tools/testeventdev.html for test details */ +/* See http://doc.dpdk.org/guides/tools/testeventdev.html for test details */ static inline __attribute__((always_inline)) void order_atq_process_stage_0(struct rte_event *const ev) @@ -179,10 +151,14 @@ order_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) if (ret) return ret; - ret = evt_service_setup(opt->dev_id); - if (ret) { - evt_err("No service lcore found to run event dev."); - return ret; + if (!evt_has_distributed_sched(opt->dev_id)) { + uint32_t service_id; + rte_event_dev_service_id_get(opt->dev_id, &service_id); + ret = evt_service_setup(service_id); + if (ret) { + evt_err("No service lcore found to run event dev."); + return ret; + } } ret = rte_event_dev_start(opt->dev_id); diff --git a/dpdk/app/test-eventdev/test_order_common.c b/dpdk/app/test-eventdev/test_order_common.c index 7cfe7fac..8a342013 100644 --- a/dpdk/app/test-eventdev/test_order_common.c +++ b/dpdk/app/test-eventdev/test_order_common.c @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #include "test_order_common.h" diff --git a/dpdk/app/test-eventdev/test_order_common.h b/dpdk/app/test-eventdev/test_order_common.h index 57bc76e0..22a1cc83 100644 --- a/dpdk/app/test-eventdev/test_order_common.h +++ b/dpdk/app/test-eventdev/test_order_common.h @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #ifndef _TEST_ORDER_COMMON_ diff --git a/dpdk/app/test-eventdev/test_order_queue.c b/dpdk/app/test-eventdev/test_order_queue.c index eef69a4c..a272c7a5 100644 --- a/dpdk/app/test-eventdev/test_order_queue.c +++ b/dpdk/app/test-eventdev/test_order_queue.c @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #include @@ -35,7 +7,7 @@ #include "test_order_common.h" -/* See http://dpdk.org/doc/guides/tools/testeventdev.html for test details */ +/* See http://doc.dpdk.org/guides/tools/testeventdev.html for test details */ static inline __attribute__((always_inline)) void order_queue_process_stage_0(struct rte_event *const ev) @@ -192,10 +164,14 @@ order_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) if (ret) return ret; - ret = evt_service_setup(opt->dev_id); - if (ret) { - evt_err("No service lcore found to run event dev."); - return ret; + if (!evt_has_distributed_sched(opt->dev_id)) { + uint32_t service_id; + rte_event_dev_service_id_get(opt->dev_id, &service_id); + ret = evt_service_setup(service_id); + if (ret) { + evt_err("No service lcore found to run event dev."); + return ret; + } } ret = rte_event_dev_start(opt->dev_id); diff --git a/dpdk/app/test-eventdev/test_perf_atq.c b/dpdk/app/test-eventdev/test_perf_atq.c index 0e9f2db0..b76ca605 100644 --- a/dpdk/app/test-eventdev/test_perf_atq.c +++ b/dpdk/app/test-eventdev/test_perf_atq.c @@ -1,44 +1,17 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #include "test_perf_common.h" -/* See http://dpdk.org/doc/guides/tools/testeventdev.html for test details */ +/* See http://doc.dpdk.org/guides/tools/testeventdev.html for test details */ static inline int atq_nb_event_queues(struct evt_options *opt) { /* nb_queues = number of producers */ - return evt_nr_active_lcores(opt->plcores); + return opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR ? + rte_eth_dev_count_avail() : evt_nr_active_lcores(opt->plcores); } static inline __attribute__((always_inline)) void @@ -70,15 +43,12 @@ perf_atq_worker(void *arg, const int enable_fwd_latency) while (t->done == false) { uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); - if (enable_fwd_latency) - rte_prefetch0(ev.event_ptr); - if (!event) { rte_pause(); continue; } - if (enable_fwd_latency) + if (enable_fwd_latency && !prod_timer_type) /* first stage in pipeline, mark ts to compute fwd latency */ atq_mark_fwd_latency(&ev); @@ -117,7 +87,7 @@ perf_atq_worker_burst(void *arg, const int enable_fwd_latency) } for (i = 0; i < nb_rx; i++) { - if (enable_fwd_latency) { + if (enable_fwd_latency && !prod_timer_type) { rte_prefetch0(ev[i+1].event_ptr); /* first stage in pipeline. * mark time stamp to compute fwd latency @@ -185,14 +155,33 @@ perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) { int ret; uint8_t queue; + uint8_t nb_queues; + uint8_t nb_ports; + struct rte_event_dev_info dev_info; + + nb_ports = evt_nr_active_lcores(opt->wlcores); + nb_ports += (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR || + opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) ? 0 : + evt_nr_active_lcores(opt->plcores); + + nb_queues = atq_nb_event_queues(opt); + + memset(&dev_info, 0, sizeof(struct rte_event_dev_info)); + ret = rte_event_dev_info_get(opt->dev_id, &dev_info); + if (ret) { + evt_err("failed to get eventdev info %d", opt->dev_id); + return ret; + } const struct rte_event_dev_config config = { - .nb_event_queues = atq_nb_event_queues(opt), - .nb_event_ports = perf_nb_event_ports(opt), - .nb_events_limit = 4096, + .nb_event_queues = nb_queues, + .nb_event_ports = nb_ports, + .nb_events_limit = dev_info.max_num_events, .nb_event_queue_flows = opt->nb_flows, - .nb_event_port_dequeue_depth = 128, - .nb_event_port_enqueue_depth = 128, + .nb_event_port_dequeue_depth = + dev_info.max_event_port_dequeue_depth, + .nb_event_port_enqueue_depth = + dev_info.max_event_port_enqueue_depth, }; ret = rte_event_dev_configure(opt->dev_id, &config); @@ -208,7 +197,7 @@ perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) .nb_atomic_order_sequences = opt->nb_flows, }; /* queue configurations */ - for (queue = 0; queue < atq_nb_event_queues(opt); queue++) { + for (queue = 0; queue < nb_queues; queue++) { ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf); if (ret) { evt_err("failed to setup queue=%d", queue); @@ -216,15 +205,29 @@ perf_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) } } - ret = perf_event_dev_port_setup(test, opt, 1 /* stride */, - atq_nb_event_queues(opt)); + if (opt->wkr_deq_dep > dev_info.max_event_port_dequeue_depth) + opt->wkr_deq_dep = dev_info.max_event_port_dequeue_depth; + + /* port configuration */ + const struct rte_event_port_conf p_conf = { + .dequeue_depth = opt->wkr_deq_dep, + .enqueue_depth = dev_info.max_event_port_dequeue_depth, + .new_event_threshold = dev_info.max_num_events, + }; + + ret = perf_event_dev_port_setup(test, opt, 1 /* stride */, nb_queues, + &p_conf); if (ret) return ret; - ret = evt_service_setup(opt->dev_id); - if (ret) { - evt_err("No service lcore found to run event dev."); - return ret; + if (!evt_has_distributed_sched(opt->dev_id)) { + uint32_t service_id; + rte_event_dev_service_id_get(opt->dev_id, &service_id); + ret = evt_service_setup(service_id); + if (ret) { + evt_err("No service lcore found to run event dev."); + return ret; + } } ret = rte_event_dev_start(opt->dev_id); @@ -271,11 +274,13 @@ static const struct evt_test_ops perf_atq = { .opt_check = perf_atq_opt_check, .opt_dump = perf_atq_opt_dump, .test_setup = perf_test_setup, + .ethdev_setup = perf_ethdev_setup, .mempool_setup = perf_mempool_setup, .eventdev_setup = perf_atq_eventdev_setup, .launch_lcores = perf_atq_launch_lcores, .eventdev_destroy = perf_eventdev_destroy, .mempool_destroy = perf_mempool_destroy, + .ethdev_destroy = perf_ethdev_destroy, .test_result = perf_test_result, .test_destroy = perf_test_destroy, }; diff --git a/dpdk/app/test-eventdev/test_perf_common.c b/dpdk/app/test-eventdev/test_perf_common.c index e77b4723..86187753 100644 --- a/dpdk/app/test-eventdev/test_perf_common.c +++ b/dpdk/app/test-eventdev/test_perf_common.c @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #include "test_perf_common.h" @@ -36,8 +8,20 @@ int perf_test_result(struct evt_test *test, struct evt_options *opt) { RTE_SET_USED(opt); + int i; + uint64_t total = 0; struct test_perf *t = evt_test_priv(test); + printf("Packet distribution across worker cores :\n"); + for (i = 0; i < t->nb_workers; i++) + total += t->worker[i].processed_pkts; + for (i = 0; i < t->nb_workers; i++) + printf("Worker %d packets: "CLGRN"%"PRIx64" "CLNRM"percentage:" + CLGRN" %3.2f\n"CLNRM, i, + t->worker[i].processed_pkts, + (((double)t->worker[i].processed_pkts)/total) + * 100); + return t->result; } @@ -88,6 +72,145 @@ perf_producer(void *arg) return 0; } +static inline int +perf_event_timer_producer(void *arg) +{ + struct prod_data *p = arg; + struct test_perf *t = p->t; + struct evt_options *opt = t->opt; + uint32_t flow_counter = 0; + uint64_t count = 0; + uint64_t arm_latency = 0; + const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs; + const uint32_t nb_flows = t->nb_flows; + const uint64_t nb_timers = opt->nb_timers; + struct rte_mempool *pool = t->pool; + struct perf_elt *m; + struct rte_event_timer_adapter **adptr = t->timer_adptr; + struct rte_event_timer tim; + uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; + + memset(&tim, 0, sizeof(struct rte_event_timer)); + timeout_ticks = opt->optm_timer_tick_nsec ? + (timeout_ticks * opt->timer_tick_nsec) + / opt->optm_timer_tick_nsec : timeout_ticks; + timeout_ticks += timeout_ticks ? 0 : 1; + tim.ev.event_type = RTE_EVENT_TYPE_TIMER; + tim.ev.op = RTE_EVENT_OP_NEW; + tim.ev.sched_type = t->opt->sched_type_list[0]; + tim.ev.queue_id = p->queue_id; + tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; + tim.state = RTE_EVENT_TIMER_NOT_ARMED; + tim.timeout_ticks = timeout_ticks; + + if (opt->verbose_level > 1) + printf("%s(): lcore %d\n", __func__, rte_lcore_id()); + + while (count < nb_timers && t->done == false) { + if (rte_mempool_get(pool, (void **)&m) < 0) + continue; + + m->tim = tim; + m->tim.ev.flow_id = flow_counter++ % nb_flows; + m->tim.ev.event_ptr = m; + m->timestamp = rte_get_timer_cycles(); + while (rte_event_timer_arm_burst( + adptr[flow_counter % nb_timer_adptrs], + (struct rte_event_timer **)&m, 1) != 1) { + if (t->done) + break; + rte_pause(); + m->timestamp = rte_get_timer_cycles(); + } + arm_latency += rte_get_timer_cycles() - m->timestamp; + count++; + } + fflush(stdout); + rte_delay_ms(1000); + printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", + __func__, rte_lcore_id(), (float)(arm_latency / count) / + (rte_get_timer_hz() / 1000000)); + return 0; +} + +static inline int +perf_event_timer_producer_burst(void *arg) +{ + int i; + struct prod_data *p = arg; + struct test_perf *t = p->t; + struct evt_options *opt = t->opt; + uint32_t flow_counter = 0; + uint64_t count = 0; + uint64_t arm_latency = 0; + const uint8_t nb_timer_adptrs = opt->nb_timer_adptrs; + const uint32_t nb_flows = t->nb_flows; + const uint64_t nb_timers = opt->nb_timers; + struct rte_mempool *pool = t->pool; + struct perf_elt *m[BURST_SIZE + 1] = {NULL}; + struct rte_event_timer_adapter **adptr = t->timer_adptr; + struct rte_event_timer tim; + uint64_t timeout_ticks = opt->expiry_nsec / opt->timer_tick_nsec; + + memset(&tim, 0, sizeof(struct rte_event_timer)); + timeout_ticks = opt->optm_timer_tick_nsec ? + (timeout_ticks * opt->timer_tick_nsec) + / opt->optm_timer_tick_nsec : timeout_ticks; + timeout_ticks += timeout_ticks ? 0 : 1; + tim.ev.event_type = RTE_EVENT_TYPE_TIMER; + tim.ev.op = RTE_EVENT_OP_NEW; + tim.ev.sched_type = t->opt->sched_type_list[0]; + tim.ev.queue_id = p->queue_id; + tim.ev.priority = RTE_EVENT_DEV_PRIORITY_NORMAL; + tim.state = RTE_EVENT_TIMER_NOT_ARMED; + tim.timeout_ticks = timeout_ticks; + + if (opt->verbose_level > 1) + printf("%s(): lcore %d\n", __func__, rte_lcore_id()); + + while (count < nb_timers && t->done == false) { + if (rte_mempool_get_bulk(pool, (void **)m, BURST_SIZE) < 0) + continue; + for (i = 0; i < BURST_SIZE; i++) { + rte_prefetch0(m[i + 1]); + m[i]->tim = tim; + m[i]->tim.ev.flow_id = flow_counter++ % nb_flows; + m[i]->tim.ev.event_ptr = m[i]; + m[i]->timestamp = rte_get_timer_cycles(); + } + rte_event_timer_arm_tmo_tick_burst( + adptr[flow_counter % nb_timer_adptrs], + (struct rte_event_timer **)m, + tim.timeout_ticks, + BURST_SIZE); + arm_latency += rte_get_timer_cycles() - m[i - 1]->timestamp; + count += BURST_SIZE; + } + fflush(stdout); + rte_delay_ms(1000); + printf("%s(): lcore %d Average event timer arm latency = %.3f us\n", + __func__, rte_lcore_id(), (float)(arm_latency / count) / + (rte_get_timer_hz() / 1000000)); + return 0; +} + +static int +perf_producer_wrapper(void *arg) +{ + struct prod_data *p = arg; + struct test_perf *t = p->t; + /* Launch the producer function only in case of synthetic producer. */ + if (t->opt->prod_type == EVT_PROD_TYPE_SYNT) + return perf_producer(arg); + else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && + !t->opt->timdev_use_burst) + return perf_event_timer_producer(arg); + else if (t->opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR && + t->opt->timdev_use_burst) + return perf_event_timer_producer_burst(arg); + return 0; +} + static inline uint64_t processed_pkts(struct test_perf *t) { @@ -142,8 +265,8 @@ perf_launch_lcores(struct evt_test *test, struct evt_options *opt, if (!(opt->plcores[lcore_id])) continue; - ret = rte_eal_remote_launch(perf_producer, &t->prod[port_idx], - lcore_id); + ret = rte_eal_remote_launch(perf_producer_wrapper, + &t->prod[port_idx], lcore_id); if (ret) { evt_err("failed to launch perf_producer %d", lcore_id); return ret; @@ -151,8 +274,7 @@ perf_launch_lcores(struct evt_test *test, struct evt_options *opt, port_idx++; } - const uint64_t total_pkts = opt->nb_pkts * - evt_nr_active_lcores(opt->plcores); + const uint64_t total_pkts = t->outstand_pkts; uint64_t dead_lock_cycles = rte_get_timer_cycles(); int64_t dead_lock_remaining = total_pkts; @@ -193,14 +315,19 @@ perf_launch_lcores(struct evt_test *test, struct evt_options *opt, fflush(stdout); if (remaining <= 0) { - t->done = true; t->result = EVT_TEST_SUCCESS; - rte_smp_wmb(); - break; + if (opt->prod_type == EVT_PROD_TYPE_SYNT || + opt->prod_type == + EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { + t->done = true; + rte_smp_wmb(); + break; + } } } - if (new_cycles - dead_lock_cycles > dead_lock_sample) { + if (new_cycles - dead_lock_cycles > dead_lock_sample && + opt->prod_type == EVT_PROD_TYPE_SYNT) { remaining = t->outstand_pkts - processed_pkts(t); if (dead_lock_remaining == remaining) { rte_event_dev_dump(opt->dev_id, stdout); @@ -217,21 +344,142 @@ perf_launch_lcores(struct evt_test *test, struct evt_options *opt, return 0; } +static int +perf_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride, + struct rte_event_port_conf prod_conf) +{ + int ret = 0; + uint16_t prod; + struct rte_event_eth_rx_adapter_queue_conf queue_conf; + + memset(&queue_conf, 0, + sizeof(struct rte_event_eth_rx_adapter_queue_conf)); + queue_conf.ev.sched_type = opt->sched_type_list[0]; + RTE_ETH_FOREACH_DEV(prod) { + uint32_t cap; + + ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id, + prod, &cap); + if (ret) { + evt_err("failed to get event rx adapter[%d]" + " capabilities", + opt->dev_id); + return ret; + } + queue_conf.ev.queue_id = prod * stride; + ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id, + &prod_conf); + if (ret) { + evt_err("failed to create rx adapter[%d]", prod); + return ret; + } + ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1, + &queue_conf); + if (ret) { + evt_err("failed to add rx queues to adapter[%d]", prod); + return ret; + } + + if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) { + uint32_t service_id; + + rte_event_eth_rx_adapter_service_id_get(prod, + &service_id); + ret = evt_service_setup(service_id); + if (ret) { + evt_err("Failed to setup service core" + " for Rx adapter\n"); + return ret; + } + } + + ret = rte_eth_dev_start(prod); + if (ret) { + evt_err("Ethernet dev [%d] failed to start." + " Using synthetic producer", prod); + return ret; + } + + ret = rte_event_eth_rx_adapter_start(prod); + if (ret) { + evt_err("Rx adapter[%d] start failed", prod); + return ret; + } + printf("%s: Port[%d] using Rx adapter[%d] started\n", __func__, + prod, prod); + } + + return ret; +} + +static int +perf_event_timer_adapter_setup(struct test_perf *t) +{ + int i; + int ret; + struct rte_event_timer_adapter_info adapter_info; + struct rte_event_timer_adapter *wl; + uint8_t nb_producers = evt_nr_active_lcores(t->opt->plcores); + uint8_t flags = RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES; + + if (nb_producers == 1) + flags |= RTE_EVENT_TIMER_ADAPTER_F_SP_PUT; + + for (i = 0; i < t->opt->nb_timer_adptrs; i++) { + struct rte_event_timer_adapter_conf config = { + .event_dev_id = t->opt->dev_id, + .timer_adapter_id = i, + .timer_tick_ns = t->opt->timer_tick_nsec, + .max_tmo_ns = t->opt->max_tmo_nsec, + .nb_timers = 2 * 1024 * 1024, + .flags = flags, + }; + + wl = rte_event_timer_adapter_create(&config); + if (wl == NULL) { + evt_err("failed to create event timer ring %d", i); + return rte_errno; + } + + memset(&adapter_info, 0, + sizeof(struct rte_event_timer_adapter_info)); + rte_event_timer_adapter_get_info(wl, &adapter_info); + t->opt->optm_timer_tick_nsec = adapter_info.min_resolution_ns; + + if (!(adapter_info.caps & + RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT)) { + uint32_t service_id; + + rte_event_timer_adapter_service_id_get(wl, + &service_id); + ret = evt_service_setup(service_id); + if (ret) { + evt_err("Failed to setup service core" + " for timer adapter\n"); + return ret; + } + rte_service_runstate_set(service_id, 1); + } + + ret = rte_event_timer_adapter_start(wl); + if (ret) { + evt_err("failed to Start event timer adapter %d", i); + return ret; + } + t->timer_adptr[i] = wl; + } + return 0; +} + int perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, - uint8_t stride, uint8_t nb_queues) + uint8_t stride, uint8_t nb_queues, + const struct rte_event_port_conf *port_conf) { struct test_perf *t = evt_test_priv(test); - uint8_t port, prod; + uint16_t port, prod; int ret = -1; - /* port configuration */ - const struct rte_event_port_conf wkr_p_conf = { - .dequeue_depth = opt->wkr_deq_dep, - .enqueue_depth = 64, - .new_event_threshold = 4096, - }; - /* setup one port per worker, linking to all queues */ for (port = 0; port < evt_nr_active_lcores(opt->wlcores); port++) { @@ -243,7 +491,7 @@ perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, w->processed_pkts = 0; w->latency = 0; - ret = rte_event_port_setup(opt->dev_id, port, &wkr_p_conf); + ret = rte_event_port_setup(opt->dev_id, port, port_conf); if (ret) { evt_err("failed to setup port %d", port); return ret; @@ -257,26 +505,45 @@ perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, } /* port for producers, no links */ - const struct rte_event_port_conf prod_conf = { - .dequeue_depth = 8, - .enqueue_depth = 32, - .new_event_threshold = 1200, - }; - prod = 0; - for ( ; port < perf_nb_event_ports(opt); port++) { - struct prod_data *p = &t->prod[port]; - - p->dev_id = opt->dev_id; - p->port_id = port; - p->queue_id = prod * stride; - p->t = t; - - ret = rte_event_port_setup(opt->dev_id, port, &prod_conf); - if (ret) { - evt_err("failed to setup port %d", port); - return ret; + if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { + for ( ; port < perf_nb_event_ports(opt); port++) { + struct prod_data *p = &t->prod[port]; + p->t = t; + } + + ret = perf_event_rx_adapter_setup(opt, stride, *port_conf); + if (ret) + return ret; + } else if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { + prod = 0; + for ( ; port < perf_nb_event_ports(opt); port++) { + struct prod_data *p = &t->prod[port]; + p->queue_id = prod * stride; + p->t = t; + prod++; + } + + ret = perf_event_timer_adapter_setup(t); + if (ret) + return ret; + } else { + prod = 0; + for ( ; port < perf_nb_event_ports(opt); port++) { + struct prod_data *p = &t->prod[port]; + + p->dev_id = opt->dev_id; + p->port_id = port; + p->queue_id = prod * stride; + p->t = t; + + ret = rte_event_port_setup(opt->dev_id, port, + port_conf); + if (ret) { + evt_err("failed to setup port %d", port); + return ret; + } + prod++; } - prod++; } return ret; @@ -287,8 +554,10 @@ perf_opt_check(struct evt_options *opt, uint64_t nb_queues) { unsigned int lcores; - /* N producer + N worker + 1 master */ - lcores = 3; + /* N producer + N worker + 1 master when producer cores are used + * Else N worker + 1 master when Rx adapter is used + */ + lcores = opt->prod_type == EVT_PROD_TYPE_SYNT ? 3 : 2; if (rte_lcore_count() < lcores) { evt_err("test need minimum %d lcores", lcores); @@ -313,18 +582,21 @@ perf_opt_check(struct evt_options *opt, uint64_t nb_queues) return -1; } - /* Validate producer lcores */ - if (evt_lcores_has_overlap(opt->plcores, rte_get_master_lcore())) { - evt_err("producer lcores overlaps with master lcore"); - return -1; - } - if (evt_has_disabled_lcore(opt->plcores)) { - evt_err("one or more producer lcores are not enabled"); - return -1; - } - if (!evt_has_active_lcore(opt->plcores)) { - evt_err("minimum one producer is required"); - return -1; + if (opt->prod_type == EVT_PROD_TYPE_SYNT) { + /* Validate producer lcores */ + if (evt_lcores_has_overlap(opt->plcores, + rte_get_master_lcore())) { + evt_err("producer lcores overlaps with master lcore"); + return -1; + } + if (evt_has_disabled_lcore(opt->plcores)) { + evt_err("one or more producer lcores are not enabled"); + return -1; + } + if (!evt_has_active_lcore(opt->plcores)) { + evt_err("minimum one producer is required"); + return -1; + } } if (evt_has_invalid_stage(opt)) @@ -343,10 +615,13 @@ perf_opt_check(struct evt_options *opt, uint64_t nb_queues) } /* Fixups */ - if (opt->nb_stages == 1 && opt->fwd_latency) { + if ((opt->nb_stages == 1 && + opt->prod_type != EVT_PROD_TYPE_EVENT_TIMER_ADPTR) && + opt->fwd_latency) { evt_info("fwd_latency is valid when nb_stages > 1, disabling"); opt->fwd_latency = 0; } + if (opt->fwd_latency && !opt->q_priority) { evt_info("enabled queue priority for latency measurement"); opt->q_priority = 1; @@ -369,13 +644,19 @@ perf_opt_dump(struct evt_options *opt, uint8_t nb_queues) evt_dump("nb_evdev_queues", "%d", nb_queues); evt_dump_queue_priority(opt); evt_dump_sched_type_list(opt); + evt_dump_producer_type(opt); } void perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt) { - RTE_SET_USED(test); + int i; + struct test_perf *t = evt_test_priv(test); + if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { + for (i = 0; i < opt->nb_timer_adptrs; i++) + rte_event_timer_adapter_stop(t->timer_adptr[i]); + } rte_event_dev_stop(opt->dev_id); rte_event_dev_close(opt->dev_id); } @@ -387,18 +668,115 @@ perf_elt_init(struct rte_mempool *mp, void *arg __rte_unused, memset(obj, 0, mp->elt_size); } +#define NB_RX_DESC 128 +#define NB_TX_DESC 512 +int +perf_ethdev_setup(struct evt_test *test, struct evt_options *opt) +{ + uint16_t i; + struct test_perf *t = evt_test_priv(test); + struct rte_eth_conf port_conf = { + .rxmode = { + .mq_mode = ETH_MQ_RX_RSS, + .max_rx_pkt_len = ETHER_MAX_LEN, + .split_hdr_size = 0, + }, + .rx_adv_conf = { + .rss_conf = { + .rss_key = NULL, + .rss_hf = ETH_RSS_IP, + }, + }, + }; + + if (opt->prod_type == EVT_PROD_TYPE_SYNT || + opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) + return 0; + + if (!rte_eth_dev_count_avail()) { + evt_err("No ethernet ports found."); + return -ENODEV; + } + + RTE_ETH_FOREACH_DEV(i) { + struct rte_eth_dev_info dev_info; + struct rte_eth_conf local_port_conf = port_conf; + + rte_eth_dev_info_get(i, &dev_info); + + local_port_conf.rx_adv_conf.rss_conf.rss_hf &= + dev_info.flow_type_rss_offloads; + if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != + port_conf.rx_adv_conf.rss_conf.rss_hf) { + evt_info("Port %u modified RSS hash function based on hardware support," + "requested:%#"PRIx64" configured:%#"PRIx64"\n", + i, + port_conf.rx_adv_conf.rss_conf.rss_hf, + local_port_conf.rx_adv_conf.rss_conf.rss_hf); + } + + if (rte_eth_dev_configure(i, 1, 1, &local_port_conf) < 0) { + evt_err("Failed to configure eth port [%d]", i); + return -EINVAL; + } + + if (rte_eth_rx_queue_setup(i, 0, NB_RX_DESC, + rte_socket_id(), NULL, t->pool) < 0) { + evt_err("Failed to setup eth port [%d] rx_queue: %d.", + i, 0); + return -EINVAL; + } + + if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC, + rte_socket_id(), NULL) < 0) { + evt_err("Failed to setup eth port [%d] tx_queue: %d.", + i, 0); + return -EINVAL; + } + + rte_eth_promiscuous_enable(i); + } + + return 0; +} + +void perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt) +{ + uint16_t i; + RTE_SET_USED(test); + + if (opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR) { + RTE_ETH_FOREACH_DEV(i) { + rte_event_eth_rx_adapter_stop(i); + rte_eth_dev_stop(i); + } + } +} + int perf_mempool_setup(struct evt_test *test, struct evt_options *opt) { struct test_perf *t = evt_test_priv(test); - t->pool = rte_mempool_create(test->name, /* mempool name */ + if (opt->prod_type == EVT_PROD_TYPE_SYNT || + opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { + t->pool = rte_mempool_create(test->name, /* mempool name */ opt->pool_sz, /* number of elements*/ sizeof(struct perf_elt), /* element size*/ 512, /* cache size*/ 0, NULL, NULL, perf_elt_init, /* obj constructor */ NULL, opt->socket_id, 0); /* flags */ + } else { + t->pool = rte_pktmbuf_pool_create(test->name, /* mempool name */ + opt->pool_sz, /* number of elements*/ + 512, /* cache size*/ + 0, + RTE_MBUF_DEFAULT_BUF_SIZE, + opt->socket_id); /* flags */ + + } + if (t->pool == NULL) { evt_err("failed to create mempool"); return -ENOMEM; @@ -431,10 +809,18 @@ perf_test_setup(struct evt_test *test, struct evt_options *opt) struct test_perf *t = evt_test_priv(test); - t->outstand_pkts = opt->nb_pkts * evt_nr_active_lcores(opt->plcores); + if (opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR) { + t->outstand_pkts = opt->nb_timers * + evt_nr_active_lcores(opt->plcores); + t->nb_pkts = opt->nb_timers; + } else { + t->outstand_pkts = opt->nb_pkts * + evt_nr_active_lcores(opt->plcores); + t->nb_pkts = opt->nb_pkts; + } + t->nb_workers = evt_nr_active_lcores(opt->wlcores); t->done = false; - t->nb_pkts = opt->nb_pkts; t->nb_flows = opt->nb_flows; t->result = EVT_TEST_FAILED; t->opt = opt; diff --git a/dpdk/app/test-eventdev/test_perf_common.h b/dpdk/app/test-eventdev/test_perf_common.h index c6fc70cd..d8fbee6d 100644 --- a/dpdk/app/test-eventdev/test_perf_common.h +++ b/dpdk/app/test-eventdev/test_perf_common.h @@ -1,33 +1,5 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #ifndef _TEST_PERF_COMMON_ @@ -38,7 +10,10 @@ #include #include +#include #include +#include +#include #include #include #include @@ -65,6 +40,7 @@ struct prod_data { struct test_perf *t; } __rte_cache_aligned; + struct test_perf { /* Don't change the offset of "done". Signal handler use this memory * to terminate all lcores work. @@ -80,10 +56,18 @@ struct test_perf { struct worker_data worker[EVT_MAX_PORTS]; struct evt_options *opt; uint8_t sched_type_list[EVT_MAX_STAGES] __rte_cache_aligned; + struct rte_event_timer_adapter *timer_adptr[ + RTE_EVENT_TIMER_ADAPTER_NUM_MAX] __rte_cache_aligned; } __rte_cache_aligned; struct perf_elt { - uint64_t timestamp; + union { + struct rte_event_timer tim; + struct { + char pad[offsetof(struct rte_event_timer, user_meta)]; + uint64_t timestamp; + }; + }; } __rte_cache_aligned; #define BURST_SIZE 16 @@ -94,6 +78,8 @@ struct perf_elt { struct evt_options *opt = t->opt;\ const uint8_t dev = w->dev_id;\ const uint8_t port = w->port_id;\ + const uint8_t prod_timer_type = \ + opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR;\ uint8_t *const sched_type_list = &t->sched_type_list[0];\ struct rte_mempool *const pool = t->pool;\ const uint8_t nb_stages = t->opt->nb_stages;\ @@ -156,15 +142,18 @@ perf_nb_event_ports(struct evt_options *opt) int perf_test_result(struct evt_test *test, struct evt_options *opt); int perf_opt_check(struct evt_options *opt, uint64_t nb_queues); int perf_test_setup(struct evt_test *test, struct evt_options *opt); +int perf_ethdev_setup(struct evt_test *test, struct evt_options *opt); int perf_mempool_setup(struct evt_test *test, struct evt_options *opt); int perf_event_dev_port_setup(struct evt_test *test, struct evt_options *opt, - uint8_t stride, uint8_t nb_queues); + uint8_t stride, uint8_t nb_queues, + const struct rte_event_port_conf *port_conf); int perf_event_dev_service_setup(uint8_t dev_id); int perf_launch_lcores(struct evt_test *test, struct evt_options *opt, int (*worker)(void *)); void perf_opt_dump(struct evt_options *opt, uint8_t nb_queues); void perf_test_destroy(struct evt_test *test, struct evt_options *opt); void perf_eventdev_destroy(struct evt_test *test, struct evt_options *opt); +void perf_ethdev_destroy(struct evt_test *test, struct evt_options *opt); void perf_mempool_destroy(struct evt_test *test, struct evt_options *opt); #endif /* _TEST_PERF_COMMON_ */ diff --git a/dpdk/app/test-eventdev/test_perf_queue.c b/dpdk/app/test-eventdev/test_perf_queue.c index d843eea1..8efdec6f 100644 --- a/dpdk/app/test-eventdev/test_perf_queue.c +++ b/dpdk/app/test-eventdev/test_perf_queue.c @@ -1,44 +1,18 @@ -/* - * BSD LICENSE - * - * Copyright (C) Cavium, Inc 2017. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Cavium, Inc nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc */ #include "test_perf_common.h" -/* See http://dpdk.org/doc/guides/tools/testeventdev.html for test details */ +/* See http://doc.dpdk.org/guides/tools/testeventdev.html for test details */ static inline int perf_queue_nb_event_queues(struct evt_options *opt) { /* nb_queues = number of producers * number of stages */ - return evt_nr_active_lcores(opt->plcores) * opt->nb_stages; + uint8_t nb_prod = opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR ? + rte_eth_dev_count_avail() : evt_nr_active_lcores(opt->plcores); + return nb_prod * opt->nb_stages; } static inline __attribute__((always_inline)) void @@ -75,7 +49,7 @@ perf_queue_worker(void *arg, const int enable_fwd_latency) rte_pause(); continue; } - if (enable_fwd_latency) + if (enable_fwd_latency && !prod_timer_type) /* first q in pipeline, mark timestamp to compute fwd latency */ mark_fwd_latency(&ev, nb_stages); @@ -114,7 +88,7 @@ perf_queue_worker_burst(void *arg, const int enable_fwd_latency) } for (i = 0; i < nb_rx; i++) { - if (enable_fwd_latency) { + if (enable_fwd_latency && !prod_timer_type) { rte_prefetch0(ev[i+1].event_ptr); /* first queue in pipeline. * mark time stamp to compute fwd latency @@ -182,14 +156,33 @@ perf_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) uint8_t queue; int nb_stages = opt->nb_stages; int ret; + int nb_ports; + int nb_queues; + struct rte_event_dev_info dev_info; + + nb_ports = evt_nr_active_lcores(opt->wlcores); + nb_ports += opt->prod_type == EVT_PROD_TYPE_ETH_RX_ADPTR || + opt->prod_type == EVT_PROD_TYPE_EVENT_TIMER_ADPTR ? 0 : + evt_nr_active_lcores(opt->plcores); + + nb_queues = perf_queue_nb_event_queues(opt); + + memset(&dev_info, 0, sizeof(struct rte_event_dev_info)); + ret = rte_event_dev_info_get(opt->dev_id, &dev_info); + if (ret) { + evt_err("failed to get eventdev info %d", opt->dev_id); + return ret; + } const struct rte_event_dev_config config = { - .nb_event_queues = perf_queue_nb_event_queues(opt), - .nb_event_ports = perf_nb_event_ports(opt), - .nb_events_limit = 4096, + .nb_event_queues = nb_queues, + .nb_event_ports = nb_ports, + .nb_events_limit = dev_info.max_num_events, .nb_event_queue_flows = opt->nb_flows, - .nb_event_port_dequeue_depth = 128, - .nb_event_port_enqueue_depth = 128, + .nb_event_port_dequeue_depth = + dev_info.max_event_port_dequeue_depth, + .nb_event_port_enqueue_depth = + dev_info.max_event_port_enqueue_depth, }; ret = rte_event_dev_configure(opt->dev_id, &config); @@ -204,7 +197,7 @@ perf_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) .nb_atomic_order_sequences = opt->nb_flows, }; /* queue configurations */ - for (queue = 0; queue < perf_queue_nb_event_queues(opt); queue++) { + for (queue = 0; queue < nb_queues; queue++) { q_conf.schedule_type = (opt->sched_type_list[queue % nb_stages]); @@ -227,15 +220,29 @@ perf_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) } } + if (opt->wkr_deq_dep > dev_info.max_event_port_dequeue_depth) + opt->wkr_deq_dep = dev_info.max_event_port_dequeue_depth; + + /* port configuration */ + const struct rte_event_port_conf p_conf = { + .dequeue_depth = opt->wkr_deq_dep, + .enqueue_depth = dev_info.max_event_port_dequeue_depth, + .new_event_threshold = dev_info.max_num_events, + }; + ret = perf_event_dev_port_setup(test, opt, nb_stages /* stride */, - perf_queue_nb_event_queues(opt)); + nb_queues, &p_conf); if (ret) return ret; - ret = evt_service_setup(opt->dev_id); - if (ret) { - evt_err("No service lcore found to run event dev."); - return ret; + if (!evt_has_distributed_sched(opt->dev_id)) { + uint32_t service_id; + rte_event_dev_service_id_get(opt->dev_id, &service_id); + ret = evt_service_setup(service_id); + if (ret) { + evt_err("No service lcore found to run event dev."); + return ret; + } } ret = rte_event_dev_start(opt->dev_id); @@ -283,10 +290,12 @@ static const struct evt_test_ops perf_queue = { .opt_dump = perf_queue_opt_dump, .test_setup = perf_test_setup, .mempool_setup = perf_mempool_setup, + .ethdev_setup = perf_ethdev_setup, .eventdev_setup = perf_queue_eventdev_setup, .launch_lcores = perf_queue_launch_lcores, .eventdev_destroy = perf_eventdev_destroy, .mempool_destroy = perf_mempool_destroy, + .ethdev_destroy = perf_ethdev_destroy, .test_result = perf_test_result, .test_destroy = perf_test_destroy, }; diff --git a/dpdk/app/test-eventdev/test_pipeline_atq.c b/dpdk/app/test-eventdev/test_pipeline_atq.c new file mode 100644 index 00000000..fc4cb3bb --- /dev/null +++ b/dpdk/app/test-eventdev/test_pipeline_atq.c @@ -0,0 +1,527 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 Cavium, Inc. + */ + +#include "test_pipeline_common.h" + +/* See http://doc.dpdk.org/guides/tools/testeventdev.html for test details */ + +static __rte_always_inline int +pipeline_atq_nb_event_queues(struct evt_options *opt) +{ + RTE_SET_USED(opt); + + return rte_eth_dev_count_avail(); +} + +static __rte_noinline int +pipeline_atq_worker_single_stage_tx(void *arg) +{ + PIPELINE_WORKER_SINGLE_STAGE_INIT; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + pipeline_event_tx(dev, port, &ev); + w->processed_pkts++; + } + + return 0; +} + +static __rte_noinline int +pipeline_atq_worker_single_stage_fwd(void *arg) +{ + PIPELINE_WORKER_SINGLE_STAGE_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + ev.queue_id = tx_queue[ev.mbuf->port]; + pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); + pipeline_event_enqueue(dev, port, &ev); + w->processed_pkts++; + } + + return 0; +} + +static __rte_noinline int +pipeline_atq_worker_single_stage_burst_tx(void *arg) +{ + PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0); + } + + pipeline_event_tx_burst(dev, port, ev, nb_rx); + w->processed_pkts += nb_rx; + } + + return 0; +} + +static __rte_noinline int +pipeline_atq_worker_single_stage_burst_fwd(void *arg) +{ + PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0); + ev[i].queue_id = tx_queue[ev[i].mbuf->port]; + pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC); + } + + pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + w->processed_pkts += nb_rx; + } + + return 0; +} + +static __rte_noinline int +pipeline_atq_worker_multi_stage_tx(void *arg) +{ + PIPELINE_WORKER_MULTI_STAGE_INIT; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + cq_id = ev.sub_event_type % nb_stages; + + if (cq_id == last_queue) { + pipeline_event_tx(dev, port, &ev); + w->processed_pkts++; + continue; + } + + ev.sub_event_type++; + pipeline_fwd_event(&ev, sched_type_list[cq_id]); + pipeline_event_enqueue(dev, port, &ev); + } + + return 0; +} + +static __rte_noinline int +pipeline_atq_worker_multi_stage_fwd(void *arg) +{ + PIPELINE_WORKER_MULTI_STAGE_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + cq_id = ev.sub_event_type % nb_stages; + + if (cq_id == last_queue) { + ev.queue_id = tx_queue[ev.mbuf->port]; + pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); + w->processed_pkts++; + } else { + ev.sub_event_type++; + pipeline_fwd_event(&ev, sched_type_list[cq_id]); + } + + pipeline_event_enqueue(dev, port, &ev); + } + + return 0; +} + +static __rte_noinline int +pipeline_atq_worker_multi_stage_burst_tx(void *arg) +{ + PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + cq_id = ev[i].sub_event_type % nb_stages; + + if (cq_id == last_queue) { + pipeline_event_tx(dev, port, &ev[i]); + ev[i].op = RTE_EVENT_OP_RELEASE; + w->processed_pkts++; + continue; + } + + ev[i].sub_event_type++; + pipeline_fwd_event(&ev[i], sched_type_list[cq_id]); + } + + pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + } + + return 0; +} + +static __rte_noinline int +pipeline_atq_worker_multi_stage_burst_fwd(void *arg) +{ + PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + cq_id = ev[i].sub_event_type % nb_stages; + + if (cq_id == last_queue) { + w->processed_pkts++; + ev[i].queue_id = tx_queue[ev[i].mbuf->port]; + pipeline_fwd_event(&ev[i], + RTE_SCHED_TYPE_ATOMIC); + } else { + ev[i].sub_event_type++; + pipeline_fwd_event(&ev[i], + sched_type_list[cq_id]); + } + } + + pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + } + + return 0; +} + +static int +worker_wrapper(void *arg) +{ + struct worker_data *w = arg; + struct evt_options *opt = w->t->opt; + const bool burst = evt_has_burst_mode(w->dev_id); + const bool internal_port = w->t->internal_port; + const uint8_t nb_stages = opt->nb_stages; + RTE_SET_USED(opt); + + if (nb_stages == 1) { + if (!burst && internal_port) + return pipeline_atq_worker_single_stage_tx(arg); + else if (!burst && !internal_port) + return pipeline_atq_worker_single_stage_fwd(arg); + else if (burst && internal_port) + return pipeline_atq_worker_single_stage_burst_tx(arg); + else if (burst && !internal_port) + return pipeline_atq_worker_single_stage_burst_fwd(arg); + } else { + if (!burst && internal_port) + return pipeline_atq_worker_multi_stage_tx(arg); + else if (!burst && !internal_port) + return pipeline_atq_worker_multi_stage_fwd(arg); + if (burst && internal_port) + return pipeline_atq_worker_multi_stage_burst_tx(arg); + else if (burst && !internal_port) + return pipeline_atq_worker_multi_stage_burst_fwd(arg); + } + + rte_panic("invalid worker\n"); +} + +static int +pipeline_atq_launch_lcores(struct evt_test *test, struct evt_options *opt) +{ + return pipeline_launch_lcores(test, opt, worker_wrapper); +} + +static int +pipeline_atq_eventdev_setup(struct evt_test *test, struct evt_options *opt) +{ + int ret; + int nb_ports; + int nb_queues; + uint8_t queue; + uint8_t tx_evqueue_id[RTE_MAX_ETHPORTS]; + uint8_t queue_arr[RTE_EVENT_MAX_QUEUES_PER_DEV]; + uint8_t nb_worker_queues = 0; + uint8_t tx_evport_id = 0; + uint16_t prod = 0; + struct rte_event_dev_info info; + struct test_pipeline *t = evt_test_priv(test); + + nb_ports = evt_nr_active_lcores(opt->wlcores); + nb_queues = rte_eth_dev_count_avail(); + + memset(tx_evqueue_id, 0, sizeof(uint8_t) * RTE_MAX_ETHPORTS); + memset(queue_arr, 0, sizeof(uint8_t) * RTE_EVENT_MAX_QUEUES_PER_DEV); + /* One queue for Tx adapter per port */ + if (!t->internal_port) { + RTE_ETH_FOREACH_DEV(prod) { + tx_evqueue_id[prod] = nb_queues; + nb_queues++; + } + } + + rte_event_dev_info_get(opt->dev_id, &info); + + const struct rte_event_dev_config config = { + .nb_event_queues = nb_queues, + .nb_event_ports = nb_ports, + .nb_events_limit = info.max_num_events, + .nb_event_queue_flows = opt->nb_flows, + .nb_event_port_dequeue_depth = + info.max_event_port_dequeue_depth, + .nb_event_port_enqueue_depth = + info.max_event_port_enqueue_depth, + }; + ret = rte_event_dev_configure(opt->dev_id, &config); + if (ret) { + evt_err("failed to configure eventdev %d", opt->dev_id); + return ret; + } + + struct rte_event_queue_conf q_conf = { + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .nb_atomic_flows = opt->nb_flows, + .nb_atomic_order_sequences = opt->nb_flows, + }; + /* queue configurations */ + for (queue = 0; queue < nb_queues; queue++) { + q_conf.event_queue_cfg = RTE_EVENT_QUEUE_CFG_ALL_TYPES; + + if (!t->internal_port) { + RTE_ETH_FOREACH_DEV(prod) { + if (queue == tx_evqueue_id[prod]) { + q_conf.event_queue_cfg = + RTE_EVENT_QUEUE_CFG_SINGLE_LINK; + } else { + queue_arr[nb_worker_queues] = queue; + nb_worker_queues++; + } + } + } + + ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf); + if (ret) { + evt_err("failed to setup queue=%d", queue); + return ret; + } + } + + if (opt->wkr_deq_dep > info.max_event_port_dequeue_depth) + opt->wkr_deq_dep = info.max_event_port_dequeue_depth; + + /* port configuration */ + const struct rte_event_port_conf p_conf = { + .dequeue_depth = opt->wkr_deq_dep, + .enqueue_depth = info.max_event_port_dequeue_depth, + .new_event_threshold = info.max_num_events, + }; + + if (!t->internal_port) + ret = pipeline_event_port_setup(test, opt, queue_arr, + nb_worker_queues, p_conf); + else + ret = pipeline_event_port_setup(test, opt, NULL, nb_queues, + p_conf); + + if (ret) + return ret; + + /* + * The pipelines are setup in the following manner: + * + * eth_dev_count = 2, nb_stages = 2, atq mode + * + * eth0, eth1 have Internal port capability : + * queues = 2 + * stride = 1 + * + * event queue pipelines: + * eth0 -> q0 ->Tx + * eth1 -> q1 ->Tx + * + * q0, q1 are configured as ATQ so, all the different stages can + * be enqueued on the same queue. + * + * eth0, eth1 use Tx adapters service core : + * queues = 4 + * stride = 1 + * + * event queue pipelines: + * eth0 -> q0 -> q2 -> Tx + * eth1 -> q1 -> q3 -> Tx + * + * q0, q1 are configured as stated above. + * q2, q3 configured as SINGLE_LINK. + */ + ret = pipeline_event_rx_adapter_setup(opt, 1, p_conf); + if (ret) + return ret; + ret = pipeline_event_tx_adapter_setup(opt, p_conf); + if (ret) + return ret; + + if (!evt_has_distributed_sched(opt->dev_id)) { + uint32_t service_id; + rte_event_dev_service_id_get(opt->dev_id, &service_id); + ret = evt_service_setup(service_id); + if (ret) { + evt_err("No service lcore found to run event dev."); + return ret; + } + } + + /* Connect the tx_evqueue_id to the Tx adapter port */ + if (!t->internal_port) { + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_event_eth_tx_adapter_event_port_get(prod, + &tx_evport_id); + if (ret) { + evt_err("Unable to get Tx adapter[%d]", prod); + return ret; + } + + if (rte_event_port_link(opt->dev_id, tx_evport_id, + &tx_evqueue_id[prod], + NULL, 1) != 1) { + evt_err("Unable to link Tx adptr[%d] evprt[%d]", + prod, tx_evport_id); + return ret; + } + } + } + + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_eth_dev_start(prod); + if (ret) { + evt_err("Ethernet dev [%d] failed to start." + " Using synthetic producer", prod); + return ret; + } + } + + ret = rte_event_dev_start(opt->dev_id); + if (ret) { + evt_err("failed to start eventdev %d", opt->dev_id); + return ret; + } + + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_event_eth_rx_adapter_start(prod); + if (ret) { + evt_err("Rx adapter[%d] start failed", prod); + return ret; + } + + ret = rte_event_eth_tx_adapter_start(prod); + if (ret) { + evt_err("Tx adapter[%d] start failed", prod); + return ret; + } + } + + memcpy(t->tx_evqueue_id, tx_evqueue_id, sizeof(uint8_t) * + RTE_MAX_ETHPORTS); + + return 0; +} + +static void +pipeline_atq_opt_dump(struct evt_options *opt) +{ + pipeline_opt_dump(opt, pipeline_atq_nb_event_queues(opt)); +} + +static int +pipeline_atq_opt_check(struct evt_options *opt) +{ + return pipeline_opt_check(opt, pipeline_atq_nb_event_queues(opt)); +} + +static bool +pipeline_atq_capability_check(struct evt_options *opt) +{ + struct rte_event_dev_info dev_info; + + rte_event_dev_info_get(opt->dev_id, &dev_info); + if (dev_info.max_event_queues < pipeline_atq_nb_event_queues(opt) || + dev_info.max_event_ports < + evt_nr_active_lcores(opt->wlcores)) { + evt_err("not enough eventdev queues=%d/%d or ports=%d/%d", + pipeline_atq_nb_event_queues(opt), + dev_info.max_event_queues, + evt_nr_active_lcores(opt->wlcores), + dev_info.max_event_ports); + } + + return true; +} + +static const struct evt_test_ops pipeline_atq = { + .cap_check = pipeline_atq_capability_check, + .opt_check = pipeline_atq_opt_check, + .opt_dump = pipeline_atq_opt_dump, + .test_setup = pipeline_test_setup, + .mempool_setup = pipeline_mempool_setup, + .ethdev_setup = pipeline_ethdev_setup, + .eventdev_setup = pipeline_atq_eventdev_setup, + .launch_lcores = pipeline_atq_launch_lcores, + .eventdev_destroy = pipeline_eventdev_destroy, + .mempool_destroy = pipeline_mempool_destroy, + .ethdev_destroy = pipeline_ethdev_destroy, + .test_result = pipeline_test_result, + .test_destroy = pipeline_test_destroy, +}; + +EVT_TEST_REGISTER(pipeline_atq); diff --git a/dpdk/app/test-eventdev/test_pipeline_common.c b/dpdk/app/test-eventdev/test_pipeline_common.c new file mode 100644 index 00000000..5db3ffde --- /dev/null +++ b/dpdk/app/test-eventdev/test_pipeline_common.c @@ -0,0 +1,467 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 Cavium, Inc. + */ + +#include "test_pipeline_common.h" + +int +pipeline_test_result(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(opt); + int i; + uint64_t total = 0; + struct test_pipeline *t = evt_test_priv(test); + + evt_info("Packet distribution across worker cores :"); + for (i = 0; i < t->nb_workers; i++) + total += t->worker[i].processed_pkts; + for (i = 0; i < t->nb_workers; i++) + evt_info("Worker %d packets: "CLGRN"%"PRIx64""CLNRM" percentage:" + CLGRN" %3.2f"CLNRM, i, + t->worker[i].processed_pkts, + (((double)t->worker[i].processed_pkts)/total) + * 100); + return t->result; +} + +void +pipeline_opt_dump(struct evt_options *opt, uint8_t nb_queues) +{ + evt_dump("nb_worker_lcores", "%d", evt_nr_active_lcores(opt->wlcores)); + evt_dump_worker_lcores(opt); + evt_dump_nb_stages(opt); + evt_dump("nb_evdev_ports", "%d", pipeline_nb_event_ports(opt)); + evt_dump("nb_evdev_queues", "%d", nb_queues); + evt_dump_queue_priority(opt); + evt_dump_sched_type_list(opt); + evt_dump_producer_type(opt); +} + +static inline uint64_t +processed_pkts(struct test_pipeline *t) +{ + uint8_t i; + uint64_t total = 0; + + rte_smp_rmb(); + for (i = 0; i < t->nb_workers; i++) + total += t->worker[i].processed_pkts; + + return total; +} + +int +pipeline_launch_lcores(struct evt_test *test, struct evt_options *opt, + int (*worker)(void *)) +{ + int ret, lcore_id; + struct test_pipeline *t = evt_test_priv(test); + + int port_idx = 0; + /* launch workers */ + RTE_LCORE_FOREACH_SLAVE(lcore_id) { + if (!(opt->wlcores[lcore_id])) + continue; + + ret = rte_eal_remote_launch(worker, + &t->worker[port_idx], lcore_id); + if (ret) { + evt_err("failed to launch worker %d", lcore_id); + return ret; + } + port_idx++; + } + + uint64_t perf_cycles = rte_get_timer_cycles(); + const uint64_t perf_sample = rte_get_timer_hz(); + + static float total_mpps; + static uint64_t samples; + + uint64_t prev_pkts = 0; + + while (t->done == false) { + const uint64_t new_cycles = rte_get_timer_cycles(); + + if ((new_cycles - perf_cycles) > perf_sample) { + const uint64_t curr_pkts = processed_pkts(t); + + float mpps = (float)(curr_pkts - prev_pkts)/1000000; + + prev_pkts = curr_pkts; + perf_cycles = new_cycles; + total_mpps += mpps; + ++samples; + printf(CLGRN"\r%.3f mpps avg %.3f mpps"CLNRM, + mpps, total_mpps/samples); + fflush(stdout); + } + } + printf("\n"); + return 0; +} + +int +pipeline_opt_check(struct evt_options *opt, uint64_t nb_queues) +{ + unsigned int lcores; + /* + * N worker + 1 master + */ + lcores = 2; + + if (!rte_eth_dev_count_avail()) { + evt_err("test needs minimum 1 ethernet dev"); + return -1; + } + + if (rte_lcore_count() < lcores) { + evt_err("test need minimum %d lcores", lcores); + return -1; + } + + /* Validate worker lcores */ + if (evt_lcores_has_overlap(opt->wlcores, rte_get_master_lcore())) { + evt_err("worker lcores overlaps with master lcore"); + return -1; + } + if (evt_has_disabled_lcore(opt->wlcores)) { + evt_err("one or more workers lcores are not enabled"); + return -1; + } + if (!evt_has_active_lcore(opt->wlcores)) { + evt_err("minimum one worker is required"); + return -1; + } + + if (nb_queues > EVT_MAX_QUEUES) { + evt_err("number of queues exceeds %d", EVT_MAX_QUEUES); + return -1; + } + if (pipeline_nb_event_ports(opt) > EVT_MAX_PORTS) { + evt_err("number of ports exceeds %d", EVT_MAX_PORTS); + return -1; + } + + if (evt_has_invalid_stage(opt)) + return -1; + + if (evt_has_invalid_sched_type(opt)) + return -1; + + return 0; +} + +#define NB_RX_DESC 128 +#define NB_TX_DESC 512 +int +pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt) +{ + uint16_t i; + uint8_t nb_queues = 1; + struct test_pipeline *t = evt_test_priv(test); + struct rte_eth_rxconf rx_conf; + struct rte_eth_conf port_conf = { + .rxmode = { + .mq_mode = ETH_MQ_RX_RSS, + .max_rx_pkt_len = ETHER_MAX_LEN, + }, + .rx_adv_conf = { + .rss_conf = { + .rss_key = NULL, + .rss_hf = ETH_RSS_IP, + }, + }, + }; + + RTE_SET_USED(opt); + if (!rte_eth_dev_count_avail()) { + evt_err("No ethernet ports found."); + return -ENODEV; + } + + t->internal_port = 1; + RTE_ETH_FOREACH_DEV(i) { + struct rte_eth_dev_info dev_info; + struct rte_eth_conf local_port_conf = port_conf; + uint32_t caps = 0; + + rte_event_eth_tx_adapter_caps_get(opt->dev_id, i, &caps); + if (!(caps & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT)) + t->internal_port = 0; + + rte_eth_dev_info_get(i, &dev_info); + rx_conf = dev_info.default_rxconf; + rx_conf.offloads = port_conf.rxmode.offloads; + + local_port_conf.rx_adv_conf.rss_conf.rss_hf &= + dev_info.flow_type_rss_offloads; + if (local_port_conf.rx_adv_conf.rss_conf.rss_hf != + port_conf.rx_adv_conf.rss_conf.rss_hf) { + evt_info("Port %u modified RSS hash function based on hardware support," + "requested:%#"PRIx64" configured:%#"PRIx64"", + i, + port_conf.rx_adv_conf.rss_conf.rss_hf, + local_port_conf.rx_adv_conf.rss_conf.rss_hf); + } + + if (rte_eth_dev_configure(i, nb_queues, nb_queues, + &local_port_conf) + < 0) { + evt_err("Failed to configure eth port [%d]", i); + return -EINVAL; + } + + if (rte_eth_rx_queue_setup(i, 0, NB_RX_DESC, + rte_socket_id(), &rx_conf, t->pool) < 0) { + evt_err("Failed to setup eth port [%d] rx_queue: %d.", + i, 0); + return -EINVAL; + } + if (rte_eth_tx_queue_setup(i, 0, NB_TX_DESC, + rte_socket_id(), NULL) < 0) { + evt_err("Failed to setup eth port [%d] tx_queue: %d.", + i, 0); + return -EINVAL; + } + + rte_eth_promiscuous_enable(i); + } + + return 0; +} + +int +pipeline_event_port_setup(struct evt_test *test, struct evt_options *opt, + uint8_t *queue_arr, uint8_t nb_queues, + const struct rte_event_port_conf p_conf) +{ + int ret; + uint8_t port; + struct test_pipeline *t = evt_test_priv(test); + + + /* setup one port per worker, linking to all queues */ + for (port = 0; port < evt_nr_active_lcores(opt->wlcores); port++) { + struct worker_data *w = &t->worker[port]; + + w->dev_id = opt->dev_id; + w->port_id = port; + w->t = t; + w->processed_pkts = 0; + + ret = rte_event_port_setup(opt->dev_id, port, &p_conf); + if (ret) { + evt_err("failed to setup port %d", port); + return ret; + } + + if (rte_event_port_link(opt->dev_id, port, queue_arr, NULL, + nb_queues) != nb_queues) + goto link_fail; + } + + return 0; + +link_fail: + evt_err("failed to link queues to port %d", port); + return -EINVAL; +} + +int +pipeline_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride, + struct rte_event_port_conf prod_conf) +{ + int ret = 0; + uint16_t prod; + struct rte_event_eth_rx_adapter_queue_conf queue_conf; + + memset(&queue_conf, 0, + sizeof(struct rte_event_eth_rx_adapter_queue_conf)); + queue_conf.ev.sched_type = opt->sched_type_list[0]; + RTE_ETH_FOREACH_DEV(prod) { + uint32_t cap; + + ret = rte_event_eth_rx_adapter_caps_get(opt->dev_id, + prod, &cap); + if (ret) { + evt_err("failed to get event rx adapter[%d]" + " capabilities", + opt->dev_id); + return ret; + } + queue_conf.ev.queue_id = prod * stride; + ret = rte_event_eth_rx_adapter_create(prod, opt->dev_id, + &prod_conf); + if (ret) { + evt_err("failed to create rx adapter[%d]", prod); + return ret; + } + ret = rte_event_eth_rx_adapter_queue_add(prod, prod, -1, + &queue_conf); + if (ret) { + evt_err("failed to add rx queues to adapter[%d]", prod); + return ret; + } + + if (!(cap & RTE_EVENT_ETH_RX_ADAPTER_CAP_INTERNAL_PORT)) { + uint32_t service_id; + + rte_event_eth_rx_adapter_service_id_get(prod, + &service_id); + ret = evt_service_setup(service_id); + if (ret) { + evt_err("Failed to setup service core" + " for Rx adapter"); + return ret; + } + } + + evt_info("Port[%d] using Rx adapter[%d] configured", prod, + prod); + } + + return ret; +} + +int +pipeline_event_tx_adapter_setup(struct evt_options *opt, + struct rte_event_port_conf port_conf) +{ + int ret = 0; + uint16_t consm; + + RTE_ETH_FOREACH_DEV(consm) { + uint32_t cap; + + ret = rte_event_eth_tx_adapter_caps_get(opt->dev_id, + consm, &cap); + if (ret) { + evt_err("failed to get event tx adapter[%d] caps", + consm); + return ret; + } + + ret = rte_event_eth_tx_adapter_create(consm, opt->dev_id, + &port_conf); + if (ret) { + evt_err("failed to create tx adapter[%d]", consm); + return ret; + } + + ret = rte_event_eth_tx_adapter_queue_add(consm, consm, -1); + if (ret) { + evt_err("failed to add tx queues to adapter[%d]", + consm); + return ret; + } + + if (!(cap & RTE_EVENT_ETH_TX_ADAPTER_CAP_INTERNAL_PORT)) { + uint32_t service_id; + + rte_event_eth_tx_adapter_service_id_get(consm, + &service_id); + ret = evt_service_setup(service_id); + if (ret) { + evt_err("Failed to setup service core" + " for Tx adapter\n"); + return ret; + } + } + + evt_info("Port[%d] using Tx adapter[%d] Configured", consm, + consm); + } + + return ret; +} + +void +pipeline_ethdev_destroy(struct evt_test *test, struct evt_options *opt) +{ + uint16_t i; + RTE_SET_USED(test); + RTE_SET_USED(opt); + + RTE_ETH_FOREACH_DEV(i) { + rte_event_eth_rx_adapter_stop(i); + rte_event_eth_tx_adapter_stop(i); + rte_eth_dev_stop(i); + } +} + +void +pipeline_eventdev_destroy(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(test); + + rte_event_dev_close(opt->dev_id); +} + +int +pipeline_mempool_setup(struct evt_test *test, struct evt_options *opt) +{ + struct test_pipeline *t = evt_test_priv(test); + + t->pool = rte_pktmbuf_pool_create(test->name, /* mempool name */ + opt->pool_sz, /* number of elements*/ + 512, /* cache size*/ + 0, + RTE_MBUF_DEFAULT_BUF_SIZE, + opt->socket_id); /* flags */ + + if (t->pool == NULL) { + evt_err("failed to create mempool"); + return -ENOMEM; + } + + return 0; +} + +void +pipeline_mempool_destroy(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(opt); + struct test_pipeline *t = evt_test_priv(test); + + rte_mempool_free(t->pool); +} + +int +pipeline_test_setup(struct evt_test *test, struct evt_options *opt) +{ + void *test_pipeline; + + test_pipeline = rte_zmalloc_socket(test->name, + sizeof(struct test_pipeline), RTE_CACHE_LINE_SIZE, + opt->socket_id); + if (test_pipeline == NULL) { + evt_err("failed to allocate test_pipeline memory"); + goto nomem; + } + test->test_priv = test_pipeline; + + struct test_pipeline *t = evt_test_priv(test); + + t->nb_workers = evt_nr_active_lcores(opt->wlcores); + t->outstand_pkts = opt->nb_pkts * evt_nr_active_lcores(opt->wlcores); + t->done = false; + t->nb_flows = opt->nb_flows; + t->result = EVT_TEST_FAILED; + t->opt = opt; + opt->prod_type = EVT_PROD_TYPE_ETH_RX_ADPTR; + memcpy(t->sched_type_list, opt->sched_type_list, + sizeof(opt->sched_type_list)); + return 0; +nomem: + return -ENOMEM; +} + +void +pipeline_test_destroy(struct evt_test *test, struct evt_options *opt) +{ + RTE_SET_USED(opt); + + rte_free(test->test_priv); +} diff --git a/dpdk/app/test-eventdev/test_pipeline_common.h b/dpdk/app/test-eventdev/test_pipeline_common.h new file mode 100644 index 00000000..0440b9e2 --- /dev/null +++ b/dpdk/app/test-eventdev/test_pipeline_common.h @@ -0,0 +1,173 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 Cavium, Inc. + */ + +#ifndef _TEST_PIPELINE_COMMON_ +#define _TEST_PIPELINE_COMMON_ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "evt_common.h" +#include "evt_options.h" +#include "evt_test.h" + +struct test_pipeline; + +struct worker_data { + uint64_t processed_pkts; + uint8_t dev_id; + uint8_t port_id; + struct test_pipeline *t; +} __rte_cache_aligned; + +struct test_pipeline { + /* Don't change the offset of "done". Signal handler use this memory + * to terminate all lcores work. + */ + int done; + uint8_t nb_workers; + uint8_t internal_port; + uint8_t tx_evqueue_id[RTE_MAX_ETHPORTS]; + enum evt_test_result result; + uint32_t nb_flows; + uint64_t outstand_pkts; + struct rte_mempool *pool; + struct worker_data worker[EVT_MAX_PORTS]; + struct evt_options *opt; + uint8_t sched_type_list[EVT_MAX_STAGES] __rte_cache_aligned; +} __rte_cache_aligned; + +#define BURST_SIZE 16 + +#define PIPELINE_WORKER_SINGLE_STAGE_INIT \ + struct worker_data *w = arg; \ + struct test_pipeline *t = w->t; \ + const uint8_t dev = w->dev_id; \ + const uint8_t port = w->port_id; \ + struct rte_event ev __rte_cache_aligned + +#define PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT \ + int i; \ + struct worker_data *w = arg; \ + struct test_pipeline *t = w->t; \ + const uint8_t dev = w->dev_id; \ + const uint8_t port = w->port_id; \ + struct rte_event ev[BURST_SIZE + 1] __rte_cache_aligned + +#define PIPELINE_WORKER_MULTI_STAGE_INIT \ + struct worker_data *w = arg; \ + struct test_pipeline *t = w->t; \ + uint8_t cq_id; \ + const uint8_t dev = w->dev_id; \ + const uint8_t port = w->port_id; \ + const uint8_t last_queue = t->opt->nb_stages - 1; \ + uint8_t *const sched_type_list = &t->sched_type_list[0]; \ + const uint8_t nb_stages = t->opt->nb_stages + 1; \ + struct rte_event ev __rte_cache_aligned + +#define PIPELINE_WORKER_MULTI_STAGE_BURST_INIT \ + int i; \ + struct worker_data *w = arg; \ + struct test_pipeline *t = w->t; \ + uint8_t cq_id; \ + const uint8_t dev = w->dev_id; \ + const uint8_t port = w->port_id; \ + const uint8_t last_queue = t->opt->nb_stages - 1; \ + uint8_t *const sched_type_list = &t->sched_type_list[0]; \ + const uint8_t nb_stages = t->opt->nb_stages + 1; \ + struct rte_event ev[BURST_SIZE + 1] __rte_cache_aligned + +static __rte_always_inline void +pipeline_fwd_event(struct rte_event *ev, uint8_t sched) +{ + ev->event_type = RTE_EVENT_TYPE_CPU; + ev->op = RTE_EVENT_OP_FORWARD; + ev->sched_type = sched; +} + +static __rte_always_inline void +pipeline_event_tx(const uint8_t dev, const uint8_t port, + struct rte_event * const ev) +{ + rte_event_eth_tx_adapter_txq_set(ev->mbuf, 0); + while (!rte_event_eth_tx_adapter_enqueue(dev, port, ev, 1)) + rte_pause(); +} + +static __rte_always_inline void +pipeline_event_tx_burst(const uint8_t dev, const uint8_t port, + struct rte_event *ev, const uint16_t nb_rx) +{ + uint16_t enq; + + enq = rte_event_eth_tx_adapter_enqueue(dev, port, ev, nb_rx); + while (enq < nb_rx) { + enq += rte_event_eth_tx_adapter_enqueue(dev, port, + ev + enq, nb_rx - enq); + } +} + +static __rte_always_inline void +pipeline_event_enqueue(const uint8_t dev, const uint8_t port, + struct rte_event *ev) +{ + while (rte_event_enqueue_burst(dev, port, ev, 1) != 1) + rte_pause(); +} + +static __rte_always_inline void +pipeline_event_enqueue_burst(const uint8_t dev, const uint8_t port, + struct rte_event *ev, const uint16_t nb_rx) +{ + uint16_t enq; + + enq = rte_event_enqueue_burst(dev, port, ev, nb_rx); + while (enq < nb_rx) { + enq += rte_event_enqueue_burst(dev, port, + ev + enq, nb_rx - enq); + } +} + +static inline int +pipeline_nb_event_ports(struct evt_options *opt) +{ + return evt_nr_active_lcores(opt->wlcores); +} + +int pipeline_test_result(struct evt_test *test, struct evt_options *opt); +int pipeline_opt_check(struct evt_options *opt, uint64_t nb_queues); +int pipeline_test_setup(struct evt_test *test, struct evt_options *opt); +int pipeline_ethdev_setup(struct evt_test *test, struct evt_options *opt); +int pipeline_event_rx_adapter_setup(struct evt_options *opt, uint8_t stride, + struct rte_event_port_conf prod_conf); +int pipeline_event_tx_adapter_setup(struct evt_options *opt, + struct rte_event_port_conf prod_conf); +int pipeline_mempool_setup(struct evt_test *test, struct evt_options *opt); +int pipeline_event_port_setup(struct evt_test *test, struct evt_options *opt, + uint8_t *queue_arr, uint8_t nb_queues, + const struct rte_event_port_conf p_conf); +int pipeline_launch_lcores(struct evt_test *test, struct evt_options *opt, + int (*worker)(void *)); +void pipeline_opt_dump(struct evt_options *opt, uint8_t nb_queues); +void pipeline_test_destroy(struct evt_test *test, struct evt_options *opt); +void pipeline_eventdev_destroy(struct evt_test *test, struct evt_options *opt); +void pipeline_ethdev_destroy(struct evt_test *test, struct evt_options *opt); +void pipeline_mempool_destroy(struct evt_test *test, struct evt_options *opt); + +#endif /* _TEST_PIPELINE_COMMON_ */ diff --git a/dpdk/app/test-eventdev/test_pipeline_queue.c b/dpdk/app/test-eventdev/test_pipeline_queue.c new file mode 100644 index 00000000..e35e9eb4 --- /dev/null +++ b/dpdk/app/test-eventdev/test_pipeline_queue.c @@ -0,0 +1,542 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 Cavium, Inc. + */ + +#include "test_pipeline_common.h" + +/* See http://doc.dpdk.org/guides/tools/testeventdev.html for test details */ + +static __rte_always_inline int +pipeline_queue_nb_event_queues(struct evt_options *opt) +{ + uint16_t eth_count = rte_eth_dev_count_avail(); + + return (eth_count * opt->nb_stages) + eth_count; +} + +static __rte_noinline int +pipeline_queue_worker_single_stage_tx(void *arg) +{ + PIPELINE_WORKER_SINGLE_STAGE_INIT; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + if (ev.sched_type == RTE_SCHED_TYPE_ATOMIC) { + pipeline_event_tx(dev, port, &ev); + w->processed_pkts++; + } else { + ev.queue_id++; + pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); + pipeline_event_enqueue(dev, port, &ev); + } + } + + return 0; +} + +static __rte_noinline int +pipeline_queue_worker_single_stage_fwd(void *arg) +{ + PIPELINE_WORKER_SINGLE_STAGE_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + ev.queue_id = tx_queue[ev.mbuf->port]; + rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0); + pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); + pipeline_event_enqueue(dev, port, &ev); + w->processed_pkts++; + } + + return 0; +} + +static __rte_noinline int +pipeline_queue_worker_single_stage_burst_tx(void *arg) +{ + PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + if (ev[i].sched_type == RTE_SCHED_TYPE_ATOMIC) { + pipeline_event_tx(dev, port, &ev[i]); + ev[i].op = RTE_EVENT_OP_RELEASE; + w->processed_pkts++; + } else { + ev[i].queue_id++; + pipeline_fwd_event(&ev[i], + RTE_SCHED_TYPE_ATOMIC); + } + } + + pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + } + + return 0; +} + +static __rte_noinline int +pipeline_queue_worker_single_stage_burst_fwd(void *arg) +{ + PIPELINE_WORKER_SINGLE_STAGE_BURST_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + ev[i].queue_id = tx_queue[ev[i].mbuf->port]; + rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0); + pipeline_fwd_event(&ev[i], RTE_SCHED_TYPE_ATOMIC); + } + + pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + w->processed_pkts += nb_rx; + } + + return 0; +} + + +static __rte_noinline int +pipeline_queue_worker_multi_stage_tx(void *arg) +{ + PIPELINE_WORKER_MULTI_STAGE_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + cq_id = ev.queue_id % nb_stages; + + if (ev.queue_id == tx_queue[ev.mbuf->port]) { + pipeline_event_tx(dev, port, &ev); + w->processed_pkts++; + continue; + } + + ev.queue_id++; + pipeline_fwd_event(&ev, cq_id != last_queue ? + sched_type_list[cq_id] : + RTE_SCHED_TYPE_ATOMIC); + pipeline_event_enqueue(dev, port, &ev); + } + + return 0; +} + +static __rte_noinline int +pipeline_queue_worker_multi_stage_fwd(void *arg) +{ + PIPELINE_WORKER_MULTI_STAGE_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; + + while (t->done == false) { + uint16_t event = rte_event_dequeue_burst(dev, port, &ev, 1, 0); + + if (!event) { + rte_pause(); + continue; + } + + cq_id = ev.queue_id % nb_stages; + + if (cq_id == last_queue) { + ev.queue_id = tx_queue[ev.mbuf->port]; + rte_event_eth_tx_adapter_txq_set(ev.mbuf, 0); + pipeline_fwd_event(&ev, RTE_SCHED_TYPE_ATOMIC); + w->processed_pkts++; + } else { + ev.queue_id++; + pipeline_fwd_event(&ev, sched_type_list[cq_id]); + } + + pipeline_event_enqueue(dev, port, &ev); + } + + return 0; +} + +static __rte_noinline int +pipeline_queue_worker_multi_stage_burst_tx(void *arg) +{ + PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + cq_id = ev[i].queue_id % nb_stages; + + if (ev[i].queue_id == tx_queue[ev[i].mbuf->port]) { + pipeline_event_tx(dev, port, &ev[i]); + ev[i].op = RTE_EVENT_OP_RELEASE; + w->processed_pkts++; + continue; + } + + ev[i].queue_id++; + pipeline_fwd_event(&ev[i], cq_id != last_queue ? + sched_type_list[cq_id] : + RTE_SCHED_TYPE_ATOMIC); + } + + pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + } + + return 0; +} + +static __rte_noinline int +pipeline_queue_worker_multi_stage_burst_fwd(void *arg) +{ + PIPELINE_WORKER_MULTI_STAGE_BURST_INIT; + const uint8_t *tx_queue = t->tx_evqueue_id; + + while (t->done == false) { + uint16_t nb_rx = rte_event_dequeue_burst(dev, port, ev, + BURST_SIZE, 0); + + if (!nb_rx) { + rte_pause(); + continue; + } + + for (i = 0; i < nb_rx; i++) { + rte_prefetch0(ev[i + 1].mbuf); + cq_id = ev[i].queue_id % nb_stages; + + if (cq_id == last_queue) { + ev[i].queue_id = tx_queue[ev[i].mbuf->port]; + rte_event_eth_tx_adapter_txq_set(ev[i].mbuf, 0); + pipeline_fwd_event(&ev[i], + RTE_SCHED_TYPE_ATOMIC); + w->processed_pkts++; + } else { + ev[i].queue_id++; + pipeline_fwd_event(&ev[i], + sched_type_list[cq_id]); + } + } + + pipeline_event_enqueue_burst(dev, port, ev, nb_rx); + } + + return 0; +} + +static int +worker_wrapper(void *arg) +{ + struct worker_data *w = arg; + struct evt_options *opt = w->t->opt; + const bool burst = evt_has_burst_mode(w->dev_id); + const bool internal_port = w->t->internal_port; + const uint8_t nb_stages = opt->nb_stages; + RTE_SET_USED(opt); + + if (nb_stages == 1) { + if (!burst && internal_port) + return pipeline_queue_worker_single_stage_tx(arg); + else if (!burst && !internal_port) + return pipeline_queue_worker_single_stage_fwd(arg); + else if (burst && internal_port) + return pipeline_queue_worker_single_stage_burst_tx(arg); + else if (burst && !internal_port) + return pipeline_queue_worker_single_stage_burst_fwd( + arg); + } else { + if (!burst && internal_port) + return pipeline_queue_worker_multi_stage_tx(arg); + else if (!burst && !internal_port) + return pipeline_queue_worker_multi_stage_fwd(arg); + else if (burst && internal_port) + return pipeline_queue_worker_multi_stage_burst_tx(arg); + else if (burst && !internal_port) + return pipeline_queue_worker_multi_stage_burst_fwd(arg); + + } + rte_panic("invalid worker\n"); +} + +static int +pipeline_queue_launch_lcores(struct evt_test *test, struct evt_options *opt) +{ + return pipeline_launch_lcores(test, opt, worker_wrapper); +} + +static int +pipeline_queue_eventdev_setup(struct evt_test *test, struct evt_options *opt) +{ + int ret; + int nb_ports; + int nb_queues; + int nb_stages = opt->nb_stages; + uint8_t queue; + uint8_t tx_evport_id = 0; + uint8_t tx_evqueue_id[RTE_MAX_ETHPORTS]; + uint8_t queue_arr[RTE_EVENT_MAX_QUEUES_PER_DEV]; + uint8_t nb_worker_queues = 0; + uint16_t prod = 0; + struct rte_event_dev_info info; + struct test_pipeline *t = evt_test_priv(test); + + nb_ports = evt_nr_active_lcores(opt->wlcores); + nb_queues = rte_eth_dev_count_avail() * (nb_stages); + + /* One queue for Tx adapter per port */ + nb_queues += rte_eth_dev_count_avail(); + + memset(tx_evqueue_id, 0, sizeof(uint8_t) * RTE_MAX_ETHPORTS); + memset(queue_arr, 0, sizeof(uint8_t) * RTE_EVENT_MAX_QUEUES_PER_DEV); + + rte_event_dev_info_get(opt->dev_id, &info); + const struct rte_event_dev_config config = { + .nb_event_queues = nb_queues, + .nb_event_ports = nb_ports, + .nb_events_limit = info.max_num_events, + .nb_event_queue_flows = opt->nb_flows, + .nb_event_port_dequeue_depth = + info.max_event_port_dequeue_depth, + .nb_event_port_enqueue_depth = + info.max_event_port_enqueue_depth, + }; + ret = rte_event_dev_configure(opt->dev_id, &config); + if (ret) { + evt_err("failed to configure eventdev %d", opt->dev_id); + return ret; + } + + struct rte_event_queue_conf q_conf = { + .priority = RTE_EVENT_DEV_PRIORITY_NORMAL, + .nb_atomic_flows = opt->nb_flows, + .nb_atomic_order_sequences = opt->nb_flows, + }; + /* queue configurations */ + for (queue = 0; queue < nb_queues; queue++) { + uint8_t slot; + + q_conf.event_queue_cfg = 0; + slot = queue % (nb_stages + 1); + if (slot == nb_stages) { + q_conf.schedule_type = RTE_SCHED_TYPE_ATOMIC; + if (!t->internal_port) { + q_conf.event_queue_cfg = + RTE_EVENT_QUEUE_CFG_SINGLE_LINK; + } + tx_evqueue_id[prod++] = queue; + } else { + q_conf.schedule_type = opt->sched_type_list[slot]; + queue_arr[nb_worker_queues] = queue; + nb_worker_queues++; + } + + ret = rte_event_queue_setup(opt->dev_id, queue, &q_conf); + if (ret) { + evt_err("failed to setup queue=%d", queue); + return ret; + } + } + + if (opt->wkr_deq_dep > info.max_event_port_dequeue_depth) + opt->wkr_deq_dep = info.max_event_port_dequeue_depth; + + /* port configuration */ + const struct rte_event_port_conf p_conf = { + .dequeue_depth = opt->wkr_deq_dep, + .enqueue_depth = info.max_event_port_dequeue_depth, + .new_event_threshold = info.max_num_events, + }; + + if (!t->internal_port) { + ret = pipeline_event_port_setup(test, opt, queue_arr, + nb_worker_queues, p_conf); + if (ret) + return ret; + } else + ret = pipeline_event_port_setup(test, opt, NULL, nb_queues, + p_conf); + + if (ret) + return ret; + /* + * The pipelines are setup in the following manner: + * + * eth_dev_count = 2, nb_stages = 2. + * + * queues = 6 + * stride = 3 + * + * event queue pipelines: + * eth0 -> q0 -> q1 -> (q2->tx) + * eth1 -> q3 -> q4 -> (q5->tx) + * + * q2, q5 configured as ATOMIC | SINGLE_LINK + * + */ + ret = pipeline_event_rx_adapter_setup(opt, nb_stages + 1, p_conf); + if (ret) + return ret; + + ret = pipeline_event_tx_adapter_setup(opt, p_conf); + if (ret) + return ret; + + if (!evt_has_distributed_sched(opt->dev_id)) { + uint32_t service_id; + rte_event_dev_service_id_get(opt->dev_id, &service_id); + ret = evt_service_setup(service_id); + if (ret) { + evt_err("No service lcore found to run event dev."); + return ret; + } + } + + /* Connect the tx_evqueue_id to the Tx adapter port */ + if (!t->internal_port) { + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_event_eth_tx_adapter_event_port_get(prod, + &tx_evport_id); + if (ret) { + evt_err("Unable to get Tx adptr[%d] evprt[%d]", + prod, tx_evport_id); + return ret; + } + + if (rte_event_port_link(opt->dev_id, tx_evport_id, + &tx_evqueue_id[prod], + NULL, 1) != 1) { + evt_err("Unable to link Tx adptr[%d] evprt[%d]", + prod, tx_evport_id); + return ret; + } + } + } + + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_eth_dev_start(prod); + if (ret) { + evt_err("Ethernet dev [%d] failed to start." + " Using synthetic producer", prod); + return ret; + } + + } + + ret = rte_event_dev_start(opt->dev_id); + if (ret) { + evt_err("failed to start eventdev %d", opt->dev_id); + return ret; + } + + RTE_ETH_FOREACH_DEV(prod) { + ret = rte_event_eth_rx_adapter_start(prod); + if (ret) { + evt_err("Rx adapter[%d] start failed", prod); + return ret; + } + + ret = rte_event_eth_tx_adapter_start(prod); + if (ret) { + evt_err("Tx adapter[%d] start failed", prod); + return ret; + } + } + + memcpy(t->tx_evqueue_id, tx_evqueue_id, sizeof(uint8_t) * + RTE_MAX_ETHPORTS); + + return 0; +} + +static void +pipeline_queue_opt_dump(struct evt_options *opt) +{ + pipeline_opt_dump(opt, pipeline_queue_nb_event_queues(opt)); +} + +static int +pipeline_queue_opt_check(struct evt_options *opt) +{ + return pipeline_opt_check(opt, pipeline_queue_nb_event_queues(opt)); +} + +static bool +pipeline_queue_capability_check(struct evt_options *opt) +{ + struct rte_event_dev_info dev_info; + + rte_event_dev_info_get(opt->dev_id, &dev_info); + if (dev_info.max_event_queues < pipeline_queue_nb_event_queues(opt) || + dev_info.max_event_ports < + evt_nr_active_lcores(opt->wlcores)) { + evt_err("not enough eventdev queues=%d/%d or ports=%d/%d", + pipeline_queue_nb_event_queues(opt), + dev_info.max_event_queues, + evt_nr_active_lcores(opt->wlcores), + dev_info.max_event_ports); + } + + return true; +} + +static const struct evt_test_ops pipeline_queue = { + .cap_check = pipeline_queue_capability_check, + .opt_check = pipeline_queue_opt_check, + .opt_dump = pipeline_queue_opt_dump, + .test_setup = pipeline_test_setup, + .mempool_setup = pipeline_mempool_setup, + .ethdev_setup = pipeline_ethdev_setup, + .eventdev_setup = pipeline_queue_eventdev_setup, + .launch_lcores = pipeline_queue_launch_lcores, + .eventdev_destroy = pipeline_eventdev_destroy, + .mempool_destroy = pipeline_mempool_destroy, + .ethdev_destroy = pipeline_ethdev_destroy, + .test_result = pipeline_test_result, + .test_destroy = pipeline_test_destroy, +}; + +EVT_TEST_REGISTER(pipeline_queue); diff --git a/dpdk/app/test-pmd/Makefile b/dpdk/app/test-pmd/Makefile index 4993c918..d5258eae 100644 --- a/dpdk/app/test-pmd/Makefile +++ b/dpdk/app/test-pmd/Makefile @@ -1,33 +1,5 @@ -# BSD LICENSE -# -# Copyright(c) 2010-2015 Intel Corporation. All rights reserved. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in -# the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Intel Corporation nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2015 Intel Corporation include $(RTE_SDK)/mk/rte.vars.mk @@ -38,8 +10,10 @@ ifeq ($(CONFIG_RTE_TEST_PMD),y) # APP = testpmd +CFLAGS += -DALLOW_EXPERIMENTAL_API CFLAGS += -O3 CFLAGS += $(WERROR_FLAGS) +CFLAGS += -Wno-deprecated-declarations # # all source are stored in SRCS-y @@ -59,10 +33,13 @@ SRCS-y += rxonly.c SRCS-y += txonly.c SRCS-y += csumonly.c SRCS-y += icmpecho.c +SRCS-y += noisy_vnf.c SRCS-$(CONFIG_RTE_LIBRTE_IEEE1588) += ieee1588fwd.c +SRCS-$(CONFIG_RTE_LIBRTE_BPF) += bpf_cmd.c +SRCS-y += util.c -ifeq ($(CONFIG_RTE_LIBRTE_PMD_SOFTNIC)$(CONFIG_RTE_LIBRTE_SCHED),yy) -SRCS-y += tm.c +ifeq ($(CONFIG_RTE_LIBRTE_PMD_SOFTNIC), y) +SRCS-y += softnicfwd.c endif ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),y) @@ -71,6 +48,12 @@ ifeq ($(CONFIG_RTE_LIBRTE_PMD_BOND),y) LDLIBS += -lrte_pmd_bond endif +ifeq ($(CONFIG_RTE_LIBRTE_DPAA_BUS)$(CONFIG_RTE_LIBRTE_DPAA_PMD),yy) +LDLIBS += -lrte_pmd_dpaa +LDLIBS += -lrte_bus_dpaa +LDLIBS += -lrte_mempool_dpaa +endif + ifeq ($(CONFIG_RTE_LIBRTE_IXGBE_PMD),y) LDLIBS += -lrte_pmd_ixgbe endif @@ -89,8 +72,6 @@ endif endif -CFLAGS_cmdline.o := -D_GNU_SOURCE - include $(RTE_SDK)/mk/rte.app.mk endif diff --git a/dpdk/app/test-pmd/bpf_cmd.c b/dpdk/app/test-pmd/bpf_cmd.c new file mode 100644 index 00000000..830bfc13 --- /dev/null +++ b/dpdk/app/test-pmd/bpf_cmd.c @@ -0,0 +1,198 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "testpmd.h" + +static const struct rte_bpf_xsym bpf_xsym[] = { + { + .name = RTE_STR(stdout), + .type = RTE_BPF_XTYPE_VAR, + .var = { + .val = &stdout, + .desc = { + .type = RTE_BPF_ARG_PTR, + .size = sizeof(stdout), + }, + }, + }, + { + .name = RTE_STR(rte_pktmbuf_dump), + .type = RTE_BPF_XTYPE_FUNC, + .func = { + .val = (void *)rte_pktmbuf_dump, + .nb_args = 3, + .args = { + [0] = { + .type = RTE_BPF_ARG_RAW, + .size = sizeof(uintptr_t), + }, + [1] = { + .type = RTE_BPF_ARG_PTR_MBUF, + .size = sizeof(struct rte_mbuf), + }, + [2] = { + .type = RTE_BPF_ARG_RAW, + .size = sizeof(uint32_t), + }, + }, + }, + }, +}; + +/* *** load BPF program *** */ +struct cmd_bpf_ld_result { + cmdline_fixed_string_t bpf; + cmdline_fixed_string_t dir; + uint8_t port; + uint16_t queue; + cmdline_fixed_string_t op; + cmdline_fixed_string_t flags; + cmdline_fixed_string_t prm; +}; + +static void +bpf_parse_flags(const char *str, struct rte_bpf_arg *arg, uint32_t *flags) +{ + uint32_t i, v; + + *flags = RTE_BPF_ETH_F_NONE; + arg->type = RTE_BPF_ARG_PTR; + arg->size = mbuf_data_size; + + for (i = 0; str[i] != 0; i++) { + v = toupper(str[i]); + if (v == 'J') + *flags |= RTE_BPF_ETH_F_JIT; + else if (v == 'M') { + arg->type = RTE_BPF_ARG_PTR_MBUF; + arg->size = sizeof(struct rte_mbuf); + arg->buf_size = mbuf_data_size; + } else if (v == '-') + continue; + else + printf("unknown flag: \'%c\'", v); + } +} + +static void cmd_operate_bpf_ld_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + int32_t rc; + uint32_t flags; + struct cmd_bpf_ld_result *res; + struct rte_bpf_prm prm; + const char *fname, *sname; + + res = parsed_result; + memset(&prm, 0, sizeof(prm)); + prm.xsym = bpf_xsym; + prm.nb_xsym = RTE_DIM(bpf_xsym); + + bpf_parse_flags(res->flags, &prm.prog_arg, &flags); + fname = res->prm; + sname = ".text"; + + if (strcmp(res->dir, "rx") == 0) { + rc = rte_bpf_eth_rx_elf_load(res->port, res->queue, &prm, + fname, sname, flags); + printf("%d:%s\n", rc, strerror(-rc)); + } else if (strcmp(res->dir, "tx") == 0) { + rc = rte_bpf_eth_tx_elf_load(res->port, res->queue, &prm, + fname, sname, flags); + printf("%d:%s\n", rc, strerror(-rc)); + } else + printf("invalid value: %s\n", res->dir); +} + +cmdline_parse_token_string_t cmd_load_bpf_start = + TOKEN_STRING_INITIALIZER(struct cmd_bpf_ld_result, + bpf, "bpf-load"); +cmdline_parse_token_string_t cmd_load_bpf_dir = + TOKEN_STRING_INITIALIZER(struct cmd_bpf_ld_result, + dir, "rx#tx"); +cmdline_parse_token_num_t cmd_load_bpf_port = + TOKEN_NUM_INITIALIZER(struct cmd_bpf_ld_result, port, UINT8); +cmdline_parse_token_num_t cmd_load_bpf_queue = + TOKEN_NUM_INITIALIZER(struct cmd_bpf_ld_result, queue, UINT16); +cmdline_parse_token_string_t cmd_load_bpf_flags = + TOKEN_STRING_INITIALIZER(struct cmd_bpf_ld_result, + flags, NULL); +cmdline_parse_token_string_t cmd_load_bpf_prm = + TOKEN_STRING_INITIALIZER(struct cmd_bpf_ld_result, + prm, NULL); + +cmdline_parse_inst_t cmd_operate_bpf_ld_parse = { + .f = cmd_operate_bpf_ld_parsed, + .data = NULL, + .help_str = "bpf-load rx|tx ", + .tokens = { + (void *)&cmd_load_bpf_start, + (void *)&cmd_load_bpf_dir, + (void *)&cmd_load_bpf_port, + (void *)&cmd_load_bpf_queue, + (void *)&cmd_load_bpf_flags, + (void *)&cmd_load_bpf_prm, + NULL, + }, +}; + +/* *** unload BPF program *** */ +struct cmd_bpf_unld_result { + cmdline_fixed_string_t bpf; + cmdline_fixed_string_t dir; + uint8_t port; + uint16_t queue; +}; + +static void cmd_operate_bpf_unld_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_bpf_unld_result *res; + + res = parsed_result; + + if (strcmp(res->dir, "rx") == 0) + rte_bpf_eth_rx_unload(res->port, res->queue); + else if (strcmp(res->dir, "tx") == 0) + rte_bpf_eth_tx_unload(res->port, res->queue); + else + printf("invalid value: %s\n", res->dir); +} + +cmdline_parse_token_string_t cmd_unload_bpf_start = + TOKEN_STRING_INITIALIZER(struct cmd_bpf_unld_result, + bpf, "bpf-unload"); +cmdline_parse_token_string_t cmd_unload_bpf_dir = + TOKEN_STRING_INITIALIZER(struct cmd_bpf_unld_result, + dir, "rx#tx"); +cmdline_parse_token_num_t cmd_unload_bpf_port = + TOKEN_NUM_INITIALIZER(struct cmd_bpf_unld_result, port, UINT8); +cmdline_parse_token_num_t cmd_unload_bpf_queue = + TOKEN_NUM_INITIALIZER(struct cmd_bpf_unld_result, queue, UINT16); + +cmdline_parse_inst_t cmd_operate_bpf_unld_parse = { + .f = cmd_operate_bpf_unld_parsed, + .data = NULL, + .help_str = "bpf-unload rx|tx ", + .tokens = { + (void *)&cmd_unload_bpf_start, + (void *)&cmd_unload_bpf_dir, + (void *)&cmd_unload_bpf_port, + (void *)&cmd_unload_bpf_queue, + NULL, + }, +}; diff --git a/dpdk/app/test-pmd/bpf_cmd.h b/dpdk/app/test-pmd/bpf_cmd.h new file mode 100644 index 00000000..5ee4c9f7 --- /dev/null +++ b/dpdk/app/test-pmd/bpf_cmd.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2018 Intel Corporation + */ + +#ifndef _BPF_CMD_H_ +#define _BPF_CMD_H_ + +#ifdef RTE_LIBRTE_BPF + + /* BPF CLI */ +extern cmdline_parse_inst_t cmd_operate_bpf_ld_parse; +extern cmdline_parse_inst_t cmd_operate_bpf_unld_parse; + +#endif /* RTE_LIBRTE_BPF */ + +#endif /* _BPF_CMD_H_ */ diff --git a/dpdk/app/test-pmd/cmdline.c b/dpdk/app/test-pmd/cmdline.c index 77c11b84..12750741 100644 --- a/dpdk/app/test-pmd/cmdline.c +++ b/dpdk/app/test-pmd/cmdline.c @@ -1,35 +1,6 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2010-2016 Intel Corporation. All rights reserved. - * Copyright(c) 2014 6WIND S.A. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2010-2016 Intel Corporation. + * Copyright(c) 2014 6WIND S.A. */ #include @@ -89,6 +60,9 @@ #include #include #endif +#if defined RTE_LIBRTE_DPAA_BUS && defined RTE_LIBRTE_DPAA_PMD +#include +#endif #ifdef RTE_LIBRTE_IXGBE_PMD #include #endif @@ -101,6 +75,7 @@ #include "testpmd.h" #include "cmdline_mtr.h" #include "cmdline_tm.h" +#include "bpf_cmd.h" static struct cmdline *testpmd_cl; @@ -192,7 +167,7 @@ static void cmd_help_long_parsed(void *parsed_result, "Display:\n" "--------\n\n" - "show port (info|stats|xstats|fdir|stat_qmap|dcb_tc|cap) (port_id|all)\n" + "show port (info|stats|summary|xstats|fdir|stat_qmap|dcb_tc|cap) (port_id|all)\n" " Display information for port_id, or all.\n\n" "show port X rss reta (size) (mask0,mask1,...)\n" @@ -200,11 +175,8 @@ static void cmd_help_long_parsed(void *parsed_result, " by masks on port X. size is used to indicate the" " hardware supported reta size\n\n" - "show port rss-hash ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|" - "ipv4-sctp|ipv4-other|ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|" - "ipv6-other|l2-payload|ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex [key]\n" - " Display the RSS hash functions and RSS hash key" - " of port X\n\n" + "show port (port_id) rss-hash [key]\n" + " Display the RSS hash functions and RSS hash key of port\n\n" "clear port (info|stats|xstats|fdir|stat_qmap) (port_id|all)\n" " Clear information for port_id, or all.\n\n" @@ -271,6 +243,9 @@ static void cmd_help_long_parsed(void *parsed_result, "set verbose (level)\n" " Set the debug verbosity level X.\n\n" + "set log global|(type) (level)\n" + " Set the log level.\n\n" + "set nbport (num)\n" " Set number of ports.\n\n" @@ -305,6 +280,9 @@ static void cmd_help_long_parsed(void *parsed_result, "set portlist (x[,y]*)\n" " Set the list of forwarding ports.\n\n" + "set port setup on (iterator|event)\n" + " Select how attached port is retrieved for setup.\n\n" + "set tx loopback (port_id) (on|off)\n" " Enable or disable tx loopback.\n\n" @@ -419,12 +397,13 @@ static void cmd_help_long_parsed(void *parsed_result, " Disable hardware insertion of a VLAN header in" " packets sent on a port.\n\n" - "csum set (ip|udp|tcp|sctp|outer-ip) (hw|sw) (port_id)\n" + "csum set (ip|udp|tcp|sctp|outer-ip|outer-udp) (hw|sw) (port_id)\n" " Select hardware or software calculation of the" " checksum when transmitting a packet using the" " csum forward engine.\n" " ip|udp|tcp|sctp always concern the inner layer.\n" " outer-ip concerns the outer IP layer in" + " outer-udp concerns the outer UDP layer in" " case the packet is recognized as a tunnel packet by" " the forward engine (vxlan, gre and ipip are supported)\n" " Please check the NIC datasheet for HW limits.\n\n" @@ -486,6 +465,9 @@ static void cmd_help_long_parsed(void *parsed_result, "set vf mac addr (port_id) (vf_id) (XX:XX:XX:XX:XX:XX)\n" " Set the MAC address for a VF from the PF.\n\n" + "set eth-peer (port_id) (peer_addr)\n" + " set the peer address for certain port.\n\n" + "set port (port_id) uta (mac_address|all) (on|off)\n" " Add/Remove a or all unicast hash filter(s)" "from port X.\n\n" @@ -665,10 +647,10 @@ static void cmd_help_long_parsed(void *parsed_result, " Set default traffic Management hierarchy on a port\n\n" #endif - "ddp add (port_id) (profile_path[,output_path])\n" + "ddp add (port_id) (profile_path[,backup_profile_path])\n" " Load a profile package on a port\n\n" - "ddp del (port_id) (profile_path)\n" + "ddp del (port_id) (backup_profile_path)\n" " Delete a profile package from a port\n\n" "ptype mapping get (port_id) (valid_only)\n" @@ -698,7 +680,10 @@ static void cmd_help_long_parsed(void *parsed_result, "set port (port_id) queue-region flush (on|off)\n" " flush all queue region related configuration\n\n" - "add port meter profile srtcm_rfc2697 (port_id) (profile_id) (cir) (cbs) (ebs) (color_aware)\n" + "show port meter cap (port_id)\n" + " Show port meter capability information\n\n" + + "add port meter profile srtcm_rfc2697 (port_id) (profile_id) (cir) (cbs) (ebs)\n" " meter profile add - srtcm rfc 2697\n\n" "add port meter profile trtcm_rfc2698 (port_id) (profile_id) (cir) (pir) (cbs) (pbs)\n" @@ -710,16 +695,30 @@ static void cmd_help_long_parsed(void *parsed_result, "del port meter profile (port_id) (profile_id)\n" " meter profile delete\n\n" - "set port meter (port_id) (mtr_id) (profile_id) (g_action) (y_action) (r_action) (stats_mask) (shared)\n" + "create port meter (port_id) (mtr_id) (profile_id) (meter_enable)\n" + "(g_action) (y_action) (r_action) (stats_mask) (shared)\n" + "(use_pre_meter_color) [(dscp_tbl_entry0) (dscp_tbl_entry1)...\n" + "(dscp_tbl_entry63)]\n" " meter create\n\n" + "enable port meter (port_id) (mtr_id)\n" + " meter enable\n\n" + + "disable port meter (port_id) (mtr_id)\n" + " meter disable\n\n" + "del port meter (port_id) (mtr_id)\n" " meter delete\n\n" "set port meter profile (port_id) (mtr_id) (profile_id)\n" " meter update meter profile\n\n" - "set port meter policer action (port_id) (mtr_id) (color) (action)\n" + "set port meter dscp table (port_id) (mtr_id) [(dscp_tbl_entry0)\n" + "(dscp_tbl_entry1)...(dscp_tbl_entry63)]\n" + " update meter dscp table entries\n\n" + + "set port meter policer action (port_id) (mtr_id) (action_mask)\n" + "(action0) [(action1) (action2)]\n" " meter update policer action\n\n" "set port meter stats mask (port_id) (mtr_id) (stats_mask)\n" @@ -729,7 +728,8 @@ static void cmd_help_long_parsed(void *parsed_result, " show all queue region related configuration info\n\n" "add port tm node shaper profile (port_id) (shaper_profile_id)" - " (tb_rate) (tb_size) (packet_length_adjust)\n" + " (cmit_tb_rate) (cmit_tb_size) (peak_tb_rate) (peak_tb_size)" + " (packet_length_adjust)\n" " Add port tm node private shaper profile.\n\n" "del port tm node shaper profile (port_id) (shaper_profile_id)\n" @@ -774,9 +774,36 @@ static void cmd_help_long_parsed(void *parsed_result, " (priority) (weight)\n" " Set port tm node parent.\n\n" + "suspend port tm node (port_id) (node_id)" + " Suspend tm node.\n\n" + + "resume port tm node (port_id) (node_id)" + " Resume tm node.\n\n" + "port tm hierarchy commit (port_id) (clean_on_fail)\n" " Commit tm hierarchy.\n\n" + "vxlan ip-version (ipv4|ipv6) vni (vni) udp-src" + " (udp-src) udp-dst (udp-dst) ip-src (ip-src) ip-dst" + " (ip-dst) eth-src (eth-src) eth-dst (eth-dst)\n" + " Configure the VXLAN encapsulation for flows.\n\n" + + "vxlan-with-vlan ip-version (ipv4|ipv6) vni (vni)" + " udp-src (udp-src) udp-dst (udp-dst) ip-src (ip-src)" + " ip-dst (ip-dst) vlan-tci (vlan-tci) eth-src (eth-src)" + " eth-dst (eth-dst)\n" + " Configure the VXLAN encapsulation for flows.\n\n" + + "nvgre ip-version (ipv4|ipv6) tni (tni) ip-src" + " (ip-src) ip-dst (ip-dst) eth-src (eth-src) eth-dst" + " (eth-dst)\n" + " Configure the NVGRE encapsulation for flows.\n\n" + + "nvgre-with-vlan ip-version (ipv4|ipv6) tni (tni)" + " ip-src (ip-src) ip-dst (ip-dst) vlan-tci (vlan-tci)" + " eth-src (eth-src) eth-dst (eth-dst)\n" + " Configure the NVGRE encapsulation for flows.\n\n" + , list_pkt_forwarding_modes() ); } @@ -809,6 +836,9 @@ static void cmd_help_long_parsed(void *parsed_result, " duplex (half|full|auto)\n" " Set speed and duplex for all ports or port_id\n\n" + "port config (port_id|all) loopback (mode)\n" + " Set loopback mode for all ports or port_id\n\n" + "port config all (rxq|txq|rxd|txd) (value)\n" " Set number for rxq/txq/rxd/txd.\n\n" @@ -821,8 +851,8 @@ static void cmd_help_long_parsed(void *parsed_result, " Set crc-strip/scatter/rx-checksum/hardware-vlan/drop_en" " for ports.\n\n" - "port config all rss (all|ip|tcp|udp|sctp|ether|port|vxlan|" - "geneve|nvgre|none|)\n" + "port config all rss (all|default|ip|tcp|udp|sctp|" + "ether|port|vxlan|geneve|nvgre|none|)\n" " Set the RSS mode.\n\n" "port config port-id rss reta (hash,queue)[,(hash,queue)]\n" @@ -846,10 +876,22 @@ static void cmd_help_long_parsed(void *parsed_result, "port config mtu X value\n" " Set the MTU of port X to a given value\n\n" + "port config (port_id) (rxq|txq) (queue_id) ring_size (value)\n" + " Set a rx/tx queue's ring size configuration, the new" + " value will take effect after command that (re-)start the port" + " or command that setup the specific queue\n\n" + "port (port_id) (rxq|txq) (queue_id) (start|stop)\n" " Start/stop a rx/tx queue of port X. Only take effect" " when port X is started\n\n" + "port (port_id) (rxq|txq) (queue_id) deferred_start (on|off)\n" + " Switch on/off a deferred start of port X rx/tx queue. Only" + " take effect when port X is stopped.\n\n" + + "port (port_id) (rxq|txq) (queue_id) setup\n" + " Setup a rx/tx queue of port X.\n\n" + "port config (port_id|all) l2-tunnel E-tag ether-type" " (value)\n" " Set the value of E-tag ether-type.\n\n" @@ -864,6 +906,18 @@ static void cmd_help_long_parsed(void *parsed_result, "port config (port_id) pctype mapping update" " (pctype_id_0[,pctype_id_1]*) (flow_type_id)\n" " Update a flow type to pctype mapping item on a port\n\n" + + "port config (port_id) pctype (pctype_id) hash_inset|" + "fdir_inset|fdir_flx_inset get|set|clear field\n" + " (field_idx)\n" + " Configure RSS|FDIR|FDIR_FLX input set for some pctype\n\n" + + "port config (port_id) pctype (pctype_id) hash_inset|" + "fdir_inset|fdir_flx_inset clear all" + " Clear RSS|FDIR|FDIR_FLX input set completely for some pctype\n\n" + + "port config (port_id) udp_tunnel_port add|rm vxlan|geneve (udp_port)\n\n" + " Add/remove UDP tunnel port for tunneling offload\n\n" ); } @@ -979,6 +1033,11 @@ static void cmd_help_long_parsed(void *parsed_result, " queue (queue_id) fd_id (fd_id_value)\n" " Add/Del a Tunnel flow director filter.\n\n" + "flow_director_filter (port_id) mode raw (add|del|update)" + " flow (flow_id) (drop|fwd) queue (queue_id)" + " fd_id (fd_id_value) packet (packet file name)\n" + " Add/Del a raw type flow director filter.\n\n" + "flush_flow_director (port_id)\n" " Flush all flow director entries of a device.\n\n" @@ -1194,6 +1253,59 @@ cmdline_parse_inst_t cmd_operate_specific_port = { }, }; +/* *** enable port setup (after attach) via iterator or event *** */ +struct cmd_set_port_setup_on_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t port; + cmdline_fixed_string_t setup; + cmdline_fixed_string_t on; + cmdline_fixed_string_t mode; +}; + +static void cmd_set_port_setup_on_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_port_setup_on_result *res = parsed_result; + + if (strcmp(res->mode, "event") == 0) + setup_on_probe_event = true; + else if (strcmp(res->mode, "iterator") == 0) + setup_on_probe_event = false; + else + printf("Unknown mode\n"); +} + +cmdline_parse_token_string_t cmd_set_port_setup_on_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_setup_on_result, + set, "set"); +cmdline_parse_token_string_t cmd_set_port_setup_on_port = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_setup_on_result, + port, "port"); +cmdline_parse_token_string_t cmd_set_port_setup_on_setup = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_setup_on_result, + setup, "setup"); +cmdline_parse_token_string_t cmd_set_port_setup_on_on = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_setup_on_result, + on, "on"); +cmdline_parse_token_string_t cmd_set_port_setup_on_mode = + TOKEN_STRING_INITIALIZER(struct cmd_set_port_setup_on_result, + mode, "iterator#event"); + +cmdline_parse_inst_t cmd_set_port_setup_on = { + .f = cmd_set_port_setup_on_parsed, + .data = NULL, + .help_str = "set port setup on iterator|event", + .tokens = { + (void *)&cmd_set_port_setup_on_set, + (void *)&cmd_set_port_setup_on_port, + (void *)&cmd_set_port_setup_on_setup, + (void *)&cmd_set_port_setup_on_on, + (void *)&cmd_set_port_setup_on_mode, + NULL, + }, +}; + /* *** attach a specified port *** */ struct cmd_operate_attach_port_result { cmdline_fixed_string_t port; @@ -1250,7 +1362,7 @@ static void cmd_operate_detach_port_parsed(void *parsed_result, struct cmd_operate_detach_port_result *res = parsed_result; if (!strcmp(res->keyword, "detach")) - detach_port(res->port_id); + detach_port_device(res->port_id); else printf("Unknown parameter\n"); } @@ -1404,7 +1516,7 @@ cmdline_parse_inst_t cmd_config_speed_all = { struct cmd_config_speed_specific { cmdline_fixed_string_t port; cmdline_fixed_string_t keyword; - uint8_t id; + portid_t id; cmdline_fixed_string_t item1; cmdline_fixed_string_t item2; cmdline_fixed_string_t value1; @@ -1444,7 +1556,7 @@ cmdline_parse_token_string_t cmd_config_speed_specific_keyword = TOKEN_STRING_INITIALIZER(struct cmd_config_speed_specific, keyword, "config"); cmdline_parse_token_num_t cmd_config_speed_specific_id = - TOKEN_NUM_INITIALIZER(struct cmd_config_speed_specific, id, UINT8); + TOKEN_NUM_INITIALIZER(struct cmd_config_speed_specific, id, UINT16); cmdline_parse_token_string_t cmd_config_speed_specific_item1 = TOKEN_STRING_INITIALIZER(struct cmd_config_speed_specific, item1, "speed"); @@ -1476,6 +1588,122 @@ cmdline_parse_inst_t cmd_config_speed_specific = { }, }; +/* *** configure loopback for all ports *** */ +struct cmd_config_loopback_all { + cmdline_fixed_string_t port; + cmdline_fixed_string_t keyword; + cmdline_fixed_string_t all; + cmdline_fixed_string_t item; + uint32_t mode; +}; + +static void +cmd_config_loopback_all_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_config_loopback_all *res = parsed_result; + portid_t pid; + + if (!all_ports_stopped()) { + printf("Please stop all ports first\n"); + return; + } + + RTE_ETH_FOREACH_DEV(pid) { + ports[pid].dev_conf.lpbk_mode = res->mode; + } + + cmd_reconfig_device_queue(RTE_PORT_ALL, 1, 1); +} + +cmdline_parse_token_string_t cmd_config_loopback_all_port = + TOKEN_STRING_INITIALIZER(struct cmd_config_loopback_all, port, "port"); +cmdline_parse_token_string_t cmd_config_loopback_all_keyword = + TOKEN_STRING_INITIALIZER(struct cmd_config_loopback_all, keyword, + "config"); +cmdline_parse_token_string_t cmd_config_loopback_all_all = + TOKEN_STRING_INITIALIZER(struct cmd_config_loopback_all, all, "all"); +cmdline_parse_token_string_t cmd_config_loopback_all_item = + TOKEN_STRING_INITIALIZER(struct cmd_config_loopback_all, item, + "loopback"); +cmdline_parse_token_num_t cmd_config_loopback_all_mode = + TOKEN_NUM_INITIALIZER(struct cmd_config_loopback_all, mode, UINT32); + +cmdline_parse_inst_t cmd_config_loopback_all = { + .f = cmd_config_loopback_all_parsed, + .data = NULL, + .help_str = "port config all loopback ", + .tokens = { + (void *)&cmd_config_loopback_all_port, + (void *)&cmd_config_loopback_all_keyword, + (void *)&cmd_config_loopback_all_all, + (void *)&cmd_config_loopback_all_item, + (void *)&cmd_config_loopback_all_mode, + NULL, + }, +}; + +/* *** configure loopback for specific port *** */ +struct cmd_config_loopback_specific { + cmdline_fixed_string_t port; + cmdline_fixed_string_t keyword; + uint16_t port_id; + cmdline_fixed_string_t item; + uint32_t mode; +}; + +static void +cmd_config_loopback_specific_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_config_loopback_specific *res = parsed_result; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + + if (!port_is_stopped(res->port_id)) { + printf("Please stop port %u first\n", res->port_id); + return; + } + + ports[res->port_id].dev_conf.lpbk_mode = res->mode; + + cmd_reconfig_device_queue(res->port_id, 1, 1); +} + + +cmdline_parse_token_string_t cmd_config_loopback_specific_port = + TOKEN_STRING_INITIALIZER(struct cmd_config_loopback_specific, port, + "port"); +cmdline_parse_token_string_t cmd_config_loopback_specific_keyword = + TOKEN_STRING_INITIALIZER(struct cmd_config_loopback_specific, keyword, + "config"); +cmdline_parse_token_num_t cmd_config_loopback_specific_id = + TOKEN_NUM_INITIALIZER(struct cmd_config_loopback_specific, port_id, + UINT16); +cmdline_parse_token_string_t cmd_config_loopback_specific_item = + TOKEN_STRING_INITIALIZER(struct cmd_config_loopback_specific, item, + "loopback"); +cmdline_parse_token_num_t cmd_config_loopback_specific_mode = + TOKEN_NUM_INITIALIZER(struct cmd_config_loopback_specific, mode, + UINT32); + +cmdline_parse_inst_t cmd_config_loopback_specific = { + .f = cmd_config_loopback_specific_parsed, + .data = NULL, + .help_str = "port config loopback ", + .tokens = { + (void *)&cmd_config_loopback_specific_port, + (void *)&cmd_config_loopback_specific_keyword, + (void *)&cmd_config_loopback_specific_id, + (void *)&cmd_config_loopback_specific_item, + (void *)&cmd_config_loopback_specific_mode, + NULL, + }, +}; + /* *** configure txq/rxq, txd/rxd *** */ struct cmd_config_rx_tx { cmdline_fixed_string_t port; @@ -1581,29 +1809,36 @@ cmd_config_max_pkt_len_parsed(void *parsed_result, __attribute__((unused)) void *data) { struct cmd_config_max_pkt_len_result *res = parsed_result; + portid_t pid; if (!all_ports_stopped()) { printf("Please stop all ports first\n"); return; } - if (!strcmp(res->name, "max-pkt-len")) { - if (res->value < ETHER_MIN_LEN) { - printf("max-pkt-len can not be less than %d\n", - ETHER_MIN_LEN); + RTE_ETH_FOREACH_DEV(pid) { + struct rte_port *port = &ports[pid]; + uint64_t rx_offloads = port->dev_conf.rxmode.offloads; + + if (!strcmp(res->name, "max-pkt-len")) { + if (res->value < ETHER_MIN_LEN) { + printf("max-pkt-len can not be less than %d\n", + ETHER_MIN_LEN); + return; + } + if (res->value == port->dev_conf.rxmode.max_rx_pkt_len) + return; + + port->dev_conf.rxmode.max_rx_pkt_len = res->value; + if (res->value > ETHER_MAX_LEN) + rx_offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME; + else + rx_offloads &= ~DEV_RX_OFFLOAD_JUMBO_FRAME; + port->dev_conf.rxmode.offloads = rx_offloads; + } else { + printf("Unknown parameter\n"); return; } - if (res->value == rx_mode.max_rx_pkt_len) - return; - - rx_mode.max_rx_pkt_len = res->value; - if (res->value > ETHER_MAX_LEN) - rx_mode.jumbo_frame = 1; - else - rx_mode.jumbo_frame = 0; - } else { - printf("Unknown parameter\n"); - return; } init_port_config(); @@ -1707,100 +1942,107 @@ cmd_config_rx_mode_flag_parsed(void *parsed_result, __attribute__((unused)) void *data) { struct cmd_config_rx_mode_flag *res = parsed_result; + portid_t pid; if (!all_ports_stopped()) { printf("Please stop all ports first\n"); return; } - if (!strcmp(res->name, "crc-strip")) { - if (!strcmp(res->value, "on")) - rx_mode.hw_strip_crc = 1; - else if (!strcmp(res->value, "off")) - rx_mode.hw_strip_crc = 0; - else { + RTE_ETH_FOREACH_DEV(pid) { + struct rte_port *port; + uint64_t rx_offloads; + + port = &ports[pid]; + rx_offloads = port->dev_conf.rxmode.offloads; + if (!strcmp(res->name, "crc-strip")) { + if (!strcmp(res->value, "on")) { + rx_offloads &= ~DEV_RX_OFFLOAD_KEEP_CRC; + } else if (!strcmp(res->value, "off")) { + rx_offloads |= DEV_RX_OFFLOAD_KEEP_CRC; + } else { + printf("Unknown parameter\n"); + return; + } + } else if (!strcmp(res->name, "scatter")) { + if (!strcmp(res->value, "on")) { + rx_offloads |= DEV_RX_OFFLOAD_SCATTER; + } else if (!strcmp(res->value, "off")) { + rx_offloads &= ~DEV_RX_OFFLOAD_SCATTER; + } else { + printf("Unknown parameter\n"); + return; + } + } else if (!strcmp(res->name, "rx-cksum")) { + if (!strcmp(res->value, "on")) + rx_offloads |= DEV_RX_OFFLOAD_CHECKSUM; + else if (!strcmp(res->value, "off")) + rx_offloads &= ~DEV_RX_OFFLOAD_CHECKSUM; + else { + printf("Unknown parameter\n"); + return; + } + } else if (!strcmp(res->name, "rx-timestamp")) { + if (!strcmp(res->value, "on")) + rx_offloads |= DEV_RX_OFFLOAD_TIMESTAMP; + else if (!strcmp(res->value, "off")) + rx_offloads &= ~DEV_RX_OFFLOAD_TIMESTAMP; + else { + printf("Unknown parameter\n"); + return; + } + } else if (!strcmp(res->name, "hw-vlan")) { + if (!strcmp(res->value, "on")) { + rx_offloads |= (DEV_RX_OFFLOAD_VLAN_FILTER | + DEV_RX_OFFLOAD_VLAN_STRIP); + } else if (!strcmp(res->value, "off")) { + rx_offloads &= ~(DEV_RX_OFFLOAD_VLAN_FILTER | + DEV_RX_OFFLOAD_VLAN_STRIP); + } else { + printf("Unknown parameter\n"); + return; + } + } else if (!strcmp(res->name, "hw-vlan-filter")) { + if (!strcmp(res->value, "on")) + rx_offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; + else if (!strcmp(res->value, "off")) + rx_offloads &= ~DEV_RX_OFFLOAD_VLAN_FILTER; + else { + printf("Unknown parameter\n"); + return; + } + } else if (!strcmp(res->name, "hw-vlan-strip")) { + if (!strcmp(res->value, "on")) + rx_offloads |= DEV_RX_OFFLOAD_VLAN_STRIP; + else if (!strcmp(res->value, "off")) + rx_offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP; + else { + printf("Unknown parameter\n"); + return; + } + } else if (!strcmp(res->name, "hw-vlan-extend")) { + if (!strcmp(res->value, "on")) + rx_offloads |= DEV_RX_OFFLOAD_VLAN_EXTEND; + else if (!strcmp(res->value, "off")) + rx_offloads &= ~DEV_RX_OFFLOAD_VLAN_EXTEND; + else { + printf("Unknown parameter\n"); + return; + } + } else if (!strcmp(res->name, "drop-en")) { + if (!strcmp(res->value, "on")) + rx_drop_en = 1; + else if (!strcmp(res->value, "off")) + rx_drop_en = 0; + else { + printf("Unknown parameter\n"); + return; + } + } else { printf("Unknown parameter\n"); return; } - } else if (!strcmp(res->name, "scatter")) { - if (!strcmp(res->value, "on")) - rx_mode.enable_scatter = 1; - else if (!strcmp(res->value, "off")) - rx_mode.enable_scatter = 0; - else { - printf("Unknown parameter\n"); - return; - } - } else if (!strcmp(res->name, "rx-cksum")) { - if (!strcmp(res->value, "on")) - rx_mode.hw_ip_checksum = 1; - else if (!strcmp(res->value, "off")) - rx_mode.hw_ip_checksum = 0; - else { - printf("Unknown parameter\n"); - return; - } - } else if (!strcmp(res->name, "rx-timestamp")) { - if (!strcmp(res->value, "on")) - rx_mode.hw_timestamp = 1; - else if (!strcmp(res->value, "off")) - rx_mode.hw_timestamp = 0; - else { - printf("Unknown parameter\n"); - return; - } - } else if (!strcmp(res->name, "hw-vlan")) { - if (!strcmp(res->value, "on")) { - rx_mode.hw_vlan_filter = 1; - rx_mode.hw_vlan_strip = 1; - } - else if (!strcmp(res->value, "off")) { - rx_mode.hw_vlan_filter = 0; - rx_mode.hw_vlan_strip = 0; - } - else { - printf("Unknown parameter\n"); - return; - } - } else if (!strcmp(res->name, "hw-vlan-filter")) { - if (!strcmp(res->value, "on")) - rx_mode.hw_vlan_filter = 1; - else if (!strcmp(res->value, "off")) - rx_mode.hw_vlan_filter = 0; - else { - printf("Unknown parameter\n"); - return; - } - } else if (!strcmp(res->name, "hw-vlan-strip")) { - if (!strcmp(res->value, "on")) - rx_mode.hw_vlan_strip = 1; - else if (!strcmp(res->value, "off")) - rx_mode.hw_vlan_strip = 0; - else { - printf("Unknown parameter\n"); - return; - } - } else if (!strcmp(res->name, "hw-vlan-extend")) { - if (!strcmp(res->value, "on")) - rx_mode.hw_vlan_extend = 1; - else if (!strcmp(res->value, "off")) - rx_mode.hw_vlan_extend = 0; - else { - printf("Unknown parameter\n"); - return; - } - } else if (!strcmp(res->name, "drop-en")) { - if (!strcmp(res->value, "on")) - rx_drop_en = 1; - else if (!strcmp(res->value, "off")) - rx_drop_en = 0; - else { - printf("Unknown parameter\n"); - return; - } - } else { - printf("Unknown parameter\n"); - return; + port->dev_conf.rxmode.offloads = rx_offloads; } init_port_config(); @@ -1854,8 +2096,11 @@ cmd_config_rss_parsed(void *parsed_result, { struct cmd_config_rss *res = parsed_result; struct rte_eth_rss_conf rss_conf = { .rss_key_len = 0, }; + struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, }; + int use_default = 0; + int all_updated = 1; int diag; - uint8_t i; + uint16_t i; if (!strcmp(res->value, "all")) rss_conf.rss_hf = ETH_RSS_IP | ETH_RSS_TCP | @@ -1881,6 +2126,8 @@ cmd_config_rss_parsed(void *parsed_result, rss_conf.rss_hf = ETH_RSS_NVGRE; else if (!strcmp(res->value, "none")) rss_conf.rss_hf = 0; + else if (!strcmp(res->value, "default")) + use_default = 1; else if (isdigit(res->value[0]) && atoi(res->value) > 0 && atoi(res->value) < 64) rss_conf.rss_hf = 1ULL << atoi(res->value); @@ -1889,13 +2136,32 @@ cmd_config_rss_parsed(void *parsed_result, return; } rss_conf.rss_key = NULL; - for (i = 0; i < rte_eth_dev_count(); i++) { - diag = rte_eth_dev_rss_hash_update(i, &rss_conf); - if (diag < 0) + /* Update global configuration for RSS types. */ + RTE_ETH_FOREACH_DEV(i) { + struct rte_eth_rss_conf local_rss_conf; + + rte_eth_dev_info_get(i, &dev_info); + if (use_default) + rss_conf.rss_hf = dev_info.flow_type_rss_offloads; + + local_rss_conf = rss_conf; + local_rss_conf.rss_hf = rss_conf.rss_hf & + dev_info.flow_type_rss_offloads; + if (local_rss_conf.rss_hf != rss_conf.rss_hf) { + printf("Port %u modified RSS hash function based on hardware support," + "requested:%#"PRIx64" configured:%#"PRIx64"\n", + i, rss_conf.rss_hf, local_rss_conf.rss_hf); + } + diag = rte_eth_dev_rss_hash_update(i, &local_rss_conf); + if (diag < 0) { + all_updated = 0; printf("Configuration of RSS hash at ethernet port %d " "failed with error (%d): %s.\n", i, -diag, strerror(-diag)); + } } + if (all_updated && !use_default) + rss_hf = rss_conf.rss_hf; } cmdline_parse_token_string_t cmd_config_rss_port = @@ -1913,7 +2179,7 @@ cmdline_parse_inst_t cmd_config_rss = { .f = cmd_config_rss_parsed, .data = NULL, .help_str = "port config all rss " - "all|ip|tcp|udp|sctp|ether|port|vxlan|geneve|nvgre|none|", + "all|default|ip|tcp|udp|sctp|ether|port|vxlan|geneve|nvgre|none|", .tokens = { (void *)&cmd_config_rss_port, (void *)&cmd_config_rss_keyword, @@ -2042,6 +2308,102 @@ cmdline_parse_inst_t cmd_config_rss_hash_key = { }, }; +/* *** configure port rxq/txq ring size *** */ +struct cmd_config_rxtx_ring_size { + cmdline_fixed_string_t port; + cmdline_fixed_string_t config; + portid_t portid; + cmdline_fixed_string_t rxtxq; + uint16_t qid; + cmdline_fixed_string_t rsize; + uint16_t size; +}; + +static void +cmd_config_rxtx_ring_size_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_config_rxtx_ring_size *res = parsed_result; + struct rte_port *port; + uint8_t isrx; + + if (port_id_is_invalid(res->portid, ENABLED_WARN)) + return; + + if (res->portid == (portid_t)RTE_PORT_ALL) { + printf("Invalid port id\n"); + return; + } + + port = &ports[res->portid]; + + if (!strcmp(res->rxtxq, "rxq")) + isrx = 1; + else if (!strcmp(res->rxtxq, "txq")) + isrx = 0; + else { + printf("Unknown parameter\n"); + return; + } + + if (isrx && rx_queue_id_is_invalid(res->qid)) + return; + else if (!isrx && tx_queue_id_is_invalid(res->qid)) + return; + + if (isrx && res->size != 0 && res->size <= rx_free_thresh) { + printf("Invalid rx ring_size, must > rx_free_thresh: %d\n", + rx_free_thresh); + return; + } + + if (isrx) + port->nb_rx_desc[res->qid] = res->size; + else + port->nb_tx_desc[res->qid] = res->size; + + cmd_reconfig_device_queue(res->portid, 0, 1); +} + +cmdline_parse_token_string_t cmd_config_rxtx_ring_size_port = + TOKEN_STRING_INITIALIZER(struct cmd_config_rxtx_ring_size, + port, "port"); +cmdline_parse_token_string_t cmd_config_rxtx_ring_size_config = + TOKEN_STRING_INITIALIZER(struct cmd_config_rxtx_ring_size, + config, "config"); +cmdline_parse_token_num_t cmd_config_rxtx_ring_size_portid = + TOKEN_NUM_INITIALIZER(struct cmd_config_rxtx_ring_size, + portid, UINT16); +cmdline_parse_token_string_t cmd_config_rxtx_ring_size_rxtxq = + TOKEN_STRING_INITIALIZER(struct cmd_config_rxtx_ring_size, + rxtxq, "rxq#txq"); +cmdline_parse_token_num_t cmd_config_rxtx_ring_size_qid = + TOKEN_NUM_INITIALIZER(struct cmd_config_rxtx_ring_size, + qid, UINT16); +cmdline_parse_token_string_t cmd_config_rxtx_ring_size_rsize = + TOKEN_STRING_INITIALIZER(struct cmd_config_rxtx_ring_size, + rsize, "ring_size"); +cmdline_parse_token_num_t cmd_config_rxtx_ring_size_size = + TOKEN_NUM_INITIALIZER(struct cmd_config_rxtx_ring_size, + size, UINT16); + +cmdline_parse_inst_t cmd_config_rxtx_ring_size = { + .f = cmd_config_rxtx_ring_size_parsed, + .data = NULL, + .help_str = "port config rxq|txq ring_size ", + .tokens = { + (void *)&cmd_config_rxtx_ring_size_port, + (void *)&cmd_config_rxtx_ring_size_config, + (void *)&cmd_config_rxtx_ring_size_portid, + (void *)&cmd_config_rxtx_ring_size_rxtxq, + (void *)&cmd_config_rxtx_ring_size_qid, + (void *)&cmd_config_rxtx_ring_size_rsize, + (void *)&cmd_config_rxtx_ring_size_size, + NULL, + }, +}; + /* *** configure port rxq/txq start/stop *** */ struct cmd_config_rxtx_queue { cmdline_fixed_string_t port; @@ -2136,6 +2498,203 @@ cmdline_parse_inst_t cmd_config_rxtx_queue = { }, }; +/* *** configure port rxq/txq deferred start on/off *** */ +struct cmd_config_deferred_start_rxtx_queue { + cmdline_fixed_string_t port; + portid_t port_id; + cmdline_fixed_string_t rxtxq; + uint16_t qid; + cmdline_fixed_string_t opname; + cmdline_fixed_string_t state; +}; + +static void +cmd_config_deferred_start_rxtx_queue_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_config_deferred_start_rxtx_queue *res = parsed_result; + struct rte_port *port; + uint8_t isrx; + uint8_t ison; + uint8_t needreconfig = 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + + if (port_is_started(res->port_id) != 0) { + printf("Please stop port %u first\n", res->port_id); + return; + } + + port = &ports[res->port_id]; + + isrx = !strcmp(res->rxtxq, "rxq"); + + if (isrx && rx_queue_id_is_invalid(res->qid)) + return; + else if (!isrx && tx_queue_id_is_invalid(res->qid)) + return; + + ison = !strcmp(res->state, "on"); + + if (isrx && port->rx_conf[res->qid].rx_deferred_start != ison) { + port->rx_conf[res->qid].rx_deferred_start = ison; + needreconfig = 1; + } else if (!isrx && port->tx_conf[res->qid].tx_deferred_start != ison) { + port->tx_conf[res->qid].tx_deferred_start = ison; + needreconfig = 1; + } + + if (needreconfig) + cmd_reconfig_device_queue(res->port_id, 0, 1); +} + +cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_port = + TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + port, "port"); +cmdline_parse_token_num_t cmd_config_deferred_start_rxtx_queue_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + port_id, UINT16); +cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_rxtxq = + TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + rxtxq, "rxq#txq"); +cmdline_parse_token_num_t cmd_config_deferred_start_rxtx_queue_qid = + TOKEN_NUM_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + qid, UINT16); +cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_opname = + TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + opname, "deferred_start"); +cmdline_parse_token_string_t cmd_config_deferred_start_rxtx_queue_state = + TOKEN_STRING_INITIALIZER(struct cmd_config_deferred_start_rxtx_queue, + state, "on#off"); + +cmdline_parse_inst_t cmd_config_deferred_start_rxtx_queue = { + .f = cmd_config_deferred_start_rxtx_queue_parsed, + .data = NULL, + .help_str = "port rxq|txq deferred_start on|off", + .tokens = { + (void *)&cmd_config_deferred_start_rxtx_queue_port, + (void *)&cmd_config_deferred_start_rxtx_queue_port_id, + (void *)&cmd_config_deferred_start_rxtx_queue_rxtxq, + (void *)&cmd_config_deferred_start_rxtx_queue_qid, + (void *)&cmd_config_deferred_start_rxtx_queue_opname, + (void *)&cmd_config_deferred_start_rxtx_queue_state, + NULL, + }, +}; + +/* *** configure port rxq/txq setup *** */ +struct cmd_setup_rxtx_queue { + cmdline_fixed_string_t port; + portid_t portid; + cmdline_fixed_string_t rxtxq; + uint16_t qid; + cmdline_fixed_string_t setup; +}; + +/* Common CLI fields for queue setup */ +cmdline_parse_token_string_t cmd_setup_rxtx_queue_port = + TOKEN_STRING_INITIALIZER(struct cmd_setup_rxtx_queue, port, "port"); +cmdline_parse_token_num_t cmd_setup_rxtx_queue_portid = + TOKEN_NUM_INITIALIZER(struct cmd_setup_rxtx_queue, portid, UINT16); +cmdline_parse_token_string_t cmd_setup_rxtx_queue_rxtxq = + TOKEN_STRING_INITIALIZER(struct cmd_setup_rxtx_queue, rxtxq, "rxq#txq"); +cmdline_parse_token_num_t cmd_setup_rxtx_queue_qid = + TOKEN_NUM_INITIALIZER(struct cmd_setup_rxtx_queue, qid, UINT16); +cmdline_parse_token_string_t cmd_setup_rxtx_queue_setup = + TOKEN_STRING_INITIALIZER(struct cmd_setup_rxtx_queue, setup, "setup"); + +static void +cmd_setup_rxtx_queue_parsed( + void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_setup_rxtx_queue *res = parsed_result; + struct rte_port *port; + struct rte_mempool *mp; + unsigned int socket_id; + uint8_t isrx = 0; + int ret; + + if (port_id_is_invalid(res->portid, ENABLED_WARN)) + return; + + if (res->portid == (portid_t)RTE_PORT_ALL) { + printf("Invalid port id\n"); + return; + } + + if (!strcmp(res->rxtxq, "rxq")) + isrx = 1; + else if (!strcmp(res->rxtxq, "txq")) + isrx = 0; + else { + printf("Unknown parameter\n"); + return; + } + + if (isrx && rx_queue_id_is_invalid(res->qid)) { + printf("Invalid rx queue\n"); + return; + } else if (!isrx && tx_queue_id_is_invalid(res->qid)) { + printf("Invalid tx queue\n"); + return; + } + + port = &ports[res->portid]; + if (isrx) { + socket_id = rxring_numa[res->portid]; + if (!numa_support || socket_id == NUMA_NO_CONFIG) + socket_id = port->socket_id; + + mp = mbuf_pool_find(socket_id); + if (mp == NULL) { + printf("Failed to setup RX queue: " + "No mempool allocation" + " on the socket %d\n", + rxring_numa[res->portid]); + return; + } + ret = rte_eth_rx_queue_setup(res->portid, + res->qid, + port->nb_rx_desc[res->qid], + socket_id, + &port->rx_conf[res->qid], + mp); + if (ret) + printf("Failed to setup RX queue\n"); + } else { + socket_id = txring_numa[res->portid]; + if (!numa_support || socket_id == NUMA_NO_CONFIG) + socket_id = port->socket_id; + + ret = rte_eth_tx_queue_setup(res->portid, + res->qid, + port->nb_tx_desc[res->qid], + socket_id, + &port->tx_conf[res->qid]); + if (ret) + printf("Failed to setup TX queue\n"); + } +} + +cmdline_parse_inst_t cmd_setup_rxtx_queue = { + .f = cmd_setup_rxtx_queue_parsed, + .data = NULL, + .help_str = "port rxq|txq setup", + .tokens = { + (void *)&cmd_setup_rxtx_queue_port, + (void *)&cmd_setup_rxtx_queue_portid, + (void *)&cmd_setup_rxtx_queue_rxtxq, + (void *)&cmd_setup_rxtx_queue_qid, + (void *)&cmd_setup_rxtx_queue_setup, + NULL, + }, +}; + + /* *** Configure RSS RETA *** */ struct cmd_config_rss_reta { cmdline_fixed_string_t port; @@ -2400,8 +2959,7 @@ static void cmd_showport_rss_hash_parsed(void *parsed_result, { struct cmd_showport_rss_hash *res = parsed_result; - port_rss_hash_conf_show(res->port_id, res->rss_type, - show_rss_key != NULL); + port_rss_hash_conf_show(res->port_id, show_rss_key != NULL); } cmdline_parse_token_string_t cmd_showport_rss_hash_show = @@ -2413,28 +2971,18 @@ cmdline_parse_token_num_t cmd_showport_rss_hash_port_id = cmdline_parse_token_string_t cmd_showport_rss_hash_rss_hash = TOKEN_STRING_INITIALIZER(struct cmd_showport_rss_hash, rss_hash, "rss-hash"); -cmdline_parse_token_string_t cmd_showport_rss_hash_rss_hash_info = - TOKEN_STRING_INITIALIZER(struct cmd_showport_rss_hash, rss_type, - "ipv4#ipv4-frag#ipv4-tcp#ipv4-udp#ipv4-sctp#" - "ipv4-other#ipv6#ipv6-frag#ipv6-tcp#ipv6-udp#" - "ipv6-sctp#ipv6-other#l2-payload#ipv6-ex#" - "ipv6-tcp-ex#ipv6-udp-ex"); cmdline_parse_token_string_t cmd_showport_rss_hash_rss_key = TOKEN_STRING_INITIALIZER(struct cmd_showport_rss_hash, key, "key"); cmdline_parse_inst_t cmd_showport_rss_hash = { .f = cmd_showport_rss_hash_parsed, .data = NULL, - .help_str = "show port rss-hash " - "ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|ipv4-sctp|ipv4-other|" - "ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|" - "l2-payload|ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex", + .help_str = "show port rss-hash", .tokens = { (void *)&cmd_showport_rss_hash_show, (void *)&cmd_showport_rss_hash_port, (void *)&cmd_showport_rss_hash_port_id, (void *)&cmd_showport_rss_hash_rss_hash, - (void *)&cmd_showport_rss_hash_rss_hash_info, NULL, }, }; @@ -2442,16 +2990,12 @@ cmdline_parse_inst_t cmd_showport_rss_hash = { cmdline_parse_inst_t cmd_showport_rss_hash_key = { .f = cmd_showport_rss_hash_parsed, .data = (void *)1, - .help_str = "show port rss-hash " - "ipv4|ipv4-frag|ipv4-tcp|ipv4-udp|ipv4-sctp|ipv4-other|" - "ipv6|ipv6-frag|ipv6-tcp|ipv6-udp|ipv6-sctp|ipv6-other|" - "l2-payload|ipv6-ex|ipv6-tcp-ex|ipv6-udp-ex key", + .help_str = "show port rss-hash key", .tokens = { (void *)&cmd_showport_rss_hash_show, (void *)&cmd_showport_rss_hash_port, (void *)&cmd_showport_rss_hash_port_id, (void *)&cmd_showport_rss_hash_rss_hash, - (void *)&cmd_showport_rss_hash_rss_hash_info, (void *)&cmd_showport_rss_hash_rss_key, NULL, }, @@ -2574,6 +3118,8 @@ cmd_config_burst_parsed(void *parsed_result, __attribute__((unused)) void *data) { struct cmd_config_burst *res = parsed_result; + struct rte_eth_dev_info dev_info; + uint16_t rec_nb_pkts; if (!all_ports_stopped()) { printf("Please stop all ports first\n"); @@ -2581,11 +3127,34 @@ cmd_config_burst_parsed(void *parsed_result, } if (!strcmp(res->name, "burst")) { - if (res->value < 1 || res->value > MAX_PKT_BURST) { + if (res->value == 0) { + /* If user gives a value of zero, query the PMD for + * its recommended Rx burst size. Testpmd uses a single + * size for all ports, so assume all ports are the same + * NIC model and use the values from Port 0. + */ + rte_eth_dev_info_get(0, &dev_info); + rec_nb_pkts = dev_info.default_rxportconf.burst_size; + + if (rec_nb_pkts == 0) { + printf("PMD does not recommend a burst size.\n" + "User provided value must be between" + " 1 and %d\n", MAX_PKT_BURST); + return; + } else if (rec_nb_pkts > MAX_PKT_BURST) { + printf("PMD recommended burst size of %d" + " exceeds maximum value of %d\n", + rec_nb_pkts, MAX_PKT_BURST); + return; + } + printf("Using PMD-provided burst value of %d\n", + rec_nb_pkts); + nb_pkt_per_burst = rec_nb_pkts; + } else if (res->value > MAX_PKT_BURST) { printf("burst must be >= 1 && <= %d\n", MAX_PKT_BURST); return; - } - nb_pkt_per_burst = res->value; + } else + nb_pkt_per_burst = res->value; } else { printf("Unknown parameter\n"); return; @@ -3004,6 +3573,55 @@ cmdline_parse_inst_t cmd_set_numbers = { }, }; +/* *** SET LOG LEVEL CONFIGURATION *** */ + +struct cmd_set_log_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t log; + cmdline_fixed_string_t type; + uint32_t level; +}; + +static void +cmd_set_log_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_log_result *res; + int ret; + + res = parsed_result; + if (!strcmp(res->type, "global")) + rte_log_set_global_level(res->level); + else { + ret = rte_log_set_level_regexp(res->type, res->level); + if (ret < 0) + printf("Unable to set log level\n"); + } +} + +cmdline_parse_token_string_t cmd_set_log_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_log_result, set, "set"); +cmdline_parse_token_string_t cmd_set_log_log = + TOKEN_STRING_INITIALIZER(struct cmd_set_log_result, log, "log"); +cmdline_parse_token_string_t cmd_set_log_type = + TOKEN_STRING_INITIALIZER(struct cmd_set_log_result, type, NULL); +cmdline_parse_token_num_t cmd_set_log_level = + TOKEN_NUM_INITIALIZER(struct cmd_set_log_result, level, UINT32); + +cmdline_parse_inst_t cmd_set_log = { + .f = cmd_set_log_parsed, + .data = NULL, + .help_str = "set log global| ", + .tokens = { + (void *)&cmd_set_log_set, + (void *)&cmd_set_log_log, + (void *)&cmd_set_log_type, + (void *)&cmd_set_log_level, + NULL, + }, +}; + /* *** SET SEGMENT LENGTHS OF TXONLY PACKETS *** */ struct cmd_set_txpkts_result { @@ -3091,74 +3709,6 @@ cmdline_parse_inst_t cmd_set_txsplit = { }, }; -/* *** CONFIG TX QUEUE FLAGS *** */ - -struct cmd_config_txqflags_result { - cmdline_fixed_string_t port; - cmdline_fixed_string_t config; - cmdline_fixed_string_t all; - cmdline_fixed_string_t what; - int32_t hexvalue; -}; - -static void cmd_config_txqflags_parsed(void *parsed_result, - __attribute__((unused)) struct cmdline *cl, - __attribute__((unused)) void *data) -{ - struct cmd_config_txqflags_result *res = parsed_result; - - if (!all_ports_stopped()) { - printf("Please stop all ports first\n"); - return; - } - - if (strcmp(res->what, "txqflags")) { - printf("Unknown parameter\n"); - return; - } - - if (res->hexvalue >= 0) { - txq_flags = res->hexvalue; - } else { - printf("txqflags must be >= 0\n"); - return; - } - - init_port_config(); - - cmd_reconfig_device_queue(RTE_PORT_ALL, 1, 1); -} - -cmdline_parse_token_string_t cmd_config_txqflags_port = - TOKEN_STRING_INITIALIZER(struct cmd_config_txqflags_result, port, - "port"); -cmdline_parse_token_string_t cmd_config_txqflags_config = - TOKEN_STRING_INITIALIZER(struct cmd_config_txqflags_result, config, - "config"); -cmdline_parse_token_string_t cmd_config_txqflags_all = - TOKEN_STRING_INITIALIZER(struct cmd_config_txqflags_result, all, - "all"); -cmdline_parse_token_string_t cmd_config_txqflags_what = - TOKEN_STRING_INITIALIZER(struct cmd_config_txqflags_result, what, - "txqflags"); -cmdline_parse_token_num_t cmd_config_txqflags_value = - TOKEN_NUM_INITIALIZER(struct cmd_config_txqflags_result, - hexvalue, INT32); - -cmdline_parse_inst_t cmd_config_txqflags = { - .f = cmd_config_txqflags_parsed, - .data = NULL, - .help_str = "port config all txqflags ", - .tokens = { - (void *)&cmd_config_txqflags_port, - (void *)&cmd_config_txqflags_config, - (void *)&cmd_config_txqflags_all, - (void *)&cmd_config_txqflags_what, - (void *)&cmd_config_txqflags_value, - NULL, - }, -}; - /* *** ADD/REMOVE ALL VLAN IDENTIFIERS TO/FROM A PORT VLAN RX FILTER *** */ struct cmd_rx_vlan_filter_all_result { cmdline_fixed_string_t rx_vlan; @@ -3438,7 +3988,14 @@ cmd_tx_vlan_set_parsed(void *parsed_result, { struct cmd_tx_vlan_set_result *res = parsed_result; + if (!port_is_stopped(res->port_id)) { + printf("Please stop port %d first\n", res->port_id); + return; + } + tx_vlan_set(res->port_id, res->vlan_id); + + cmd_reconfig_device_queue(res->port_id, 1, 1); } cmdline_parse_token_string_t cmd_tx_vlan_set_tx_vlan = @@ -3485,7 +4042,14 @@ cmd_tx_vlan_set_qinq_parsed(void *parsed_result, { struct cmd_tx_vlan_set_qinq_result *res = parsed_result; + if (!port_is_stopped(res->port_id)) { + printf("Please stop port %d first\n", res->port_id); + return; + } + tx_qinq_set(res->port_id, res->vlan_id, res->vlan_id_outer); + + cmd_reconfig_device_queue(res->port_id, 1, 1); } cmdline_parse_token_string_t cmd_tx_vlan_set_qinq_tx_vlan = @@ -3591,7 +4155,14 @@ cmd_tx_vlan_reset_parsed(void *parsed_result, { struct cmd_tx_vlan_reset_result *res = parsed_result; + if (!port_is_stopped(res->port_id)) { + printf("Please stop port %d first\n", res->port_id); + return; + } + tx_vlan_reset(res->port_id); + + cmd_reconfig_device_queue(res->port_id, 1, 1); } cmdline_parse_token_string_t cmd_tx_vlan_reset_tx_vlan = @@ -3631,49 +4202,57 @@ static void csum_show(int port_id) { struct rte_eth_dev_info dev_info; - uint16_t ol_flags; + uint64_t tx_offloads; - ol_flags = ports[port_id].tx_ol_flags; + tx_offloads = ports[port_id].dev_conf.txmode.offloads; printf("Parse tunnel is %s\n", - (ol_flags & TESTPMD_TX_OFFLOAD_PARSE_TUNNEL) ? "on" : "off"); + (ports[port_id].parse_tunnel) ? "on" : "off"); printf("IP checksum offload is %s\n", - (ol_flags & TESTPMD_TX_OFFLOAD_IP_CKSUM) ? "hw" : "sw"); + (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) ? "hw" : "sw"); printf("UDP checksum offload is %s\n", - (ol_flags & TESTPMD_TX_OFFLOAD_UDP_CKSUM) ? "hw" : "sw"); + (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) ? "hw" : "sw"); printf("TCP checksum offload is %s\n", - (ol_flags & TESTPMD_TX_OFFLOAD_TCP_CKSUM) ? "hw" : "sw"); + (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) ? "hw" : "sw"); printf("SCTP checksum offload is %s\n", - (ol_flags & TESTPMD_TX_OFFLOAD_SCTP_CKSUM) ? "hw" : "sw"); + (tx_offloads & DEV_TX_OFFLOAD_SCTP_CKSUM) ? "hw" : "sw"); printf("Outer-Ip checksum offload is %s\n", - (ol_flags & TESTPMD_TX_OFFLOAD_OUTER_IP_CKSUM) ? "hw" : "sw"); + (tx_offloads & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) ? "hw" : "sw"); + printf("Outer-Udp checksum offload is %s\n", + (tx_offloads & DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) ? "hw" : "sw"); /* display warnings if configuration is not supported by the NIC */ rte_eth_dev_info_get(port_id, &dev_info); - if ((ol_flags & TESTPMD_TX_OFFLOAD_IP_CKSUM) && + if ((tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) && (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPV4_CKSUM) == 0) { printf("Warning: hardware IP checksum enabled but not " "supported by port %d\n", port_id); } - if ((ol_flags & TESTPMD_TX_OFFLOAD_UDP_CKSUM) && + if ((tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) && (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_CKSUM) == 0) { printf("Warning: hardware UDP checksum enabled but not " "supported by port %d\n", port_id); } - if ((ol_flags & TESTPMD_TX_OFFLOAD_TCP_CKSUM) && + if ((tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) && (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_CKSUM) == 0) { printf("Warning: hardware TCP checksum enabled but not " "supported by port %d\n", port_id); } - if ((ol_flags & TESTPMD_TX_OFFLOAD_SCTP_CKSUM) && + if ((tx_offloads & DEV_TX_OFFLOAD_SCTP_CKSUM) && (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_SCTP_CKSUM) == 0) { printf("Warning: hardware SCTP checksum enabled but not " "supported by port %d\n", port_id); } - if ((ol_flags & TESTPMD_TX_OFFLOAD_OUTER_IP_CKSUM) && + if ((tx_offloads & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) && (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM) == 0) { printf("Warning: hardware outer IP checksum enabled but not " "supported by port %d\n", port_id); } + if ((tx_offloads & DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) && + (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_OUTER_UDP_CKSUM) + == 0) { + printf("Warning: hardware outer UDP checksum enabled but not " + "supported by port %d\n", port_id); + } } static void @@ -3683,36 +4262,87 @@ cmd_csum_parsed(void *parsed_result, { struct cmd_csum_result *res = parsed_result; int hw = 0; - uint16_t mask = 0; + uint64_t csum_offloads = 0; + struct rte_eth_dev_info dev_info; if (port_id_is_invalid(res->port_id, ENABLED_WARN)) { printf("invalid port %d\n", res->port_id); return; } + if (!port_is_stopped(res->port_id)) { + printf("Please stop port %d first\n", res->port_id); + return; + } + rte_eth_dev_info_get(res->port_id, &dev_info); if (!strcmp(res->mode, "set")) { if (!strcmp(res->hwsw, "hw")) hw = 1; if (!strcmp(res->proto, "ip")) { - mask = TESTPMD_TX_OFFLOAD_IP_CKSUM; + if (hw == 0 || (dev_info.tx_offload_capa & + DEV_TX_OFFLOAD_IPV4_CKSUM)) { + csum_offloads |= DEV_TX_OFFLOAD_IPV4_CKSUM; + } else { + printf("IP checksum offload is not supported " + "by port %u\n", res->port_id); + } } else if (!strcmp(res->proto, "udp")) { - mask = TESTPMD_TX_OFFLOAD_UDP_CKSUM; + if (hw == 0 || (dev_info.tx_offload_capa & + DEV_TX_OFFLOAD_UDP_CKSUM)) { + csum_offloads |= DEV_TX_OFFLOAD_UDP_CKSUM; + } else { + printf("UDP checksum offload is not supported " + "by port %u\n", res->port_id); + } } else if (!strcmp(res->proto, "tcp")) { - mask = TESTPMD_TX_OFFLOAD_TCP_CKSUM; + if (hw == 0 || (dev_info.tx_offload_capa & + DEV_TX_OFFLOAD_TCP_CKSUM)) { + csum_offloads |= DEV_TX_OFFLOAD_TCP_CKSUM; + } else { + printf("TCP checksum offload is not supported " + "by port %u\n", res->port_id); + } } else if (!strcmp(res->proto, "sctp")) { - mask = TESTPMD_TX_OFFLOAD_SCTP_CKSUM; + if (hw == 0 || (dev_info.tx_offload_capa & + DEV_TX_OFFLOAD_SCTP_CKSUM)) { + csum_offloads |= DEV_TX_OFFLOAD_SCTP_CKSUM; + } else { + printf("SCTP checksum offload is not supported " + "by port %u\n", res->port_id); + } } else if (!strcmp(res->proto, "outer-ip")) { - mask = TESTPMD_TX_OFFLOAD_OUTER_IP_CKSUM; + if (hw == 0 || (dev_info.tx_offload_capa & + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) { + csum_offloads |= + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM; + } else { + printf("Outer IP checksum offload is not " + "supported by port %u\n", res->port_id); + } + } else if (!strcmp(res->proto, "outer-udp")) { + if (hw == 0 || (dev_info.tx_offload_capa & + DEV_TX_OFFLOAD_OUTER_UDP_CKSUM)) { + csum_offloads |= + DEV_TX_OFFLOAD_OUTER_UDP_CKSUM; + } else { + printf("Outer UDP checksum offload is not " + "supported by port %u\n", res->port_id); + } } - if (hw) - ports[res->port_id].tx_ol_flags |= mask; - else - ports[res->port_id].tx_ol_flags &= (~mask); + if (hw) { + ports[res->port_id].dev_conf.txmode.offloads |= + csum_offloads; + } else { + ports[res->port_id].dev_conf.txmode.offloads &= + (~csum_offloads); + } } csum_show(res->port_id); + + cmd_reconfig_device_queue(res->port_id, 1, 1); } cmdline_parse_token_string_t cmd_csum_csum = @@ -3723,7 +4353,7 @@ cmdline_parse_token_string_t cmd_csum_mode = mode, "set"); cmdline_parse_token_string_t cmd_csum_proto = TOKEN_STRING_INITIALIZER(struct cmd_csum_result, - proto, "ip#tcp#udp#sctp#outer-ip"); + proto, "ip#tcp#udp#sctp#outer-ip#outer-udp"); cmdline_parse_token_string_t cmd_csum_hwsw = TOKEN_STRING_INITIALIZER(struct cmd_csum_result, hwsw, "hw#sw"); @@ -3734,7 +4364,7 @@ cmdline_parse_token_num_t cmd_csum_portid = cmdline_parse_inst_t cmd_csum_set = { .f = cmd_csum_parsed, .data = NULL, - .help_str = "csum set ip|tcp|udp|sctp|outer-ip hw|sw : " + .help_str = "csum set ip|tcp|udp|sctp|outer-ip|outer-udp hw|sw : " "Enable/Disable hardware calculation of L3/L4 checksum when " "using csum forward engine", .tokens = { @@ -3782,11 +4412,9 @@ cmd_csum_tunnel_parsed(void *parsed_result, return; if (!strcmp(res->onoff, "on")) - ports[res->port_id].tx_ol_flags |= - TESTPMD_TX_OFFLOAD_PARSE_TUNNEL; + ports[res->port_id].parse_tunnel = 1; else - ports[res->port_id].tx_ol_flags &= - (~TESTPMD_TX_OFFLOAD_PARSE_TUNNEL); + ports[res->port_id].parse_tunnel = 0; csum_show(res->port_id); } @@ -3796,7 +4424,7 @@ cmdline_parse_token_string_t cmd_csum_tunnel_csum = csum, "csum"); cmdline_parse_token_string_t cmd_csum_tunnel_parse = TOKEN_STRING_INITIALIZER(struct cmd_csum_tunnel_result, - parse, "parse_tunnel"); + parse, "parse-tunnel"); cmdline_parse_token_string_t cmd_csum_tunnel_onoff = TOKEN_STRING_INITIALIZER(struct cmd_csum_tunnel_result, onoff, "on#off"); @@ -3807,7 +4435,7 @@ cmdline_parse_token_num_t cmd_csum_tunnel_portid = cmdline_parse_inst_t cmd_csum_tunnel = { .f = cmd_csum_tunnel_parsed, .data = NULL, - .help_str = "csum parse_tunnel on|off : " + .help_str = "csum parse-tunnel on|off : " "Enable/Disable parsing of tunnels for csum engine", .tokens = { (void *)&cmd_csum_tunnel_csum, @@ -3836,15 +4464,32 @@ cmd_tso_set_parsed(void *parsed_result, if (port_id_is_invalid(res->port_id, ENABLED_WARN)) return; + if (!port_is_stopped(res->port_id)) { + printf("Please stop port %d first\n", res->port_id); + return; + } if (!strcmp(res->mode, "set")) ports[res->port_id].tso_segsz = res->tso_segsz; - if (ports[res->port_id].tso_segsz == 0) + rte_eth_dev_info_get(res->port_id, &dev_info); + if ((ports[res->port_id].tso_segsz != 0) && + (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_TCP_TSO) == 0) { + printf("Error: TSO is not supported by port %d\n", + res->port_id); + return; + } + + if (ports[res->port_id].tso_segsz == 0) { + ports[res->port_id].dev_conf.txmode.offloads &= + ~DEV_TX_OFFLOAD_TCP_TSO; printf("TSO for non-tunneled packets is disabled\n"); - else + } else { + ports[res->port_id].dev_conf.txmode.offloads |= + DEV_TX_OFFLOAD_TCP_TSO; printf("TSO segment size for non-tunneled packets is %d\n", ports[res->port_id].tso_segsz); + } /* display warnings if configuration is not supported by the NIC */ rte_eth_dev_info_get(res->port_id, &dev_info); @@ -3853,6 +4498,8 @@ cmd_tso_set_parsed(void *parsed_result, printf("Warning: TSO enabled but not " "supported by port %d\n", res->port_id); } + + cmd_reconfig_device_queue(res->port_id, 1, 1); } cmdline_parse_token_string_t cmd_tso_set_tso = @@ -3909,24 +4556,31 @@ struct cmd_tunnel_tso_set_result { portid_t port_id; }; -static void +static struct rte_eth_dev_info check_tunnel_tso_nic_support(portid_t port_id) { struct rte_eth_dev_info dev_info; rte_eth_dev_info_get(port_id, &dev_info); if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_VXLAN_TNL_TSO)) - printf("Warning: TSO enabled but VXLAN TUNNEL TSO not " - "supported by port %d\n", port_id); + printf("Warning: VXLAN TUNNEL TSO not supported therefore " + "not enabled for port %d\n", port_id); if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_GRE_TNL_TSO)) - printf("Warning: TSO enabled but GRE TUNNEL TSO not " - "supported by port %d\n", port_id); + printf("Warning: GRE TUNNEL TSO not supported therefore " + "not enabled for port %d\n", port_id); if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IPIP_TNL_TSO)) - printf("Warning: TSO enabled but IPIP TUNNEL TSO not " - "supported by port %d\n", port_id); + printf("Warning: IPIP TUNNEL TSO not supported therefore " + "not enabled for port %d\n", port_id); if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_GENEVE_TNL_TSO)) - printf("Warning: TSO enabled but GENEVE TUNNEL TSO not " - "supported by port %d\n", port_id); + printf("Warning: GENEVE TUNNEL TSO not supported therefore " + "not enabled for port %d\n", port_id); + if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_IP_TNL_TSO)) + printf("Warning: IP TUNNEL TSO not supported therefore " + "not enabled for port %d\n", port_id); + if (!(dev_info.tx_offload_capa & DEV_TX_OFFLOAD_UDP_TNL_TSO)) + printf("Warning: UDP TUNNEL TSO not supported therefore " + "not enabled for port %d\n", port_id); + return dev_info; } static void @@ -3935,16 +4589,38 @@ cmd_tunnel_tso_set_parsed(void *parsed_result, __attribute__((unused)) void *data) { struct cmd_tunnel_tso_set_result *res = parsed_result; + struct rte_eth_dev_info dev_info; if (port_id_is_invalid(res->port_id, ENABLED_WARN)) return; + if (!port_is_stopped(res->port_id)) { + printf("Please stop port %d first\n", res->port_id); + return; + } if (!strcmp(res->mode, "set")) ports[res->port_id].tunnel_tso_segsz = res->tso_segsz; - if (ports[res->port_id].tunnel_tso_segsz == 0) + dev_info = check_tunnel_tso_nic_support(res->port_id); + if (ports[res->port_id].tunnel_tso_segsz == 0) { + ports[res->port_id].dev_conf.txmode.offloads &= + ~(DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO | + DEV_TX_OFFLOAD_IPIP_TNL_TSO | + DEV_TX_OFFLOAD_GENEVE_TNL_TSO | + DEV_TX_OFFLOAD_IP_TNL_TSO | + DEV_TX_OFFLOAD_UDP_TNL_TSO); printf("TSO for tunneled packets is disabled\n"); - else { + } else { + uint64_t tso_offloads = (DEV_TX_OFFLOAD_VXLAN_TNL_TSO | + DEV_TX_OFFLOAD_GRE_TNL_TSO | + DEV_TX_OFFLOAD_IPIP_TNL_TSO | + DEV_TX_OFFLOAD_GENEVE_TNL_TSO | + DEV_TX_OFFLOAD_IP_TNL_TSO | + DEV_TX_OFFLOAD_UDP_TNL_TSO); + + ports[res->port_id].dev_conf.txmode.offloads |= + (tso_offloads & dev_info.tx_offload_capa); printf("TSO segment size for tunneled packets is %d\n", ports[res->port_id].tunnel_tso_segsz); @@ -3959,17 +4635,17 @@ cmd_tunnel_tso_set_parsed(void *parsed_result, * is not necessary for IPv6 tunneled pkts because there's no * checksum in IP header anymore. */ - check_tunnel_tso_nic_support(res->port_id); - if (!(ports[res->port_id].tx_ol_flags & - TESTPMD_TX_OFFLOAD_PARSE_TUNNEL)) + if (!ports[res->port_id].parse_tunnel) printf("Warning: csum parse_tunnel must be set " "so that tunneled packets are recognized\n"); - if (!(ports[res->port_id].tx_ol_flags & - TESTPMD_TX_OFFLOAD_OUTER_IP_CKSUM)) + if (!(ports[res->port_id].dev_conf.txmode.offloads & + DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) printf("Warning: csum set outer-ip must be set to hw " "if outer L3 is IPv4; not necessary for IPv6\n"); } + + cmd_reconfig_device_queue(res->port_id, 1, 1); } cmdline_parse_token_string_t cmd_tunnel_tso_set_tso = @@ -4294,8 +4970,9 @@ cmd_gso_show_parsed(void *parsed_result, if (gso_ports[res->cmd_pid].enable) { printf("Max GSO'd packet size: %uB\n" "Supported GSO types: TCP/IPv4, " - "VxLAN with inner TCP/IPv4 packet, " - "GRE with inner TCP/IPv4 packet\n", + "UDP/IPv4, VxLAN with inner " + "TCP/IPv4 packet, GRE with inner " + "TCP/IPv4 packet\n", gso_max_segment_size); } else printf("GSO is not enabled on Port %u\n", res->cmd_pid); @@ -5297,9 +5974,11 @@ static void cmd_create_bonded_device_parsed(void *parsed_result, port_id); /* Update number of ports */ - nb_ports = rte_eth_dev_count(); + nb_ports = rte_eth_dev_count_avail(); reconfig(port_id, res->socket); rte_eth_promiscuous_enable(port_id); + ports[port_id].need_setup = 0; + ports[port_id].port_status = RTE_PORT_STOPPED; } } @@ -5406,11 +6085,6 @@ static void cmd_set_bond_mon_period_parsed(void *parsed_result, struct cmd_set_bond_mon_period_result *res = parsed_result; int ret; - if (res->port_num >= nb_ports) { - printf("Port id %d must be less than %d\n", res->port_num, nb_ports); - return; - } - ret = rte_eth_bond_link_monitoring_set(res->port_num, res->period_ms); /* check the return value and print it if is < 0 */ @@ -5467,12 +6141,6 @@ cmd_set_bonding_agg_mode(void *parsed_result, struct cmd_set_bonding_agg_mode_policy_result *res = parsed_result; uint8_t policy = AGG_BANDWIDTH; - if (res->port_num >= nb_ports) { - printf("Port id %d must be less than %d\n", - res->port_num, nb_ports); - return; - } - if (!strcmp(res->policy, "bandwidth")) policy = AGG_BANDWIDTH; else if (!strcmp(res->policy, "stable")) @@ -5755,7 +6423,7 @@ cmdline_parse_token_string_t cmd_setpromisc_portall = "all"); cmdline_parse_token_num_t cmd_setpromisc_portnum = TOKEN_NUM_INITIALIZER(struct cmd_set_promisc_mode_result, port_num, - UINT8); + UINT16); cmdline_parse_token_string_t cmd_setpromisc_mode = TOKEN_STRING_INITIALIZER(struct cmd_set_promisc_mode_result, mode, "on#off"); @@ -6552,6 +7220,11 @@ static void cmd_showportall_parsed(void *parsed_result, } else if (!strcmp(res->what, "info")) RTE_ETH_FOREACH_DEV(i) port_infos_display(i); + else if (!strcmp(res->what, "summary")) { + port_summary_header_display(); + RTE_ETH_FOREACH_DEV(i) + port_summary_display(i); + } else if (!strcmp(res->what, "stats")) RTE_ETH_FOREACH_DEV(i) nic_stats_display(i); @@ -6579,14 +7252,14 @@ cmdline_parse_token_string_t cmd_showportall_port = TOKEN_STRING_INITIALIZER(struct cmd_showportall_result, port, "port"); cmdline_parse_token_string_t cmd_showportall_what = TOKEN_STRING_INITIALIZER(struct cmd_showportall_result, what, - "info#stats#xstats#fdir#stat_qmap#dcb_tc#cap"); + "info#summary#stats#xstats#fdir#stat_qmap#dcb_tc#cap"); cmdline_parse_token_string_t cmd_showportall_all = TOKEN_STRING_INITIALIZER(struct cmd_showportall_result, all, "all"); cmdline_parse_inst_t cmd_showportall = { .f = cmd_showportall_parsed, .data = NULL, .help_str = "show|clear port " - "info|stats|xstats|fdir|stat_qmap|dcb_tc|cap all", + "info|summary|stats|xstats|fdir|stat_qmap|dcb_tc|cap all", .tokens = { (void *)&cmd_showportall_show, (void *)&cmd_showportall_port, @@ -6616,6 +7289,10 @@ static void cmd_showport_parsed(void *parsed_result, nic_xstats_clear(res->portnum); } else if (!strcmp(res->what, "info")) port_infos_display(res->portnum); + else if (!strcmp(res->what, "summary")) { + port_summary_header_display(); + port_summary_display(res->portnum); + } else if (!strcmp(res->what, "stats")) nic_stats_display(res->portnum); else if (!strcmp(res->what, "xstats")) @@ -6637,7 +7314,7 @@ cmdline_parse_token_string_t cmd_showport_port = TOKEN_STRING_INITIALIZER(struct cmd_showport_result, port, "port"); cmdline_parse_token_string_t cmd_showport_what = TOKEN_STRING_INITIALIZER(struct cmd_showport_result, what, - "info#stats#xstats#fdir#stat_qmap#dcb_tc#cap"); + "info#summary#stats#xstats#fdir#stat_qmap#dcb_tc#cap"); cmdline_parse_token_num_t cmd_showport_portnum = TOKEN_NUM_INITIALIZER(struct cmd_showport_result, portnum, UINT16); @@ -6645,7 +7322,7 @@ cmdline_parse_inst_t cmd_showport = { .f = cmd_showport_parsed, .data = NULL, .help_str = "show|clear port " - "info|stats|xstats|fdir|stat_qmap|dcb_tc|cap " + "info|summary|stats|xstats|fdir|stat_qmap|dcb_tc|cap " "", .tokens = { (void *)&cmd_showport_show, @@ -7052,7 +7729,6 @@ static void cmd_quit_parsed(__attribute__((unused)) void *parsed_result, struct cmdline *cl, __attribute__((unused)) void *data) { - pmd_test_exit(); cmdline_quit(cl); } @@ -7124,6 +7800,50 @@ cmdline_parse_inst_t cmd_mac_addr = { }, }; +/* *** SET THE PEER ADDRESS FOR CERTAIN PORT *** */ +struct cmd_eth_peer_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t eth_peer; + portid_t port_id; + cmdline_fixed_string_t peer_addr; +}; + +static void cmd_set_eth_peer_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_eth_peer_result *res = parsed_result; + + if (test_done == 0) { + printf("Please stop forwarding first\n"); + return; + } + if (!strcmp(res->eth_peer, "eth-peer")) { + set_fwd_eth_peer(res->port_id, res->peer_addr); + fwd_config_setup(); + } +} +cmdline_parse_token_string_t cmd_eth_peer_set = + TOKEN_STRING_INITIALIZER(struct cmd_eth_peer_result, set, "set"); +cmdline_parse_token_string_t cmd_eth_peer = + TOKEN_STRING_INITIALIZER(struct cmd_eth_peer_result, eth_peer, "eth-peer"); +cmdline_parse_token_num_t cmd_eth_peer_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_eth_peer_result, port_id, UINT16); +cmdline_parse_token_string_t cmd_eth_peer_addr = + TOKEN_STRING_INITIALIZER(struct cmd_eth_peer_result, peer_addr, NULL); + +cmdline_parse_inst_t cmd_set_fwd_eth_peer = { + .f = cmd_set_eth_peer_parsed, + .data = NULL, + .help_str = "set eth-peer ", + .tokens = { + (void *)&cmd_eth_peer_set, + (void *)&cmd_eth_peer, + (void *)&cmd_eth_peer_port_id, + (void *)&cmd_eth_peer_addr, + NULL, + }, +}; /* *** CONFIGURE QUEUE STATS COUNTER MAPPINGS *** */ struct cmd_set_qmap_result { @@ -8129,6 +8849,89 @@ cmdline_parse_inst_t cmd_tunnel_udp_config = { }, }; +struct cmd_config_tunnel_udp_port { + cmdline_fixed_string_t port; + cmdline_fixed_string_t config; + portid_t port_id; + cmdline_fixed_string_t udp_tunnel_port; + cmdline_fixed_string_t action; + cmdline_fixed_string_t tunnel_type; + uint16_t udp_port; +}; + +static void +cmd_cfg_tunnel_udp_port_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_config_tunnel_udp_port *res = parsed_result; + struct rte_eth_udp_tunnel tunnel_udp; + int ret = 0; + + if (port_id_is_invalid(res->port_id, ENABLED_WARN)) + return; + + tunnel_udp.udp_port = res->udp_port; + + if (!strcmp(res->tunnel_type, "vxlan")) { + tunnel_udp.prot_type = RTE_TUNNEL_TYPE_VXLAN; + } else if (!strcmp(res->tunnel_type, "geneve")) { + tunnel_udp.prot_type = RTE_TUNNEL_TYPE_GENEVE; + } else { + printf("Invalid tunnel type\n"); + return; + } + + if (!strcmp(res->action, "add")) + ret = rte_eth_dev_udp_tunnel_port_add(res->port_id, + &tunnel_udp); + else + ret = rte_eth_dev_udp_tunnel_port_delete(res->port_id, + &tunnel_udp); + + if (ret < 0) + printf("udp tunneling port add error: (%s)\n", strerror(-ret)); +} + +cmdline_parse_token_string_t cmd_config_tunnel_udp_port_port = + TOKEN_STRING_INITIALIZER(struct cmd_config_tunnel_udp_port, port, + "port"); +cmdline_parse_token_string_t cmd_config_tunnel_udp_port_config = + TOKEN_STRING_INITIALIZER(struct cmd_config_tunnel_udp_port, config, + "config"); +cmdline_parse_token_num_t cmd_config_tunnel_udp_port_port_id = + TOKEN_NUM_INITIALIZER(struct cmd_config_tunnel_udp_port, port_id, + UINT16); +cmdline_parse_token_string_t cmd_config_tunnel_udp_port_tunnel_port = + TOKEN_STRING_INITIALIZER(struct cmd_config_tunnel_udp_port, + udp_tunnel_port, + "udp_tunnel_port"); +cmdline_parse_token_string_t cmd_config_tunnel_udp_port_action = + TOKEN_STRING_INITIALIZER(struct cmd_config_tunnel_udp_port, action, + "add#rm"); +cmdline_parse_token_string_t cmd_config_tunnel_udp_port_tunnel_type = + TOKEN_STRING_INITIALIZER(struct cmd_config_tunnel_udp_port, tunnel_type, + "vxlan#geneve"); +cmdline_parse_token_num_t cmd_config_tunnel_udp_port_value = + TOKEN_NUM_INITIALIZER(struct cmd_config_tunnel_udp_port, udp_port, + UINT16); + +cmdline_parse_inst_t cmd_cfg_tunnel_udp_port = { + .f = cmd_cfg_tunnel_udp_port_parsed, + .data = NULL, + .help_str = "port config udp_tunnel_port add|rm vxlan|geneve ", + .tokens = { + (void *)&cmd_config_tunnel_udp_port_port, + (void *)&cmd_config_tunnel_udp_port_config, + (void *)&cmd_config_tunnel_udp_port_port_id, + (void *)&cmd_config_tunnel_udp_port_tunnel_port, + (void *)&cmd_config_tunnel_udp_port_action, + (void *)&cmd_config_tunnel_udp_port_tunnel_type, + (void *)&cmd_config_tunnel_udp_port_value, + NULL, + }, +}; + /* *** GLOBAL CONFIG *** */ struct cmd_global_config_result { cmdline_fixed_string_t cmd; @@ -8472,7 +9275,7 @@ static void cmd_dump_parsed(void *parsed_result, else if (!strcmp(res->dump, "dump_mempool")) rte_mempool_list_dump(stdout); else if (!strcmp(res->dump, "dump_devargs")) - rte_eal_devargs_dump(stdout); + rte_devargs_dump(stdout); else if (!strcmp(res->dump, "dump_log_types")) rte_log_dump(stdout); } @@ -9773,6 +10576,8 @@ struct cmd_flow_director_result { cmdline_fixed_string_t tunnel_type; cmdline_fixed_string_t tunnel_id; uint32_t tunnel_id_value; + cmdline_fixed_string_t packet; + char filepath[]; }; static inline int @@ -9922,8 +10727,62 @@ cmd_flow_director_filter_parsed(void *parsed_result, return; } } else { - if (strcmp(res->mode_value, "IP")) { - printf("Please set mode to IP.\n"); + if (!strcmp(res->mode_value, "raw")) { +#ifdef RTE_LIBRTE_I40E_PMD + struct rte_pmd_i40e_flow_type_mapping + mapping[RTE_PMD_I40E_FLOW_TYPE_MAX]; + struct rte_pmd_i40e_pkt_template_conf conf; + uint16_t flow_type = str2flowtype(res->flow_type); + uint16_t i, port = res->port_id; + uint8_t add; + + memset(&conf, 0, sizeof(conf)); + + if (flow_type == RTE_ETH_FLOW_UNKNOWN) { + printf("Invalid flow type specified.\n"); + return; + } + ret = rte_pmd_i40e_flow_type_mapping_get(res->port_id, + mapping); + if (ret) + return; + if (mapping[flow_type].pctype == 0ULL) { + printf("Invalid flow type specified.\n"); + return; + } + for (i = 0; i < RTE_PMD_I40E_PCTYPE_MAX; i++) { + if (mapping[flow_type].pctype & (1ULL << i)) { + conf.input.pctype = i; + break; + } + } + + conf.input.packet = open_file(res->filepath, + &conf.input.length); + if (!conf.input.packet) + return; + if (!strcmp(res->drop, "drop")) + conf.action.behavior = + RTE_PMD_I40E_PKT_TEMPLATE_REJECT; + else + conf.action.behavior = + RTE_PMD_I40E_PKT_TEMPLATE_ACCEPT; + conf.action.report_status = + RTE_PMD_I40E_PKT_TEMPLATE_REPORT_ID; + conf.action.rx_queue = res->queue_id; + conf.soft_id = res->fd_id_value; + add = strcmp(res->ops, "del") ? 1 : 0; + ret = rte_pmd_i40e_flow_add_del_packet_template(port, + &conf, + add); + if (ret < 0) + printf("flow director config error: (%s)\n", + strerror(-ret)); + close_file(conf.input.packet); +#endif + return; + } else if (strcmp(res->mode_value, "IP")) { + printf("Please set mode to IP or raw.\n"); return; } entry.input.flow_type = str2flowtype(res->flow_type); @@ -10095,8 +10954,7 @@ cmdline_parse_token_string_t cmd_flow_director_flow = flow, "flow"); cmdline_parse_token_string_t cmd_flow_director_flow_type = TOKEN_STRING_INITIALIZER(struct cmd_flow_director_result, - flow_type, "ipv4-other#ipv4-frag#ipv4-tcp#ipv4-udp#ipv4-sctp#" - "ipv6-other#ipv6-frag#ipv6-tcp#ipv6-udp#ipv6-sctp#l2_payload"); + flow_type, NULL); cmdline_parse_token_string_t cmd_flow_director_ether = TOKEN_STRING_INITIALIZER(struct cmd_flow_director_result, ether, "ether"); @@ -10188,6 +11046,9 @@ cmdline_parse_token_string_t cmd_flow_director_mode_mac_vlan = cmdline_parse_token_string_t cmd_flow_director_mode_tunnel = TOKEN_STRING_INITIALIZER(struct cmd_flow_director_result, mode_value, "Tunnel"); +cmdline_parse_token_string_t cmd_flow_director_mode_raw = + TOKEN_STRING_INITIALIZER(struct cmd_flow_director_result, + mode_value, "raw"); cmdline_parse_token_string_t cmd_flow_director_mac = TOKEN_STRING_INITIALIZER(struct cmd_flow_director_result, mac, "mac"); @@ -10206,6 +11067,12 @@ cmdline_parse_token_string_t cmd_flow_director_tunnel_id = cmdline_parse_token_num_t cmd_flow_director_tunnel_id_value = TOKEN_NUM_INITIALIZER(struct cmd_flow_director_result, tunnel_id_value, UINT32); +cmdline_parse_token_string_t cmd_flow_director_packet = + TOKEN_STRING_INITIALIZER(struct cmd_flow_director_result, + packet, "packet"); +cmdline_parse_token_string_t cmd_flow_director_filepath = + TOKEN_STRING_INITIALIZER(struct cmd_flow_director_result, + filepath, NULL); cmdline_parse_inst_t cmd_add_del_ip_flow_director = { .f = cmd_flow_director_filter_parsed, @@ -10409,6 +11276,30 @@ cmdline_parse_inst_t cmd_add_del_tunnel_flow_director = { }, }; +cmdline_parse_inst_t cmd_add_del_raw_flow_director = { + .f = cmd_flow_director_filter_parsed, + .data = NULL, + .help_str = "flow_director_filter ... : Add or delete a raw flow " + "director entry on NIC", + .tokens = { + (void *)&cmd_flow_director_filter, + (void *)&cmd_flow_director_port_id, + (void *)&cmd_flow_director_mode, + (void *)&cmd_flow_director_mode_raw, + (void *)&cmd_flow_director_ops, + (void *)&cmd_flow_director_flow, + (void *)&cmd_flow_director_flow_type, + (void *)&cmd_flow_director_drop, + (void *)&cmd_flow_director_queue, + (void *)&cmd_flow_director_queue_id, + (void *)&cmd_flow_director_fd_id, + (void *)&cmd_flow_director_fd_id_value, + (void *)&cmd_flow_director_packet, + (void *)&cmd_flow_director_filepath, + NULL, + }, +}; + struct cmd_flush_flow_director_result { cmdline_fixed_string_t flush_flow_director; portid_t port_id; @@ -10488,11 +11379,6 @@ cmd_flow_director_mask_parsed(void *parsed_result, struct rte_eth_fdir_masks *mask; struct rte_port *port; - if (res->port_id > nb_ports) { - printf("Invalid port, range is [0, %d]\n", nb_ports - 1); - return; - } - port = &ports[res->port_id]; /** Check if the port is not started **/ if (port->port_status != RTE_PORT_STOPPED) { @@ -10685,15 +11571,10 @@ cmd_flow_director_flex_mask_parsed(void *parsed_result, struct rte_eth_fdir_info fdir_info; struct rte_eth_fdir_flex_mask flex_mask; struct rte_port *port; - uint32_t flow_type_mask; + uint64_t flow_type_mask; uint16_t i; int ret; - if (res->port_id > nb_ports) { - printf("Invalid port, range is [0, %d]\n", nb_ports - 1); - return; - } - port = &ports[res->port_id]; /** Check if the port is not started **/ if (port->port_status != RTE_PORT_STOPPED) { @@ -10738,7 +11619,7 @@ cmd_flow_director_flex_mask_parsed(void *parsed_result, return; } for (i = RTE_ETH_FLOW_UNKNOWN; i < RTE_ETH_FLOW_MAX; i++) { - if (flow_type_mask & (1 << i)) { + if (flow_type_mask & (1ULL << i)) { flex_mask.flow_type = i; fdir_set_flex_mask(res->port_id, &flex_mask); } @@ -10747,7 +11628,7 @@ cmd_flow_director_flex_mask_parsed(void *parsed_result, return; } flex_mask.flow_type = str2flowtype(res->flow_type); - if (!(flow_type_mask & (1 << flex_mask.flow_type))) { + if (!(flow_type_mask & (1ULL << flex_mask.flow_type))) { printf("Flow type %s not supported on port %d\n", res->flow_type, res->port_id); return; @@ -10845,11 +11726,6 @@ cmd_flow_director_flxpld_parsed(void *parsed_result, struct rte_port *port; int ret = 0; - if (res->port_id > nb_ports) { - printf("Invalid port, range is [0, %d]\n", nb_ports - 1); - return; - } - port = &ports[res->port_id]; /** Check if the port is not started **/ if (port->port_status != RTE_PORT_STOPPED) { @@ -11109,10 +11985,10 @@ cmd_get_hash_global_config_parsed(void *parsed_result, } for (i = 0; i < RTE_ETH_FLOW_MAX; i++) { - idx = i / UINT32_BIT; - offset = i % UINT32_BIT; + idx = i / UINT64_BIT; + offset = i % UINT64_BIT; if (!(info.info.global_conf.valid_bit_mask[idx] & - (1UL << offset))) + (1ULL << offset))) continue; str = flowtype_to_str(i); if (!str) @@ -11120,7 +11996,7 @@ cmd_get_hash_global_config_parsed(void *parsed_result, printf("Symmetric hash is %s globally for flow type %s " "by port %d\n", ((info.info.global_conf.sym_hash_enable_mask[idx] & - (1UL << offset)) ? "enabled" : "disabled"), str, + (1ULL << offset)) ? "enabled" : "disabled"), str, res->port_id); } } @@ -11181,12 +12057,12 @@ cmd_set_hash_global_config_parsed(void *parsed_result, RTE_ETH_HASH_FUNCTION_DEFAULT; ftype = str2flowtype(res->flow_type); - idx = ftype / (CHAR_BIT * sizeof(uint32_t)); - offset = ftype % (CHAR_BIT * sizeof(uint32_t)); - info.info.global_conf.valid_bit_mask[idx] |= (1UL << offset); + idx = ftype / UINT64_BIT; + offset = ftype % UINT64_BIT; + info.info.global_conf.valid_bit_mask[idx] |= (1ULL << offset); if (!strcmp(res->enable, "enable")) info.info.global_conf.sym_hash_enable_mask[idx] |= - (1UL << offset); + (1ULL << offset); ret = rte_eth_dev_filter_ctrl(res->port_id, RTE_ETH_FILTER_HASH, RTE_ETH_FILTER_SET, &info); if (ret < 0) @@ -11491,7 +12367,7 @@ struct cmd_config_l2_tunnel_eth_type_result { cmdline_fixed_string_t port; cmdline_fixed_string_t config; cmdline_fixed_string_t all; - uint8_t id; + portid_t id; cmdline_fixed_string_t l2_tunnel; cmdline_fixed_string_t l2_tunnel_type; cmdline_fixed_string_t eth_type; @@ -11513,7 +12389,7 @@ cmdline_parse_token_string_t cmd_config_l2_tunnel_eth_type_all_str = cmdline_parse_token_num_t cmd_config_l2_tunnel_eth_type_id = TOKEN_NUM_INITIALIZER (struct cmd_config_l2_tunnel_eth_type_result, - id, UINT8); + id, UINT16); cmdline_parse_token_string_t cmd_config_l2_tunnel_eth_type_l2_tunnel = TOKEN_STRING_INITIALIZER (struct cmd_config_l2_tunnel_eth_type_result, @@ -11626,7 +12502,7 @@ struct cmd_config_l2_tunnel_en_dis_result { cmdline_fixed_string_t port; cmdline_fixed_string_t config; cmdline_fixed_string_t all; - uint8_t id; + portid_t id; cmdline_fixed_string_t l2_tunnel; cmdline_fixed_string_t l2_tunnel_type; cmdline_fixed_string_t en_dis; @@ -11647,7 +12523,7 @@ cmdline_parse_token_string_t cmd_config_l2_tunnel_en_dis_all_str = cmdline_parse_token_num_t cmd_config_l2_tunnel_en_dis_id = TOKEN_NUM_INITIALIZER (struct cmd_config_l2_tunnel_en_dis_result, - id, UINT8); + id, UINT16); cmdline_parse_token_string_t cmd_config_l2_tunnel_en_dis_l2_tunnel = TOKEN_STRING_INITIALIZER (struct cmd_config_l2_tunnel_en_dis_result, @@ -12624,6 +13500,10 @@ cmd_set_tx_loopback_parsed( if (ret == -ENOTSUP) ret = rte_pmd_bnxt_set_tx_loopback(res->port_id, is_on); #endif +#if defined RTE_LIBRTE_DPAA_BUS && defined RTE_LIBRTE_DPAA_PMD + if (ret == -ENOTSUP) + ret = rte_pmd_dpaa_set_tx_loopback(res->port_id, is_on); +#endif switch (ret) { case 0: @@ -13008,19 +13888,29 @@ cmd_set_macsec_offload_on_parsed( portid_t port_id = res->port_id; int en = (strcmp(res->en_on_off, "on") == 0) ? 1 : 0; int rp = (strcmp(res->rp_on_off, "on") == 0) ? 1 : 0; + struct rte_eth_dev_info dev_info; if (port_id_is_invalid(port_id, ENABLED_WARN)) return; + if (!port_is_stopped(port_id)) { + printf("Please stop port %d first\n", port_id); + return; + } - ports[port_id].tx_ol_flags |= TESTPMD_TX_OFFLOAD_MACSEC; + rte_eth_dev_info_get(port_id, &dev_info); + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MACSEC_INSERT) { #ifdef RTE_LIBRTE_IXGBE_PMD - ret = rte_pmd_ixgbe_macsec_enable(port_id, en, rp); + ret = rte_pmd_ixgbe_macsec_enable(port_id, en, rp); #endif + } RTE_SET_USED(en); RTE_SET_USED(rp); switch (ret) { case 0: + ports[port_id].dev_conf.txmode.offloads |= + DEV_TX_OFFLOAD_MACSEC_INSERT; + cmd_reconfig_device_queue(port_id, 1, 1); break; case -ENODEV: printf("invalid port_id %d\n", port_id); @@ -13091,18 +13981,27 @@ cmd_set_macsec_offload_off_parsed( { struct cmd_macsec_offload_off_result *res = parsed_result; int ret = -ENOTSUP; + struct rte_eth_dev_info dev_info; portid_t port_id = res->port_id; if (port_id_is_invalid(port_id, ENABLED_WARN)) return; + if (!port_is_stopped(port_id)) { + printf("Please stop port %d first\n", port_id); + return; + } - ports[port_id].tx_ol_flags &= ~TESTPMD_TX_OFFLOAD_MACSEC; + rte_eth_dev_info_get(port_id, &dev_info); + if (dev_info.tx_offload_capa & DEV_TX_OFFLOAD_MACSEC_INSERT) { #ifdef RTE_LIBRTE_IXGBE_PMD - ret = rte_pmd_ixgbe_macsec_disable(port_id); + ret = rte_pmd_ixgbe_macsec_disable(port_id); #endif - + } switch (ret) { case 0: + ports[port_id].dev_conf.txmode.offloads &= + ~DEV_TX_OFFLOAD_MACSEC_INSERT; + cmd_reconfig_device_queue(port_id, 1, 1); break; case -ENODEV: printf("invalid port_id %d\n", port_id); @@ -14096,20 +14995,14 @@ static void cmd_set_port_tm_hierarchy_default_parsed(void *parsed_result, p = &ports[port_id]; - /* Port tm flag */ - if (p->softport.tm_flag == 0) { - printf(" tm not enabled on port %u (error)\n", port_id); - return; - } - /* Forward mode: tm */ - if (strcmp(cur_fwd_config.fwd_eng->fwd_mode_name, "tm")) { - printf(" tm mode not enabled(error)\n"); + if (strcmp(cur_fwd_config.fwd_eng->fwd_mode_name, "softnic")) { + printf(" softnicfwd mode not enabled(error)\n"); return; } /* Set the default tm hierarchy */ - p->softport.tm.default_hierarchy_enable = 1; + p->softport.default_tm_hierarchy_enable = 1; } cmdline_parse_inst_t cmd_set_port_tm_hierarchy_default = { @@ -14128,6 +15021,951 @@ cmdline_parse_inst_t cmd_set_port_tm_hierarchy_default = { }; #endif +/** Set VXLAN encapsulation details */ +struct cmd_set_vxlan_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t vxlan; + cmdline_fixed_string_t pos_token; + cmdline_fixed_string_t ip_version; + uint32_t vlan_present:1; + uint32_t vni; + uint16_t udp_src; + uint16_t udp_dst; + cmdline_ipaddr_t ip_src; + cmdline_ipaddr_t ip_dst; + uint16_t tci; + struct ether_addr eth_src; + struct ether_addr eth_dst; +}; + +cmdline_parse_token_string_t cmd_set_vxlan_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, set, "set"); +cmdline_parse_token_string_t cmd_set_vxlan_vxlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, vxlan, "vxlan"); +cmdline_parse_token_string_t cmd_set_vxlan_vxlan_with_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, vxlan, + "vxlan-with-vlan"); +cmdline_parse_token_string_t cmd_set_vxlan_ip_version = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token, + "ip-version"); +cmdline_parse_token_string_t cmd_set_vxlan_ip_version_value = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, ip_version, + "ipv4#ipv6"); +cmdline_parse_token_string_t cmd_set_vxlan_vni = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token, + "vni"); +cmdline_parse_token_num_t cmd_set_vxlan_vni_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_vxlan_result, vni, UINT32); +cmdline_parse_token_string_t cmd_set_vxlan_udp_src = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token, + "udp-src"); +cmdline_parse_token_num_t cmd_set_vxlan_udp_src_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_vxlan_result, udp_src, UINT16); +cmdline_parse_token_string_t cmd_set_vxlan_udp_dst = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token, + "udp-dst"); +cmdline_parse_token_num_t cmd_set_vxlan_udp_dst_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_vxlan_result, udp_dst, UINT16); +cmdline_parse_token_string_t cmd_set_vxlan_ip_src = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token, + "ip-src"); +cmdline_parse_token_ipaddr_t cmd_set_vxlan_ip_src_value = + TOKEN_IPADDR_INITIALIZER(struct cmd_set_vxlan_result, ip_src); +cmdline_parse_token_string_t cmd_set_vxlan_ip_dst = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token, + "ip-dst"); +cmdline_parse_token_ipaddr_t cmd_set_vxlan_ip_dst_value = + TOKEN_IPADDR_INITIALIZER(struct cmd_set_vxlan_result, ip_dst); +cmdline_parse_token_string_t cmd_set_vxlan_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token, + "vlan-tci"); +cmdline_parse_token_num_t cmd_set_vxlan_vlan_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_vxlan_result, tci, UINT16); +cmdline_parse_token_string_t cmd_set_vxlan_eth_src = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token, + "eth-src"); +cmdline_parse_token_etheraddr_t cmd_set_vxlan_eth_src_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_vxlan_result, eth_src); +cmdline_parse_token_string_t cmd_set_vxlan_eth_dst = + TOKEN_STRING_INITIALIZER(struct cmd_set_vxlan_result, pos_token, + "eth-dst"); +cmdline_parse_token_etheraddr_t cmd_set_vxlan_eth_dst_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_vxlan_result, eth_dst); + +static void cmd_set_vxlan_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_vxlan_result *res = parsed_result; + union { + uint32_t vxlan_id; + uint8_t vni[4]; + } id = { + .vxlan_id = rte_cpu_to_be_32(res->vni) & RTE_BE32(0x00ffffff), + }; + + if (strcmp(res->vxlan, "vxlan") == 0) + vxlan_encap_conf.select_vlan = 0; + else if (strcmp(res->vxlan, "vxlan-with-vlan") == 0) + vxlan_encap_conf.select_vlan = 1; + if (strcmp(res->ip_version, "ipv4") == 0) + vxlan_encap_conf.select_ipv4 = 1; + else if (strcmp(res->ip_version, "ipv6") == 0) + vxlan_encap_conf.select_ipv4 = 0; + else + return; + rte_memcpy(vxlan_encap_conf.vni, &id.vni[1], 3); + vxlan_encap_conf.udp_src = rte_cpu_to_be_16(res->udp_src); + vxlan_encap_conf.udp_dst = rte_cpu_to_be_16(res->udp_dst); + if (vxlan_encap_conf.select_ipv4) { + IPV4_ADDR_TO_UINT(res->ip_src, vxlan_encap_conf.ipv4_src); + IPV4_ADDR_TO_UINT(res->ip_dst, vxlan_encap_conf.ipv4_dst); + } else { + IPV6_ADDR_TO_ARRAY(res->ip_src, vxlan_encap_conf.ipv6_src); + IPV6_ADDR_TO_ARRAY(res->ip_dst, vxlan_encap_conf.ipv6_dst); + } + if (vxlan_encap_conf.select_vlan) + vxlan_encap_conf.vlan_tci = rte_cpu_to_be_16(res->tci); + rte_memcpy(vxlan_encap_conf.eth_src, res->eth_src.addr_bytes, + ETHER_ADDR_LEN); + rte_memcpy(vxlan_encap_conf.eth_dst, res->eth_dst.addr_bytes, + ETHER_ADDR_LEN); +} + +cmdline_parse_inst_t cmd_set_vxlan = { + .f = cmd_set_vxlan_parsed, + .data = NULL, + .help_str = "set vxlan ip-version ipv4|ipv6 vni udp-src" + " udp-dst ip-src ip-dst " + " eth-src eth-dst ", + .tokens = { + (void *)&cmd_set_vxlan_set, + (void *)&cmd_set_vxlan_vxlan, + (void *)&cmd_set_vxlan_ip_version, + (void *)&cmd_set_vxlan_ip_version_value, + (void *)&cmd_set_vxlan_vni, + (void *)&cmd_set_vxlan_vni_value, + (void *)&cmd_set_vxlan_udp_src, + (void *)&cmd_set_vxlan_udp_src_value, + (void *)&cmd_set_vxlan_udp_dst, + (void *)&cmd_set_vxlan_udp_dst_value, + (void *)&cmd_set_vxlan_ip_src, + (void *)&cmd_set_vxlan_ip_src_value, + (void *)&cmd_set_vxlan_ip_dst, + (void *)&cmd_set_vxlan_ip_dst_value, + (void *)&cmd_set_vxlan_eth_src, + (void *)&cmd_set_vxlan_eth_src_value, + (void *)&cmd_set_vxlan_eth_dst, + (void *)&cmd_set_vxlan_eth_dst_value, + NULL, + }, +}; + +cmdline_parse_inst_t cmd_set_vxlan_with_vlan = { + .f = cmd_set_vxlan_parsed, + .data = NULL, + .help_str = "set vxlan-with-vlan ip-version ipv4|ipv6 vni " + " udp-src udp-dst ip-src ip-dst" + " vlan-tci eth-src eth-dst" + " ", + .tokens = { + (void *)&cmd_set_vxlan_set, + (void *)&cmd_set_vxlan_vxlan_with_vlan, + (void *)&cmd_set_vxlan_ip_version, + (void *)&cmd_set_vxlan_ip_version_value, + (void *)&cmd_set_vxlan_vni, + (void *)&cmd_set_vxlan_vni_value, + (void *)&cmd_set_vxlan_udp_src, + (void *)&cmd_set_vxlan_udp_src_value, + (void *)&cmd_set_vxlan_udp_dst, + (void *)&cmd_set_vxlan_udp_dst_value, + (void *)&cmd_set_vxlan_ip_src, + (void *)&cmd_set_vxlan_ip_src_value, + (void *)&cmd_set_vxlan_ip_dst, + (void *)&cmd_set_vxlan_ip_dst_value, + (void *)&cmd_set_vxlan_vlan, + (void *)&cmd_set_vxlan_vlan_value, + (void *)&cmd_set_vxlan_eth_src, + (void *)&cmd_set_vxlan_eth_src_value, + (void *)&cmd_set_vxlan_eth_dst, + (void *)&cmd_set_vxlan_eth_dst_value, + NULL, + }, +}; + +/** Set NVGRE encapsulation details */ +struct cmd_set_nvgre_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t nvgre; + cmdline_fixed_string_t pos_token; + cmdline_fixed_string_t ip_version; + uint32_t tni; + cmdline_ipaddr_t ip_src; + cmdline_ipaddr_t ip_dst; + uint16_t tci; + struct ether_addr eth_src; + struct ether_addr eth_dst; +}; + +cmdline_parse_token_string_t cmd_set_nvgre_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, set, "set"); +cmdline_parse_token_string_t cmd_set_nvgre_nvgre = + TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, nvgre, "nvgre"); +cmdline_parse_token_string_t cmd_set_nvgre_nvgre_with_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, nvgre, + "nvgre-with-vlan"); +cmdline_parse_token_string_t cmd_set_nvgre_ip_version = + TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token, + "ip-version"); +cmdline_parse_token_string_t cmd_set_nvgre_ip_version_value = + TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, ip_version, + "ipv4#ipv6"); +cmdline_parse_token_string_t cmd_set_nvgre_tni = + TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token, + "tni"); +cmdline_parse_token_num_t cmd_set_nvgre_tni_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_nvgre_result, tni, UINT32); +cmdline_parse_token_string_t cmd_set_nvgre_ip_src = + TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token, + "ip-src"); +cmdline_parse_token_num_t cmd_set_nvgre_ip_src_value = + TOKEN_IPADDR_INITIALIZER(struct cmd_set_nvgre_result, ip_src); +cmdline_parse_token_string_t cmd_set_nvgre_ip_dst = + TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token, + "ip-dst"); +cmdline_parse_token_ipaddr_t cmd_set_nvgre_ip_dst_value = + TOKEN_IPADDR_INITIALIZER(struct cmd_set_nvgre_result, ip_dst); +cmdline_parse_token_string_t cmd_set_nvgre_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token, + "vlan-tci"); +cmdline_parse_token_num_t cmd_set_nvgre_vlan_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_nvgre_result, tci, UINT16); +cmdline_parse_token_string_t cmd_set_nvgre_eth_src = + TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token, + "eth-src"); +cmdline_parse_token_etheraddr_t cmd_set_nvgre_eth_src_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_nvgre_result, eth_src); +cmdline_parse_token_string_t cmd_set_nvgre_eth_dst = + TOKEN_STRING_INITIALIZER(struct cmd_set_nvgre_result, pos_token, + "eth-dst"); +cmdline_parse_token_etheraddr_t cmd_set_nvgre_eth_dst_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_nvgre_result, eth_dst); + +static void cmd_set_nvgre_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_nvgre_result *res = parsed_result; + union { + uint32_t nvgre_tni; + uint8_t tni[4]; + } id = { + .nvgre_tni = rte_cpu_to_be_32(res->tni) & RTE_BE32(0x00ffffff), + }; + + if (strcmp(res->nvgre, "nvgre") == 0) + nvgre_encap_conf.select_vlan = 0; + else if (strcmp(res->nvgre, "nvgre-with-vlan") == 0) + nvgre_encap_conf.select_vlan = 1; + if (strcmp(res->ip_version, "ipv4") == 0) + nvgre_encap_conf.select_ipv4 = 1; + else if (strcmp(res->ip_version, "ipv6") == 0) + nvgre_encap_conf.select_ipv4 = 0; + else + return; + rte_memcpy(nvgre_encap_conf.tni, &id.tni[1], 3); + if (nvgre_encap_conf.select_ipv4) { + IPV4_ADDR_TO_UINT(res->ip_src, nvgre_encap_conf.ipv4_src); + IPV4_ADDR_TO_UINT(res->ip_dst, nvgre_encap_conf.ipv4_dst); + } else { + IPV6_ADDR_TO_ARRAY(res->ip_src, nvgre_encap_conf.ipv6_src); + IPV6_ADDR_TO_ARRAY(res->ip_dst, nvgre_encap_conf.ipv6_dst); + } + if (nvgre_encap_conf.select_vlan) + nvgre_encap_conf.vlan_tci = rte_cpu_to_be_16(res->tci); + rte_memcpy(nvgre_encap_conf.eth_src, res->eth_src.addr_bytes, + ETHER_ADDR_LEN); + rte_memcpy(nvgre_encap_conf.eth_dst, res->eth_dst.addr_bytes, + ETHER_ADDR_LEN); +} + +cmdline_parse_inst_t cmd_set_nvgre = { + .f = cmd_set_nvgre_parsed, + .data = NULL, + .help_str = "set nvgre ip-version tni ip-src" + " ip-dst eth-src " + " eth-dst ", + .tokens = { + (void *)&cmd_set_nvgre_set, + (void *)&cmd_set_nvgre_nvgre, + (void *)&cmd_set_nvgre_ip_version, + (void *)&cmd_set_nvgre_ip_version_value, + (void *)&cmd_set_nvgre_tni, + (void *)&cmd_set_nvgre_tni_value, + (void *)&cmd_set_nvgre_ip_src, + (void *)&cmd_set_nvgre_ip_src_value, + (void *)&cmd_set_nvgre_ip_dst, + (void *)&cmd_set_nvgre_ip_dst_value, + (void *)&cmd_set_nvgre_eth_src, + (void *)&cmd_set_nvgre_eth_src_value, + (void *)&cmd_set_nvgre_eth_dst, + (void *)&cmd_set_nvgre_eth_dst_value, + NULL, + }, +}; + +cmdline_parse_inst_t cmd_set_nvgre_with_vlan = { + .f = cmd_set_nvgre_parsed, + .data = NULL, + .help_str = "set nvgre-with-vlan ip-version tni " + " ip-src ip-dst vlan-tci " + " eth-src eth-dst ", + .tokens = { + (void *)&cmd_set_nvgre_set, + (void *)&cmd_set_nvgre_nvgre_with_vlan, + (void *)&cmd_set_nvgre_ip_version, + (void *)&cmd_set_nvgre_ip_version_value, + (void *)&cmd_set_nvgre_tni, + (void *)&cmd_set_nvgre_tni_value, + (void *)&cmd_set_nvgre_ip_src, + (void *)&cmd_set_nvgre_ip_src_value, + (void *)&cmd_set_nvgre_ip_dst, + (void *)&cmd_set_nvgre_ip_dst_value, + (void *)&cmd_set_nvgre_vlan, + (void *)&cmd_set_nvgre_vlan_value, + (void *)&cmd_set_nvgre_eth_src, + (void *)&cmd_set_nvgre_eth_src_value, + (void *)&cmd_set_nvgre_eth_dst, + (void *)&cmd_set_nvgre_eth_dst_value, + NULL, + }, +}; + +/** Set L2 encapsulation details */ +struct cmd_set_l2_encap_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t l2_encap; + cmdline_fixed_string_t pos_token; + cmdline_fixed_string_t ip_version; + uint32_t vlan_present:1; + uint16_t tci; + struct ether_addr eth_src; + struct ether_addr eth_dst; +}; + +cmdline_parse_token_string_t cmd_set_l2_encap_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, set, "set"); +cmdline_parse_token_string_t cmd_set_l2_encap_l2_encap = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, l2_encap, "l2_encap"); +cmdline_parse_token_string_t cmd_set_l2_encap_l2_encap_with_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, l2_encap, + "l2_encap-with-vlan"); +cmdline_parse_token_string_t cmd_set_l2_encap_ip_version = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, pos_token, + "ip-version"); +cmdline_parse_token_string_t cmd_set_l2_encap_ip_version_value = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, ip_version, + "ipv4#ipv6"); +cmdline_parse_token_string_t cmd_set_l2_encap_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, pos_token, + "vlan-tci"); +cmdline_parse_token_num_t cmd_set_l2_encap_vlan_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_l2_encap_result, tci, UINT16); +cmdline_parse_token_string_t cmd_set_l2_encap_eth_src = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, pos_token, + "eth-src"); +cmdline_parse_token_etheraddr_t cmd_set_l2_encap_eth_src_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_l2_encap_result, eth_src); +cmdline_parse_token_string_t cmd_set_l2_encap_eth_dst = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_encap_result, pos_token, + "eth-dst"); +cmdline_parse_token_etheraddr_t cmd_set_l2_encap_eth_dst_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_l2_encap_result, eth_dst); + +static void cmd_set_l2_encap_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_l2_encap_result *res = parsed_result; + + if (strcmp(res->l2_encap, "l2_encap") == 0) + l2_encap_conf.select_vlan = 0; + else if (strcmp(res->l2_encap, "l2_encap-with-vlan") == 0) + l2_encap_conf.select_vlan = 1; + if (strcmp(res->ip_version, "ipv4") == 0) + l2_encap_conf.select_ipv4 = 1; + else if (strcmp(res->ip_version, "ipv6") == 0) + l2_encap_conf.select_ipv4 = 0; + else + return; + if (l2_encap_conf.select_vlan) + l2_encap_conf.vlan_tci = rte_cpu_to_be_16(res->tci); + rte_memcpy(l2_encap_conf.eth_src, res->eth_src.addr_bytes, + ETHER_ADDR_LEN); + rte_memcpy(l2_encap_conf.eth_dst, res->eth_dst.addr_bytes, + ETHER_ADDR_LEN); +} + +cmdline_parse_inst_t cmd_set_l2_encap = { + .f = cmd_set_l2_encap_parsed, + .data = NULL, + .help_str = "set l2_encap ip-version ipv4|ipv6" + " eth-src eth-dst ", + .tokens = { + (void *)&cmd_set_l2_encap_set, + (void *)&cmd_set_l2_encap_l2_encap, + (void *)&cmd_set_l2_encap_ip_version, + (void *)&cmd_set_l2_encap_ip_version_value, + (void *)&cmd_set_l2_encap_eth_src, + (void *)&cmd_set_l2_encap_eth_src_value, + (void *)&cmd_set_l2_encap_eth_dst, + (void *)&cmd_set_l2_encap_eth_dst_value, + NULL, + }, +}; + +cmdline_parse_inst_t cmd_set_l2_encap_with_vlan = { + .f = cmd_set_l2_encap_parsed, + .data = NULL, + .help_str = "set l2_encap-with-vlan ip-version ipv4|ipv6" + " vlan-tci eth-src eth-dst ", + .tokens = { + (void *)&cmd_set_l2_encap_set, + (void *)&cmd_set_l2_encap_l2_encap_with_vlan, + (void *)&cmd_set_l2_encap_ip_version, + (void *)&cmd_set_l2_encap_ip_version_value, + (void *)&cmd_set_l2_encap_vlan, + (void *)&cmd_set_l2_encap_vlan_value, + (void *)&cmd_set_l2_encap_eth_src, + (void *)&cmd_set_l2_encap_eth_src_value, + (void *)&cmd_set_l2_encap_eth_dst, + (void *)&cmd_set_l2_encap_eth_dst_value, + NULL, + }, +}; + +/** Set L2 decapsulation details */ +struct cmd_set_l2_decap_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t l2_decap; + cmdline_fixed_string_t pos_token; + uint32_t vlan_present:1; +}; + +cmdline_parse_token_string_t cmd_set_l2_decap_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_decap_result, set, "set"); +cmdline_parse_token_string_t cmd_set_l2_decap_l2_decap = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_decap_result, l2_decap, + "l2_decap"); +cmdline_parse_token_string_t cmd_set_l2_decap_l2_decap_with_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_l2_decap_result, l2_decap, + "l2_decap-with-vlan"); + +static void cmd_set_l2_decap_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_l2_decap_result *res = parsed_result; + + if (strcmp(res->l2_decap, "l2_decap") == 0) + l2_decap_conf.select_vlan = 0; + else if (strcmp(res->l2_decap, "l2_decap-with-vlan") == 0) + l2_decap_conf.select_vlan = 1; +} + +cmdline_parse_inst_t cmd_set_l2_decap = { + .f = cmd_set_l2_decap_parsed, + .data = NULL, + .help_str = "set l2_decap", + .tokens = { + (void *)&cmd_set_l2_decap_set, + (void *)&cmd_set_l2_decap_l2_decap, + NULL, + }, +}; + +cmdline_parse_inst_t cmd_set_l2_decap_with_vlan = { + .f = cmd_set_l2_decap_parsed, + .data = NULL, + .help_str = "set l2_decap-with-vlan", + .tokens = { + (void *)&cmd_set_l2_decap_set, + (void *)&cmd_set_l2_decap_l2_decap_with_vlan, + NULL, + }, +}; + +/** Set MPLSoGRE encapsulation details */ +struct cmd_set_mplsogre_encap_result { + cmdline_fixed_string_t set; + cmdline_fixed_string_t mplsogre; + cmdline_fixed_string_t pos_token; + cmdline_fixed_string_t ip_version; + uint32_t vlan_present:1; + uint32_t label; + cmdline_ipaddr_t ip_src; + cmdline_ipaddr_t ip_dst; + uint16_t tci; + struct ether_addr eth_src; + struct ether_addr eth_dst; +}; + +cmdline_parse_token_string_t cmd_set_mplsogre_encap_set = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, set, + "set"); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_mplsogre_encap = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, mplsogre, + "mplsogre_encap"); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_mplsogre_encap_with_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + mplsogre, "mplsogre_encap-with-vlan"); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_ip_version = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "ip-version"); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_ip_version_value = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + ip_version, "ipv4#ipv6"); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_label = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "label"); +cmdline_parse_token_num_t cmd_set_mplsogre_encap_label_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_mplsogre_encap_result, label, + UINT32); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_ip_src = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "ip-src"); +cmdline_parse_token_ipaddr_t cmd_set_mplsogre_encap_ip_src_value = + TOKEN_IPADDR_INITIALIZER(struct cmd_set_mplsogre_encap_result, ip_src); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_ip_dst = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "ip-dst"); +cmdline_parse_token_ipaddr_t cmd_set_mplsogre_encap_ip_dst_value = + TOKEN_IPADDR_INITIALIZER(struct cmd_set_mplsogre_encap_result, ip_dst); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_vlan = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "vlan-tci"); +cmdline_parse_token_num_t cmd_set_mplsogre_encap_vlan_value = + TOKEN_NUM_INITIALIZER(struct cmd_set_mplsogre_encap_result, tci, + UINT16); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_eth_src = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "eth-src"); +cmdline_parse_token_etheraddr_t cmd_set_mplsogre_encap_eth_src_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_mplsogre_encap_result, + eth_src); +cmdline_parse_token_string_t cmd_set_mplsogre_encap_eth_dst = + TOKEN_STRING_INITIALIZER(struct cmd_set_mplsogre_encap_result, + pos_token, "eth-dst"); +cmdline_parse_token_etheraddr_t cmd_set_mplsogre_encap_eth_dst_value = + TOKEN_ETHERADDR_INITIALIZER(struct cmd_set_mplsogre_encap_result, + eth_dst); + +static void cmd_set_mplsogre_encap_parsed(void *parsed_result, + __attribute__((unused)) struct cmdline *cl, + __attribute__((unused)) void *data) +{ + struct cmd_set_mplsogre_encap_result *res = parsed_result; + union { + uint32_t mplsogre_label; + uint8_t label[3]; + } id = { + .mplsogre_label = + rte_cpu_to_be_32(res->label) & RTE_BE32(0x00ffffff), + }; + + if (strcmp(res->mplsogre, "mplsogre_encap") == 0) + mplsogre_encap_conf.select_vlan = 0; + else if (strcmp(res->mplsogre, "mplsogre_encap-with-vlan") == 0) + mplsogre_encap_conf.select_vlan = 1; + if (strcmp(res->ip_version, "ipv4") == 0) + mplsogre_encap_conf.select_ipv4 = 1; + else if (strcmp(res->ip_version, "ipv6") == 0) + mplsogre_encap_conf.select_ipv4 = 0; + else + return; + rte_memcpy(mplsogre_encap_conf.label, &id.label[1], 3); + if (mplsogre_encap_conf.select_ipv4) { + IPV4_ADDR_TO_UINT(res->ip_src, mplsogre_encap_conf.ipv4_src); + IPV4_ADDR_TO_UINT(res->ip_dst, mplsogre_encap_conf.ipv4_dst); + } else { + IPV6_ADDR_TO_ARRAY(res->ip_src, mplsogre_encap_conf.ipv6_src); + IPV6_ADDR_TO_ARRAY(res->ip_dst, mplsogre_encap_conf.ipv6_dst); + } + if (mplsogre_encap_conf.select_vlan) + mplsogre_encap_conf.vlan_tci = rte_cpu_to_be_16(res->tci); + rte_memcpy(mplsogre_encap_conf.eth_src, res->eth_src.addr_bytes, + ETHER_ADDR_LEN); + rte_memcpy(mplsogre_encap_conf.eth_dst, res->eth_dst.addr_bytes, + ETHER_ADDR_LEN); +} + +cmdline_parse_inst_t cmd_set_mplsogre_encap = { + .f = cmd_set_mplsogre_encap_parsed, + .data = NULL, + .help_str = "set mplsogre_encap ip-version ipv4|ipv6 label