编译OpenWrt系统

准备编译环境

系统为CentOS7.x mini

$ sudo yum groupinstall "Development Tools"
$ sudo yum install ncurses-devel
$ sudo yum install python3
$ sudo yum install zlib-static openssl-devel wget

下载代码

$ git clone https://git.openwrt.org/openwrt/openwrt.git
$ cd openwrt/
$ ./scripts/feeds update -a
$ ./scripts/feeds install -a

配置

$ make menuconfig

OpenWrt configuration

  • Select [x86] in [Target System]
  • Select [x86_64] in [Subtarget]
  • Select [Build the OpenWrt SDK] for cross-compilation environment
  • Select [Use glibc] in [Advanced configuration options] (for evelopers) then [ToolChain Options] and [C Library implementation]

Kernel configuration

  • CONFIG_VFIO_IOMMU_TYPE1=y
  • CONFIG_VFIO_VIRQFD=y
  • CONFIG_VFIO=y
  • CONFIG_VFIO_NOIOMMU=y
  • CONFIG_VFIO_PCI=y
  • CONFIG_VFIO_PCI_MMAP=y
  • CONFIG_HUGETLBFS=y
  • CONFIG_HUGETLB_PAGE=y
  • CONFIG_PROC_PAGE_MONITOR=y
$ vi .config
CONFIG_KERNEL_CONFIG_VFIO_IOMMU_TYPE1=y
CONFIG_KERNEL_CONFIG_VFIO_VIRQFD=y
CONFIG_KERNEL_CONFIG_VFIO=y
CONFIG_KERNEL_CONFIG_VFIO_NOIOMMU=y
CONFIG_KERNEL_CONFIG_VFIO_PCI=y
CONFIG_KERNEL_CONFIG_VFIO_PCI_MMAP=y
CONFIG_KERNEL_CONFIG_HUGETLBFS=y
CONFIG_KERNEL_CONFIG_HUGETLB_PAGE=y
CONFIG_KERNEL_CONFIG_PROC_PAGE_MONITOR=y
CONFIG_KERNEL_CONFIG_UIO=y

$ vi config/Config-kernel.in
config KERNEL_CONFIG_VFIO_IOMMU_TYPE1
        bool "KERNEL_CONFIG_VFIO_IOMMU_TYPE1"
        default y
config KERNEL_CONFIG_VFIO_VIRQFD
        bool "KERNEL_CONFIG_VFIO_VIRQFD"
        default y
config KERNEL_CONFIG_VFIO
        bool "KERNEL_CONFIG_VFIO"
        default y
config KERNEL_CONFIG_VFIO_NOIOMMU
        bool "KERNEL_CONFIG_VFIO_NOIOMMU"
        default y
config KERNEL_CONFIG_VFIO_PCI
        bool "KERNEL_CONFIG_VFIO_PCI"
        default y
config KERNEL_CONFIG_VFIO_PCI_MMAP
        bool "KERNEL_CONFIG_VFIO_PCI_MMAP"
        default y

config KERNEL_CONFIG_HUGETLBFS
        bool "KERNEL_CONFIG_HUGETLBFS"
        default y
config KERNEL_CONFIG_HUGETLB_PAGE
        bool "KERNEL_CONFIG_HUGETLB_PAGE"
        default y
config KERNEL_CONFIG_PROC_PAGE_MONITOR
        bool "KERNEL_CONFIG_PROC_PAGE_MONITOR"
        default y

config KERNEL_CONFIG_UIO
        bool "KERNEL_CONFIG_UIO"
        default y

config KERNEL_NUMA_BALANCING
        bool "KERNEL_CONFIG_NUMA_BALANCING"
        default y
config KERNEL_NUMA_BALANCING
        bool "KERNEL_CONFIG_NUMA_BALANCING"
        default y
config KERNEL_NUMA
        bool "KERNEL_CONFIG_NUMA"
        default y
config KERNEL_AMD_NUMA
        bool "KERNEL_CONFIG_AMD_NUMA"
        default y
config KERNEL_X86_64_ACPI_NUMA
        bool "KERNEL_CONFIG_X86_64_ACPI_NUMA"
        default y
config KERNEL_USE_PERCPU_NUMA_NODE_ID
        bool "KERNEL_CONFIG_USE_PERCPU_NUMA_NODE_ID"
        default y
config KERNEL_ACPI_NUMA
        bool "KERNEL_CONFIG_ACPI_NUMA"
        default y

准备编译代码文件

$ make download​ V=s

编译

$ make -j1 V=sc

NUMA is required to run DPDK in x86.

For compiling the NUMA lib, run libtool –version to ensure the libtool version >= 2.2, otherwise the compilation will fail with errors.

git clone https://github.com/numactl/numactl.git
cd numactl
git checkout v2.0.13 -b v2.0.13
./autogen.sh
autoconf -i
export PATH=<OpenWrt SDK>/glibc/openwrt-sdk-x86-64_gcc-8.3.0_glibc.Linux-x86_64/staging_dir/toolchain-x86_64_gcc-8.3.0_glibc/bin/:$PATH
./configure CC=x86_64-openwrt-linux-gnu-gcc --prefix=<OpenWrt SDK toolchain dir>
make install

编译DPDK

$ tar -xvf /opt/dl/dpdk-18.02.1.tar.gz

$ export STAGING_DIR=<OpenWrt sdk>/glibc/openwrt-sdk-x86-64_gcc-8.3.0_glibc.Linux-x86_64/staging_dir
$ export RTE_SDK=`pwd`
$ export RTE_KERNELDIR=<OpenWrt Root>/build_dir/target-x86_64_glibc/linux-x86_64/linux-4.19.81/
$ export RTE_TARGET=x86_64-native-linuxapp-gcc

/* 编译动态库版本 */
$ sed -i "s/^CONFIG_RTE_BUILD_SHARED_LIB=n/CONFIG_RTE_BUILD_SHARED_LIB=y/g" config/common_base

/* D525(march=atom) 这类处理器指令集不全(SSE4_1 SSE4_2)需要修改配置 J1900不缺少(SSE4_1 SSE4_2)
/* 根本代码中的说明dpdk需要sse4_1的_mm_testz_si128函数，而支持sse4_1的最小 march=corei7 所以J1900最终应该是使用corei7编译 并不能使用默认native因为还是有部分指令集有缺失 */
$ sed -i "s/native/corei7/g" mk/machine/native/rte.vars.mk

$ make config T=x86_64-native-linuxapp-gcc O=x86_64-native-linuxapp-gcc
$ make EXTRA_CLFAGS="-D_GNU_SOURCE" CROSS=x86_64-openwrt-linux-gnu- O=x86_64-native-linuxapp-gcc

$ cd examples
$ make EXTRA_CLFAGS="-D_GNU_SOURCE" CROSS=x86_64-openwrt-linux-gnu-

编译例子

把DPDK由静态库方式改为编译成动态库后，原本正常的应用程序就不能运行了。 在初始化的时候，rte_eth_dev_count()总是返回0，而用dpdk_nic_bind.py –status查看端口是绑定成功的。

DPDK编译成动态库后，PMD的各个驱动就单独编译成了一个个的.so文件，而在应用程序中没有指定需要链接的.so文件，因此就导致了检测不到端口的问题。

PMD有好几种驱动，为了更好的移植性，建议在Makefile中指定链接所有驱动的.so文件, LDLIBS += -lrte_pmd_e1000 -lrte_pmd_i40e -lrte_pmd_ixgbe

如果应用运行在虚拟机环境的话，还需要指定librte_pmd_virtio_uio.so和librte_pmd_vmxnet3_uio.so。

$ vi mk/rte.app.mk
ifeq ($(LINK_USING_CC),1)
O_TO_EXE = $(CC) -o $@ $(CFLAGS) $(EXTRA_CFLAGS) $(OBJS-y) -lrte_pmd_e1000 $(call linkerprefix, \
        $(LDLIBS) $(LDFLAGS) $(LDFLAGS_$(@)) $(EXTRA_LDFLAGS) \
        $(MAPFLAGS))
else
O_TO_EXE = $(LD) -o $@ $(OBJS-y) \
        $(LDLIBS) $(LDFLAGS) $(LDFLAGS_$(@)) $(EXTRA_LDFLAGS) \
        $(MAPFLAGS)
endif