Vmware中搭建 DPVS 环境

环境

• Vmware: VMware® Workstation 16 Pro
• 虚拟机：
  • 机器1 部署 DPDK 和 DPVS
  • 机器2 作为 RS 部署 nginx

DPVS VM

• 系统：CentOS Linux release 7.8.2003 (Core)

• CPU core：2

• 内存：4GB

• 网卡：

  • 网卡1：NAT模式

  • 网卡2：桥接模式

Real Server VM

• 系统：Ubuntu 20.04.1 LTS
• CPU core: 2
• 内存：4GB
• 网卡：
  • 网卡1：NAT模式

编译

下载

下载 dpdk-18.11.2 和 dpvs v1.8.4

安装必要组件

yum groupinstall "Development Tools"
yum install net-tools
yum install kernel-devel-$(uname -r)
yum install numactl-devel

编译DPDK

# 复制补丁并打补丁
cd dpvs-1.8.4/
cp patch/dpdk-stable-18.11.2/* ../dpdk-stable-18.11.2/
cd ../dpdk-stable-18.11.2/
patch -p 1 < 0001-kni-use-netlink-event-for-multicast-driver-part.patch
patch -p 1 < 0002-net-support-variable-IP-header-len-for-checksum-API.patch
# 针对虚拟机网卡修改判断条件
sed -i "s/pci_intx_mask_supported(dev)/pci_intx_mask_supported(dev)||true/g" kernel/linux/igb_uio/igb_uio.c
make config T=x86_64-native-linuxapp-gcc
make # or make -j40 to save time, where 40 is the cpu core number.
export RTE_SDK=$PWD

设置 Hugepage

echo 1024 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
mkdir /mnt/huge
mount -t hugetlbfs nodev /mnt/huge

加载模块

modprobe uio
insmod build/kmod/igb_uio.ko
insmod build/kmod/rte_kni.ko carrier=on

设置网卡

./usertools/dpdk-devbind.py --status
./usertools/dpdk-devbind.py -b igb_uio 0000:06:00.0 # 根据 status 输出修改网卡代号

编译 DPVS

# 安装依赖，可能还有其他依赖，根据需要安装
yum -y install popt* -y
yum -y install openssl-devel
yum -y install libnl libnl-devel
# 设置 RTE_SDK
cd dpdk-stable-18.11.2/
export RTE_SDK=$PWD
cd <path-of-dpvs>

make # or "make -j40" to speed up.
make install # 安装在 <path-of-dpvs>/bin/

部署启动

需要先设置配置文件 /etc/dpvs.conf，文件修改自 conf/dpvs.conf.single-nic.sample，因为虚拟网卡不支持 FDIR，所以只能是以单核运行，只设置一个slave worker，同时把连接池数量降低以免内存不足。

 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
! This is dpvs default configuration file.
!
! The attribute "<init>" denotes the configuration item at initialization stage. Item of
! this type is configured oneshoot and not reloadable. If invalid value configured in the
! file, dpvs would use its default value.
!
! Note that dpvs configuration file supports the following comment type:
!   * line comment: using '#" or '!'
!   * inline range comment: using '<' and '>', put comment in between
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

! global config
global_defs {
    log_level   DEBUG
    ! log_file    /var/log/dpvs.log
    ! log_async_mode    on
}

! netif config
netif_defs {
    <init> pktpool_size     524287
    <init> pktpool_cache    256

    <init> device dpdk0 {
        rx {
            queue_number        1
            descriptor_number   1024
            rss                 all
        }
        tx {
            queue_number        1
            descriptor_number   1024
        }
        fdir {
            mode                signature
            pballoc             64k
            status              matched
        }
        ! promisc_mode
        kni_name                dpdk0.kni
    }
}

! worker config (lcores)
worker_defs {
    <init> worker cpu0 {
        type    master
        cpu_id  0
    }

    <init> worker cpu1 {
        type    slave
        cpu_id  1
        port    dpdk0 {
            rx_queue_ids     0
            tx_queue_ids     0
            ! isol_rx_cpu_ids  9
            ! isol_rxq_ring_sz 1048576
        }
    }
}

! timer config
timer_defs {
    # cpu job loops to schedule dpdk timer management
    schedule_interval    500
}

! dpvs neighbor config
neigh_defs {
    <init> unres_queue_length  128
    timeout                    60
}

! dpvs ipv4 config
ipv4_defs {
    forwarding                 off
    <init> default_ttl         64
    fragment {
        <init> bucket_number   4096
        <init> bucket_entries  16
        <init> max_entries     4096
        <init> ttl             1
    }
}

! dpvs ipv6 config
ipv6_defs {
    disable                     off
    forwarding                  off
    route6 {
        <init> method           hlist
        recycle_time            10
    }
}

! control plane config
ctrl_defs {
    lcore_msg {
        <init> ring_size                4096
        sync_msg_timeout_us             20000
        priority_level                  low
    }
    ipc_msg {
        <init> unix_domain /var/run/dpvs_ctrl
    }
}

! ipvs config
ipvs_defs {
    conn {
        <init> conn_pool_size       131072
        <init> conn_pool_cache      256
        conn_init_timeout           3
        ! expire_quiescent_template
        ! fast_xmit_close
        <init> redirect           on
    }

    udp {
        ! defence_udp_drop
        uoa_mode        opp
        uoa_max_trail   3
        timeout {
            normal      300
            last        3
        }
    }

    tcp {
        ! defence_tcp_drop
        timeout {
            none        2
            established 90
            syn_sent    3
            syn_recv    30
            fin_wait    7
            time_wait   7
            close       3
            close_wait  7
            last_ack    7
            listen      120
            synack      30
            last        2
        }
        synproxy {
            synack_options {
                mss             1452
                ttl             63
                sack
                ! wscale
                ! timestamp
            }
            ! defer_rs_syn
            rs_syn_max_retry    3
            ack_storm_thresh    10
            max_ack_saved       3
            conn_reuse_state {
                close
                time_wait
                ! fin_wait
                ! close_wait
                ! last_ack
           }
        }
    }
}

! sa_pool config
sa_pool {
    pool_hash_size   16
}
 

然后启动 dpvs 和设置 DR 模式

cd <path-of-dpvs>/bin
./dpvs &
# 虚拟网卡网段为 192.168.253.0/24，设置 VIP 为 192.168.253.254
./dpip addr add 192.168.253.1/24 dev dpdk0
./dpip addr add 192.168.100.254/32 dev dpdk0
./dpip route add 192.168.253.0/24 dev dpdk0
./ipvsadm -A -t 192.168.253.254:80 -s rr
# 192.168.253.128 为 Real Server IP
./ipvsadm -a -t 192.168.253.254:80 -r 192.168.253.128 -g

接下来要到 RS 上设置 lo 地址为 VIP 并且关闭 arp

ip addr add 192.168.100.254/32 dev lo
sysctl -w net.ipv4.conf.lo.arp_ignore=1
sysctl -w net.ipv4.conf.ens37.arp_ignore=1 #需要抑制对应网卡的arp，不然还是会出现vip 对应 mac地址不对的问题

然后在可以连通 VIP 得机器上测试是否连通

 curl 192.168.253.254

Tunnel 模式

以下为同网段 Tunnel 模式配置，网段为192.168.214.0/24，三个机器分别为:

• DPVS 192.168.214.128
• RS1 192.168.214.130
• RS2 192.168.214.131

DPVS 设置脚本

#!/bin/sh
./dpip addr add 192.168.214.128/20 dev dpdk0
./dpip addr add 192.168.214.254/32 dev dpdk0
./dpip route add default via 192.168.214.1 src 192.168.214.128 dev dpdk0
./ipvsadm -A -t 192.168.214.254:80 -s rr
./ipvsadm -a -t 192.168.214.254:80 -r 192.168.214.130 -i
./ipvsadm -a -t 192.168.214.254:80 -r 192.168.214.131 -i

RS 设置

# 设置tunnel
ifconfig tunl0 192.168.214.254 netmask 255.255.255.255 up

将以下内容写入到 /etc/sysctl.conf，抑制 arp 和关闭反向路由验证

net.ipv4.conf.tunl0.arp_ignore = 1
net.ipv4.conf.tunl0.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
net.ipv4.conf.tunl0.rp_filter = 0
net.ipv4.conf.all.rp_filter = 0

执行 sysctl -p 将内容生效