Install Data Plane Development Kit (DPDK) on Intel 82599 10 GbE Controller

# Access to the server as root user, update and upgrade linux
root@itcs-p4srv-02:/home/mmisamor# apt update
root@itcs-p4srv-02:/home/mmisamor# apt upgrade

# Install DPDK 
root@itcs-p4srv-02:/home/mmisamor# apt install dpdk
root@itcs-p4srv-02:/home/mmisamor# apt install dpdk-dev

# Check the driver of the target
# Website documentation for the IXGBE driver of this target: https://doc.dpdk.org/guides/nics/ixgbe.html
root@itcs-p4srv-02:/home/mmisamor# lspci | grep net
02:00.0 Ethernet controller: Intel Corporation 82599 10 Gigabit Network Connection (rev 01)
05:00.0 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01)
05:00.1 Ethernet controller: Intel Corporation 82599ES 10-Gigabit SFI/SFP+ Network Connection (rev 01)
08:00.0 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01)
08:00.1 Ethernet controller: Intel Corporation I350 Gigabit Network Connection (rev 01)

# Download and install Cisco TRex 
# NOTE: get the URL from the Resources Cisco TRex website to have the latest version
root@itcs-p4srv-02:/home/mmisamor# cd /opt
root@itcs-p4srv-02:/opt# mkdir trex
root@itcs-p4srv-02:/opt# cd trex
root@itcs-p4srv-02:/opt/trex# wget https://trex-tgn.cisco.com/trex/release/v2.92.tar.gz --no-check-certificate

# Untar the files
root@itcs-p4srv-02:/opt/trex# tar xzf v2.92.tar.gz

# Check the CPU resources available on the target
# In our case, we have 2 sockets x 8 cores per socket  x 2 threads = 32 cores in total 
root@itcs-p4srv-02:/opt/trex# lscpu
Architecture:                    x86_64
CPU op-mode(s):                  32-bit, 64-bit
Byte Order:                      Little Endian
Address sizes:                   46 bits physical, 48 bits virtual
CPU(s):                          32
On-line CPU(s) list:             0-31
Thread(s) per core:              2
Core(s) per socket:              8
Socket(s):                       2
NUMA node(s):                    1
Vendor ID:                       GenuineIntel
CPU family:                      6
Model:                           62
Model name:                      Intel(R) Xeon(R) CPU E5-2650 v2 @ 2.60GHz
Stepping:                        4
CPU MHz:                         1197.312
CPU max MHz:                     3400.0000
CPU min MHz:                     1200.0000
BogoMIPS:                        5186.94
Virtualization:                  VT-x
L1d cache:                       512 KiB
L1i cache:                       512 KiB
L2 cache:                        4 MiB
L3 cache:                        40 MiB
NUMA node0 CPU(s):               0-31
Vulnerability Itlb multihit:     KVM: Mitigation: Split huge pages
Vulnerability L1tf:              Mitigation; PTE Inversion; VMX conditional cache flushes, SMT vulnerable
Vulnerability Mds:               Mitigation; Clear CPU buffers; SMT vulnerable
Vulnerability Meltdown:          Mitigation; PTI
Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp
Vulnerability Spectre v1:        Mitigation; usercopy/swapgs barriers and __user pointer sanitization
Vulnerability Spectre v2:        Mitigation; Full generic retpoline, IBPB conditional, IBRS_FW, STIBP conditional, RSB filling
Vulnerability Srbds:             Not affected
Vulnerability Tsx async abort:   Not affected
Flags:                           fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm
                                 pbe syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfm
                                 perf pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic popcnt t
                                 sc_deadline_timer aes xsave avx f16c rdrand lahf_lm cpuid_fault epb pti ssbd ibrs ibpb stibp tpr_shadow vnmi flexpri
                                 ority ept vpid fsgsbase smep erms xsaveopt dtherm ida arat pln pts md_clear flush_l1d

# Check memory resources available on the target
# In our case, we have 256 GB of memory
root@itcs-p4srv-02:/opt/trex# lsmem
RANGE                                  SIZE  STATE REMOVABLE BLOCK
0x0000000000000000-0x00000000bfffffff    3G online       yes   0-2
0x0000000100000000-0x000000403fffffff  253G online       yes 4-256

Memory block size:         1G
Total online memory:     256G
Total offline memory:      0B

# Enter in the BIOS and modify GRUB
root@itcs-p4srv-02:/opt/trex# vi /etc/default/
# Select GRUB folder
grub

# Add the following content to the line GRUB_CMDLINE_LINUX_DEFAULT="maybe-ubiquity"
# In this case, we are taking 16 pieces of 2048 MB (see hugepages parameters) but this can be even higher, for example, 32 pieces of 4096 MB
hugepagesz=1G hugepages=16 hugepagesz=2M hugepages=2048 iommu=pt intel_iommu=on
# Now this line should look like this
GRUB_CMDLINE_LINUX_DEFAULT="maybe-ubiquity hugepagesz=1G hugepages=16 hugepagesz=2M hugepages=2048 iommu=pt intel_iommu=on"

# Update GRUB, reboot and access again as root user
root@itcs-p4srv-02:/opt/trex# update-grub
root@itcs-p4srv-02:/opt/trex# reboot

# Verify that IOMMU is enabled on the kernel
root@itcs-p4srv-02:/home/mmisamor# dmesg | grep -i IOMMU
[    0.000000] Command line: BOOT_IMAGE=/vmlinuz-5.4.0-90-generic root=/dev/mapper/ubuntu--vg-ubuntu--lv ro maybe-ubiquity hugepagesz=1G hugepages=16 hugepagesz=2M hugepages=2048 iommu=pt intel_iommu=on
[    1.190056] Kernel command line: BOOT_IMAGE=/vmlinuz-5.4.0-90-generic root=/dev/mapper/ubuntu--vg-ubuntu--lv ro maybe-ubiquity hugepagesz=1G hugepages=16 hugepagesz=2M hugepages=2048 iommu=pt intel_iommu=on
[    1.190313] DMAR: IOMMU enabled
[    4.974205] iommu: Default domain type: Passthrough (set via kernel command line)

# Set up DPDK ports
root@itcs-p4srv-02:/home/mmisamor# cd /opt/trex/v2.92
root@itcs-p4srv-02:/opt/trex/v2.92# ./dpdk_setup_ports.py
Network devices using DPDK-compatible driver
============================================
0000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
0000:03:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
0000:13:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=
0000:13:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection' drv=igb_uio unused=

Network devices using kernel driver
===================================
0000:02:00.0 '82545EM Gigabit Ethernet Controller (Copper)' if=eth2 drv=e1000 unused=igb_uio *Active*

# DPDK bind status
root@itcs-p4srv-02:/opt/trex/v2.92# dpdk-devbind.py -s

Network devices using kernel driver
===================================
0000:01:00.0 'MT27500 Family [ConnectX-3] 1003' if=ibp1s0 drv=mlx4_core unused=vfio-pci
0000:02:00.0 '82599 10 Gigabit Network Connection 1557' if=ens2 drv=ixgbe unused=vfio-pci
0000:05:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=enp5s0f0 drv=ixgbe unused=vfio-pci
0000:05:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=enp5s0f1 drv=ixgbe unused=vfio-pci
0000:08:00.0 'I350 Gigabit Network Connection 1521' if=eno1 drv=igb unused=vfio-pci *Active*
0000:08:00.1 'I350 Gigabit Network Connection 1521' if=eno2 drv=igb unused=vfio-pci

# Set up IOMMU on the Ubuntu Server 20.04 target [this step is very important]
root@itcs-p4srv-02:/opt/trex/v2.92# sudo echo Y | sudo tee /sys/module/vfio/parameters/enable_unsafe_noiommu_mode

# Set up the configuration on the ports
# In this case, in server itcs-p4srv-02 we will use ens2 and enp5s0f1 -> port ID 0 and 2
root@itcs-p4srv-02:/opt/trex/v2.92# ./dpdk_setup_ports.py -i
By default, IP based configuration file will be created. Do you want to use MAC based config? (y/N)n
+----+------+---------+-------------------+------------------------------------------------+--------+----------+----------+
| ID | NUMA |   PCI   |        MAC        |                      Name                      | Driver | Linux IF |  Active  |
+====+======+=========+===================+================================================+========+==========+==========+
| 0  | 0    | 02:00.0 | 0c:c4:7a:bd:1d:70 | 82599 10 Gigabit Network Connection            | ixgbe  | ens2     |          |
+----+------+---------+-------------------+------------------------------------------------+--------+----------+----------+
| 1  | 0    | 05:00.0 | 00:1e:67:f2:80:10 | 82599ES 10-Gigabit SFI/SFP+ Network Connection | ixgbe  | enp5s0f0 |          |
+----+------+---------+-------------------+------------------------------------------------+--------+----------+----------+
| 2  | 0    | 05:00.1 | 00:1e:67:f2:80:11 | 82599ES 10-Gigabit SFI/SFP+ Network Connection | ixgbe  | enp5s0f1 |          |
+----+------+---------+-------------------+------------------------------------------------+--------+----------+----------+
| 3  | 0    | 08:00.0 | 00:1e:67:44:92:88 | I350 Gigabit Network Connection                | igb    | eno1     | *Active* |
+----+------+---------+-------------------+------------------------------------------------+--------+----------+----------+
| 4  | 0    | 08:00.1 | 00:1e:67:44:92:89 | I350 Gigabit Network Connection                | igb    | eno2     |          |
+----+------+---------+-------------------+------------------------------------------------+--------+----------+----------+
Please choose an even number of interfaces from the list above, either by ID, PCI or Linux IF
Stateful will use order of interfaces: Client1 Server1 Client2 Server2 etc. for flows.
Stateless can be in any order.
Enter list of interfaces separated by space (for example: 1 3) : 0 2

For interface 0, assuming loopback to its dual interface 2.
Putting IP 1.1.1.1, default gw 2.2.2.2 Change it?(y/N).n
For interface 2, assuming loopback to its dual interface 0.
Putting IP 2.2.2.2, default gw 1.1.1.1 Change it?(y/N).n
Print preview of generated config? (Y/n)y
### Config file generated by dpdk_setup_ports.py ###

- version: 2
  interfaces: ['02:00.0', '05:00.1']
  port_info:
      - ip: 1.1.1.1
        default_gw: 2.2.2.2
      - ip: 2.2.2.2
        default_gw: 1.1.1.1

  platform:
      master_thread_id: 0
      latency_thread_id: 8
      dual_if:
        - socket: 0
          threads: [1,2,3,4,5,6,7,16,17,18,19,20,21,22,23]


Save the config to file? (Y/n)y
Default filename is /etc/trex_cfg.yaml
Press ENTER to confirm or enter new file:
Saved to /etc/trex_cfg.yaml.

# Bind the ports to DPDK
# Here you need to use the PCI number from the table above. In this case there are 02:00.0 and 05:00.1
root@itcs-p4srv-02:/opt/trex/v2.92# dpdk-devbind.py -b vfio-pci 02:00.0
root@itcs-p4srv-02:/opt/trex/v2.92# dpdk-devbind.py -b vfio-pci 05:00.1

# Check the DPDK status
# At this point, the two ports set above SHOULD BE CHANGED into 'Network devices using DPDK-compatible driver' category
root@itcs-p4srv-02:/opt/trex/v2.92# dpdk-devbind.py -s

Network devices using DPDK-compatible driver
============================================
0000:02:00.0 '82599 10 Gigabit Network Connection 1557' drv=vfio-pci unused=ixgbe
0000:05:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' drv=vfio-pci unused=ixgbe

Network devices using kernel driver
===================================
0000:01:00.0 'MT27500 Family [ConnectX-3] 1003' if=ibp1s0 drv=mlx4_core unused=vfio-pci
0000:05:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=enp5s0f0 drv=ixgbe unused=vfio-pci
0000:08:00.0 'I350 Gigabit Network Connection 1521' if=eno1 drv=igb unused=vfio-pci *Active*
0000:08:00.1 'I350 Gigabit Network Connection 1521' if=eno2 drv=igb unused=vfio-pci

Install Cisco TRex EMU

# Get Cisco TRex EMU from GitHub
root@itcs-dhcpsrv-01:/opt/trex/v2.92/trex_emu# wget https://github.com/cisco-system-traffic-generator/trex-emu/archive/master.zip
root@itcs-dhcpsrv-01:/opt/trex/v2.92/trex_emu# unzip master.zip

# Install Cisco TRex EMU
root@itcs-dhcpsrv-01:/opt/trex/v2.92/trex_emu# cd trex-emu-master/scripts
root@itcs-dhcpsrv-01:/opt/trex/v2.92/trex_emu/trex-emu-master/scripts# source ./b
root@itcs-dhcpsrv-01:/opt/trex/v2.92/trex_emu/trex-emu-master/scripts# cd ../src/
root@itcs-dhcpsrv-01:/opt/trex/v2.92/trex_emu/trex-emu-master/scr# go install -v cmd/trex-emu.go
root@itcs-dhcpsrv-01:/opt/trex/v2.92/trex_emu/trex-emu-master/scr# cd ../scripts/

# Build the environment
root@itcs-dhcpsrv-01:/opt/trex/v2.92/trex_emu/trex-emu-master/scripts# ./utility_script build -v
root@itcs-dhcpsrv-01:/opt/trex/v2.92/trex_emu/trex-emu-master/scripts# source ./b

# Test the environment
root@itcs-dhcpsrv-01:/opt/trex/v2.92/trex_emu/trex-emu-master/scripts# ./utility_script test -v

# The result should be like this:
PASS                                                                               │shutdown -                     Shutdown the server
ok      emu/core        0.059s                                                     │stats -                        Show various statistics
ok      emu/plugins/appsim      0.556s                                             │vlan -                         Configures VLAN tagging for a port.
ok      emu/plugins/arp 0.078s                                                     │
ok      emu/plugins/cdp 0.037s                                                     │Stateless Commands:
ok      emu/plugins/dhcpv4      0.082s                                             │
ok      emu/plugins/dhcpv4srv   0.116s                                             │pause -                        Pause active traffic on specified ports on TRex
ok      emu/plugins/dhcpv6      0.048s                                             │profiles -                     Get loaded to server profiles information
ok      emu/plugins/dns 0.095s                                                     │push -                         Push a pcap file
ok      emu/plugins/dot1x       0.031s                                             │resume -                       Resume active traffic on specified ports on TRex
ok      emu/plugins/field_engine        5.055s                                     │service -                      Configures port for service mode.
ok      emu/plugins/icmp        0.032s                                             │start -                        Start selected traffic on specified ports on TRex
ok      emu/plugins/igmp        0.368s                                             │stop -                         Stop active traffic on specified ports on TRex
ok      emu/plugins/ipfix       1.301s                                             │streams -                      Get loaded to server streams information
ok      emu/plugins/ipv6        1.095s                                             │update -                       Update port(s) speed currently active
ok      emu/plugins/lldp        0.030s                                             │trex>emu
ok      emu/plugins/mdns        0.118s                                             │'emu' is an unrecognized command. type 'help' or '?' for a list
ok      emu/plugins/tdl 0.056s                                                     │
ok      emu/plugins/transport   2.136s                                             │trex>exit
Go tests passed :-)