K8S Flannel Latency Measured by QPERF

Qperf

How to Get it

apt-get install -y make gcc libc-dev
wget https://www.openfabrics.org/downloads/qperf/qperf-0.4.9.tar.gz
tar zxvf qperf-0.4.9.tar.gz
cd qperf-0.4.9/
./configure
make

Where the binary is in

src/qperf

How to use it

Qperf Server

./qperf

Qperf Client

root@mariadbcluster-1:/# ./qperf -v mariadbcluster-0.mariadbcluster tcp_bw tcp_lat
tcp_bw:
    bw              =  91.5 MB/sec
    msg_rate        =   1.4 K/sec
    send_cost       =  32.6 sec/GB
    recv_cost       =  49.6 sec/GB
    send_cpus_used  =   301 % cpus
    recv_cpus_used  =   454 % cpus
tcp_lat:
    latency        =  63.8 us
    msg_rate       =  15.7 K/sec
    loc_cpus_used  =   294 % cpus
    rem_cpus_used  =   546 % cpus

Where the latency is measured in Flannel network.

./qperf -v msg_size:1:64K:*2 mariadbcluster-0.mariadbcluster tcp_bw tcp_lat

More Command Line

Changing message size

root@mariadbcluster-1:/# ./qperf -v -oo msg_size:1:64K:*2  mariadbcluster-0.mariadbcluster tcp_lat
tcp_lat:
    latency        =  72.4 us
    msg_rate       =  13.8 K/sec
    msg_size       =     1 bytes
    loc_cpus_used  =   286 % cpus
    rem_cpus_used  =   422 % cpus
tcp_lat:
    latency        =  77.1 us
    msg_rate       =    13 K/sec
    msg_size       =     2 bytes
    loc_cpus_used  =   297 % cpus
    rem_cpus_used  =   442 % cpus
tcp_lat:
    latency        =  102 us
    msg_rate       =  9.8 K/sec
    msg_size       =    4 bytes
    loc_cpus_used  =  297 % cpus
    rem_cpus_used  =  423 % cpus
tcp_lat:
    latency        =  78.4 us
    msg_rate       =  12.8 K/sec
    msg_size       =     8 bytes
    loc_cpus_used  =   255 % cpus
    rem_cpus_used  =   458 % cpus
tcp_lat:
    latency        =  63.8 us
    msg_rate       =  15.7 K/sec
    msg_size       =    16 bytes
    loc_cpus_used  =   311 % cpus
    rem_cpus_used  =   516 % cpus
tcp_lat:
    latency        =  83.6 us
    msg_rate       =    12 K/sec
    msg_size       =    32 bytes
    loc_cpus_used  =   304 % cpus
    rem_cpus_used  =   520 % cpus
tcp_lat:
    latency        =  80.2 us
    msg_rate       =  12.5 K/sec
    msg_size       =    64 bytes
    loc_cpus_used  =   324 % cpus
    rem_cpus_used  =   414 % cpus
tcp_lat:
    latency        =  61.5 us
    msg_rate       =  16.3 K/sec
    msg_size       =   128 bytes
    loc_cpus_used  =   318 % cpus
    rem_cpus_used  =   329 % cpus
tcp_lat:
    latency        =  76.3 us
    msg_rate       =  13.1 K/sec
    msg_size       =   256 bytes
    loc_cpus_used  =   278 % cpus
    rem_cpus_used  =   358 % cpus
tcp_lat:
    latency        =  83.1 us
    msg_rate       =    12 K/sec
    msg_size       =   512 bytes
    loc_cpus_used  =   268 % cpus
    rem_cpus_used  =   446 % cpus
tcp_lat:
    latency        =    99 us
    msg_rate       =  10.1 K/sec
    msg_size       =     1 KiB (1,024)
    loc_cpus_used  =   270 % cpus
    rem_cpus_used  =   407 % cpus
tcp_lat:
    latency        =  121 us
    msg_rate       =  8.3 K/sec
    msg_size       =    2 KiB (2,048)
    loc_cpus_used  =  320 % cpus
    rem_cpus_used  =  364 % cpus
tcp_lat:
    latency        =   153 us
    msg_rate       =  6.55 K/sec
    msg_size       =     4 KiB (4,096)
    loc_cpus_used  =   260 % cpus
    rem_cpus_used  =   544 % cpus
tcp_lat:
    latency        =   206 us
    msg_rate       =  4.87 K/sec
    msg_size       =     8 KiB (8,192)
    loc_cpus_used  =   359 % cpus
    rem_cpus_used  =   447 % cpus
tcp_lat:
    latency        =  345 us
    msg_rate       =  2.9 K/sec
    msg_size       =   16 KiB (16,384)
    loc_cpus_used  =  289 % cpus
    rem_cpus_used  =  540 % cpus
tcp_lat:
    latency        =   507 us
    msg_rate       =  1.97 K/sec
    msg_size       =    32 KiB (32,768)
    loc_cpus_used  =   278 % cpus
    rem_cpus_used  =   476 % cpus
tcp_lat:
    latency        =  1.32 ms
    msg_rate       =   755 /sec
    msg_size       =    64 KiB (65,536)
    loc_cpus_used  =   312 % cpus
    rem_cpus_used  =   666 % cpus

Fix certain message size

./qperf -oo msg_size:1K  mariadbcluster-0.mariadbcluster tcp_lat