diff options
Diffstat (limited to 'automated/linux')
-rw-r--r-- | automated/linux/nginx-server/linux-ip.sh | 75 | ||||
-rwxr-xr-x | automated/linux/nginx-server/nginx-server.sh | 142 | ||||
-rw-r--r-- | automated/linux/nginx-server/odp-dpdk.sh | 91 |
3 files changed, 194 insertions, 114 deletions
diff --git a/automated/linux/nginx-server/linux-ip.sh b/automated/linux/nginx-server/linux-ip.sh new file mode 100644 index 0000000..a8937ce --- /dev/null +++ b/automated/linux/nginx-server/linux-ip.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +exit_error() { + echo "-- SERVER ERROR" + journalctl -u nginx + lava-test-case server_up --status fail +} +trap exit_error ERR + +do_configure_system() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + + ip address add "${server_ip}/24" dev "$vland_iface" + ip link set "$vland_iface" up + + sysctl -w net.ipv4.ip_local_port_range="1500 65500" + + # when using write_config, we need to have /www mounted + configure_ramdisk + + WRITE_CONFIG_CORE="worker_cpu_affinity auto;" + WRITE_CONFIG_EVENTS="" + WRITE_CONFIG_LISTEN="listen $server_ip:80 default_server reuseport so_keepalive=off;" + echo <<-EOF + WRITE_CONFIG_CORE=$WRITE_CONFIG_CORE + WRITE_CONFIG_EVENTS=$WRITE_CONFIG_EVENTS + WRITE_CONFIG_LISTEN=$WRITE_CONFIG_LISTEN + EOF +} + +do_stop_nginx() { + systemctl stop nginx +} + +do_start_nginx() { + systemctl start nginx +} + +do_write_nginx_config() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + + # when using write_config, we need to have /www mounted + write_config "$cores" /etc/nginx/nginx.conf +} + +do_pre_test_cb() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + + ethtool -L "$vland_iface" combined "$cores" +} + +do_post_test_cb() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + local pid + + systemctl status nginx + echo "-- AFFINITY $cores" + for pid in $(pgrep nginx); do + taskset -p "$pid" + done + echo "-- INTERRUPTS $cores" + grep "$vland_iface" /proc/interrupts + echo "--- MPSTAT $cores" + mpstat -P ALL | cat +} + +echo "-- linux-ip NGiNX initialized" >&2 diff --git a/automated/linux/nginx-server/nginx-server.sh b/automated/linux/nginx-server/nginx-server.sh index d4e39a9..21bbc66 100755 --- a/automated/linux/nginx-server/nginx-server.sh +++ b/automated/linux/nginx-server/nginx-server.sh @@ -1,15 +1,14 @@ #!/bin/bash set -o errexit +set -x THIS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" TEST_DEFS_DIR=$(readlink -f "${THIS_DIR}/../../..") GET_VLAND_IFACE=${TEST_DEFS_DIR}/automated/utils/vland/get_vland_interface.sh -GET_VLAND_PCI_DEV=${TEST_DEFS_DIR}/automated/utils/vland/get_vland_pci_dev.sh # vlnad name to use VLAND_NAME=${VLAND_NAME:-vlan_one} VLAND_IFACE=$($GET_VLAND_IFACE "$VLAND_NAME") -VLAND_PCI_DEV=$($GET_VLAND_PCI_DEV "$VLAND_NAME") # Do not run tests on more than MAX_CORES cores # 0 means use all cores @@ -23,82 +22,6 @@ SERVER_IP=${SERVER_IP:-192.168.1.4} # odp-dpdk: NGiNX with OFP+ODP+DPDK CONFIG_TYPE=${CONFIG_TYPE:-linux-ip} -function exit_error { - echo "-- SERVER ERROR" - journalctl -u nginx - lava-test-case server_up --status fail -} -trap exit_error ERR - -# Use this function to configure Linux IP stack -function config_linux_ip { - ip address add "${SERVER_IP}/24" dev "$VLAND_IFACE" - ip link set "$VLAND_IFACE" up - lava-test-case server_ifup --result pass - - sysctl -w net.ipv4.ip_local_port_range="1500 65500" -} - -# Use this function to configure a device for DPDK usage -function config_dpdk_dev { - local driver=${1:-igb_uio} - - if ! which dpdk-devbind &>/dev/null; then - echo "ERROR: dpdk not installed" - exit 1 - fi - - modprobe "$driver" - - dpdk-devbind -u "$VLAND_PCI_DEV" - dpdk-devbind -b "${driver}" "$VLAND_PCI_DEV" - dpdk-devbind -s - - apt-get install -y nginx - systemctl stop nginx - - # FIXME: for now NGiNX for OFP only supports one core worker - echo "-- NOTICE: setting MAX_CORES to 1" - MAX_CORES=1 -} - -# Callback to call before starting nginx when using OFP-DPDK -# First parameter of callback is the number of cores -function odp_dpdk_pre_cb { - # clean hugepages - rm -rf /dev/hugepages/* -} - -function odp_dpdk_post_cb { - local cores=$1 - - systemctl status nginx - echo "-- AFFINITY $cores" - for pid in $(pgrep nginx); do - taskset -p "$pid" - done -} - -function linux_ip_pre_cb { - local cores=$1 - - ethtool -L "$VLAND_IFACE" combined "$cores" -} - -function linux_ip_post_cb { - local cores=$1 - - systemctl status nginx - echo "-- AFFINITY $cores" - for pid in $(pgrep nginx); do - taskset -p "$pid" - done - echo "-- INTERRUPTS $cores" - grep "$VLAND_IFACE" /proc/interrupts - echo "--- MPSTAT $cores" - mpstat -P ALL | cat -} - function configure_ramdisk { local ROOT=/www mkdir "$ROOT" @@ -119,14 +42,16 @@ function configure_ramdisk { } function write_config { + local cores=${1} + local config_file=${2:-/etc/nginx/nginx.conf} + # Simple configuration file for NGiNX - cat > /etc/nginx/nginx.conf <<-EOF + cat > "$config_file" <<-EOF user www-data; - worker_processes $1; + worker_processes $cores; timer_resolution 1s; worker_rlimit_nofile 4096; error_log /dev/null crit; - pid /run/nginx.pid; ${WRITE_CONFIG_CORE} events { @@ -143,7 +68,7 @@ function write_config { open_file_cache max=10; server { # TODO: investigate backlog value - listen ${SERVER_IP}:80 default_server ${REUSEPORT} so_keepalive=off; + ${WRITE_CONFIG_LISTEN:-listen 80 default_server;} location / { root /www; } @@ -151,7 +76,7 @@ function write_config { } EOF echo "-- CONFIG FOR $1:" - cat /etc/nginx/nginx.conf + cat "$config_file" echo "-- END --" } @@ -170,32 +95,19 @@ function get_num_real_cores { echo "$num_cores" } -configure_ramdisk - -case ${CONFIG_TYPE} in - linux-ip) - echo "-- CONFIGURING Linux Kernel IP stack" - config_linux_ip - PRE_TEST_CB="linux_ip_pre_cb" - POST_TEST_CB="linux_ip_post_cb" - WRITE_CONFIG_CORE="worker_cpu_affinity auto;" - WRITE_CONFIG_EVENTS="" - REUSEPORT="reuseport" - ;; - odp-dpdk) - echo "-- CONFIGURING OFP IP Stack" - config_dpdk_dev igb_uio - PRE_TEST_CB="odp_dpdk_pre_cb" - POST_TEST_CB="odp_dpdk_post_cb" - WRITE_CONFIG_CORE="" - WRITE_CONFIG_EVENTS="use select;" - REUSEPORT="" - ;; - *) - echo "Invalid CONFIG_TYPE: $CONFIG_TYPE" - exit 1 - ;; -esac +test_functions="${THIS_DIR}/${CONFIG_TYPE}.sh" + +if [ ! -f "$test_functions" ]; then + echo "Invalid CONFIG_TYPE: $CONFIG_TYPE" + exit 1 +fi + +echo "-- Sourcing $test_functions" >&2 + +# shellcheck disable=SC1090 +. "$test_functions" + +do_configure_system "$(get_num_real_cores)" "$VLAND_IFACE" "$SERVER_IP" NUM_CORES=$(get_num_real_cores) echo ">> SEND num_cores cores=$NUM_CORES" @@ -207,22 +119,24 @@ lava-wait client_ready for num_cores in 1 $(seq 2 2 "$NUM_CORES"); do echo "-- BEGIN $num_cores" echo "-- Stopping NGiNX" - systemctl stop nginx + do_stop_nginx echo "-- Writing configuration file for $num_cores" - write_config "$num_cores" + do_write_nginx_config "$num_cores" "$VLAND_IFACE" "$SERVER_IP" echo "-- CALLING PRE-TEST CALLBACK $PRE_TEST_CB" - $PRE_TEST_CB "$num_cores" + do_pre_test_cb "$num_cores" "$VLAND_IFACE" "$SERVER_IP" echo "-- STARTING NGiNX for test $num_cores" - systemctl start nginx + do_start_nginx echo ">> SEND server_num_cores_${num_cores}_ready" lava-send "server_num_cores_${num_cores}_ready" echo "<< WAIT client_num_cores_${num_cores}_done" lava-wait "client_num_cores_${num_cores}_done" echo "-- CALLING POST-TEST CALLBACK $POST_TEST_CB" - $POST_TEST_CB "$num_cores" + do_post_test_cb "$num_cores" "$VLAND_IFACE" "$SERVER_IP" echo "-- END $num_cores" done +do_stop_nginx + echo "<< WAIT client_done" lava-wait client_done echo "A10" diff --git a/automated/linux/nginx-server/odp-dpdk.sh b/automated/linux/nginx-server/odp-dpdk.sh new file mode 100644 index 0000000..f078e05 --- /dev/null +++ b/automated/linux/nginx-server/odp-dpdk.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +exit_error() { + echo "-- SERVER ERROR" + journalctl -u nginx + lava-test-case server_up --status fail +} +trap exit_error ERR + +do_configure_system() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + local driver=${DPDK_DRIVER:-igb_uio} + local pci_dev= + + pci_dev=$(basename "$(readlink -f "/sys/class/net/${vland_iface}/device")") + + if ! which dpdk-devbind &>/dev/null; then + echo "ERROR: dpdk not installed" + exit 1 + fi + + modprobe "$driver" + + dpdk-devbind -u "$pci_dev" + dpdk-devbind -b "$driver" "$pci_dev" + dpdk-devbind -s + + apt-get install -y nginx + systemctl stop nginx + + # when using write_config, we need to have /www mounted + configure_ramdisk + + WRITE_CONFIG_CORE="" + WRITE_CONFIG_EVENTS="use select;" + WRITE_CONFIG_LISTEN="listen $server_ip:80 default_server so_keepalive=off;" + + # FIXME: for now NGiNX for OFP only supports one core worker + echo "-- NOTICE: setting MAX_CORES to 1" + MAX_CORES=1 + + echo <<-EOF + WRITE_CONFIG_CORE=$WRITE_CONFIG_CORE + WRITE_CONFIG_EVENTS=$WRITE_CONFIG_EVENTS + WRITE_CONFIG_LISTEN=$WRITE_CONFIG_LISTEN + MAX_CORES=$MAX_CORES + EOF + +} + +do_stop_nginx() { + systemctl stop nginx +} + +do_start_nginx() { + systemctl start nginx +} + +do_write_nginx_config() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + + # when using write_config, we need to have /www mounted + write_config "$cores" /etc/nginx/nginx.conf +} + +do_pre_test_cb() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + + rm -rf /dev/hugepages/* +} + +do_post_test_cb() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + local pid + + systemctl status nginx + echo "-- AFFINITY $cores" + for pid in $(pgrep nginx); do + taskset -p "$pid" + done +} + +echo "-- odp-dpdk NGiNX initialized" >&2 |