From 6cdfbb5fcf3d939d11661bc5bf4aa3c496beb898 Mon Sep 17 00:00:00 2001 From: Josep Puigdemont Date: Fri, 17 Mar 2017 10:01:49 +0100 Subject: nginx-server: use configuration files and callbaks NGiNX can be installed in different places, and can be implemented using different IP stacks (Linux IP stack, OFP, ANS...). The nginx-server.sh script was meant to be used independently of he NGiNX version in use. Due to this it contains several initialization and configuration functions for each of the currently supported implementations. This could grow a bit too large for one single file, so we've opted for creating files with a set of required functions for each implementation. The file is sourced and it should define some functions used for the tests. The name of the file should be that of the CONFIG_TYPE defined for the test definition, plus the .sh suffix. The functions required to be implemented in the files are: - do_configure_system(): used to initialize the tests. Here any configuration required for the test, like NIC configuration, kernel module loading, etc, should be done. It is called once before any test starts. - do_start_nginx(): this function is used to start NGiNX. - do_stop_nginx(): this function is used to stop NGiNX. It may be called even if NGiNX is not running. - do_write_nginx_config(): this function is called before starting NGiNX, and it is meant to write a proper nginx.conf file for the current number of cores. - do_pre_test_cb(): this function is called before starting NGiNX, it can be used for anything deemed appropriate. - do_post_test_cb(): this function is called right after the test has finished, and before NGiNX is stopped. It can be used for anything deemed appropriate. All functions are called with the following parameters in this order: - cores: number of cores of the current test. For configure_sytem, this is the total number of physical cores available. - interface: this is the interface that NGiNX is expected to use. - IP: the IP that the web server is expected to use. The return value of the functions is not checked. Change-Id: I0321697509f9ac628e705c69285a7478dd993ace Signed-off-by: Josep Puigdemont --- automated/linux/nginx-server/linux-ip.sh | 75 ++++++++++++++ automated/linux/nginx-server/nginx-server.sh | 142 ++++++--------------------- automated/linux/nginx-server/odp-dpdk.sh | 91 +++++++++++++++++ 3 files changed, 194 insertions(+), 114 deletions(-) create mode 100644 automated/linux/nginx-server/linux-ip.sh create mode 100644 automated/linux/nginx-server/odp-dpdk.sh (limited to 'automated/linux') diff --git a/automated/linux/nginx-server/linux-ip.sh b/automated/linux/nginx-server/linux-ip.sh new file mode 100644 index 0000000..a8937ce --- /dev/null +++ b/automated/linux/nginx-server/linux-ip.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +exit_error() { + echo "-- SERVER ERROR" + journalctl -u nginx + lava-test-case server_up --status fail +} +trap exit_error ERR + +do_configure_system() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + + ip address add "${server_ip}/24" dev "$vland_iface" + ip link set "$vland_iface" up + + sysctl -w net.ipv4.ip_local_port_range="1500 65500" + + # when using write_config, we need to have /www mounted + configure_ramdisk + + WRITE_CONFIG_CORE="worker_cpu_affinity auto;" + WRITE_CONFIG_EVENTS="" + WRITE_CONFIG_LISTEN="listen $server_ip:80 default_server reuseport so_keepalive=off;" + echo <<-EOF + WRITE_CONFIG_CORE=$WRITE_CONFIG_CORE + WRITE_CONFIG_EVENTS=$WRITE_CONFIG_EVENTS + WRITE_CONFIG_LISTEN=$WRITE_CONFIG_LISTEN + EOF +} + +do_stop_nginx() { + systemctl stop nginx +} + +do_start_nginx() { + systemctl start nginx +} + +do_write_nginx_config() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + + # when using write_config, we need to have /www mounted + write_config "$cores" /etc/nginx/nginx.conf +} + +do_pre_test_cb() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + + ethtool -L "$vland_iface" combined "$cores" +} + +do_post_test_cb() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + local pid + + systemctl status nginx + echo "-- AFFINITY $cores" + for pid in $(pgrep nginx); do + taskset -p "$pid" + done + echo "-- INTERRUPTS $cores" + grep "$vland_iface" /proc/interrupts + echo "--- MPSTAT $cores" + mpstat -P ALL | cat +} + +echo "-- linux-ip NGiNX initialized" >&2 diff --git a/automated/linux/nginx-server/nginx-server.sh b/automated/linux/nginx-server/nginx-server.sh index d4e39a9..21bbc66 100755 --- a/automated/linux/nginx-server/nginx-server.sh +++ b/automated/linux/nginx-server/nginx-server.sh @@ -1,15 +1,14 @@ #!/bin/bash set -o errexit +set -x THIS_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" TEST_DEFS_DIR=$(readlink -f "${THIS_DIR}/../../..") GET_VLAND_IFACE=${TEST_DEFS_DIR}/automated/utils/vland/get_vland_interface.sh -GET_VLAND_PCI_DEV=${TEST_DEFS_DIR}/automated/utils/vland/get_vland_pci_dev.sh # vlnad name to use VLAND_NAME=${VLAND_NAME:-vlan_one} VLAND_IFACE=$($GET_VLAND_IFACE "$VLAND_NAME") -VLAND_PCI_DEV=$($GET_VLAND_PCI_DEV "$VLAND_NAME") # Do not run tests on more than MAX_CORES cores # 0 means use all cores @@ -23,82 +22,6 @@ SERVER_IP=${SERVER_IP:-192.168.1.4} # odp-dpdk: NGiNX with OFP+ODP+DPDK CONFIG_TYPE=${CONFIG_TYPE:-linux-ip} -function exit_error { - echo "-- SERVER ERROR" - journalctl -u nginx - lava-test-case server_up --status fail -} -trap exit_error ERR - -# Use this function to configure Linux IP stack -function config_linux_ip { - ip address add "${SERVER_IP}/24" dev "$VLAND_IFACE" - ip link set "$VLAND_IFACE" up - lava-test-case server_ifup --result pass - - sysctl -w net.ipv4.ip_local_port_range="1500 65500" -} - -# Use this function to configure a device for DPDK usage -function config_dpdk_dev { - local driver=${1:-igb_uio} - - if ! which dpdk-devbind &>/dev/null; then - echo "ERROR: dpdk not installed" - exit 1 - fi - - modprobe "$driver" - - dpdk-devbind -u "$VLAND_PCI_DEV" - dpdk-devbind -b "${driver}" "$VLAND_PCI_DEV" - dpdk-devbind -s - - apt-get install -y nginx - systemctl stop nginx - - # FIXME: for now NGiNX for OFP only supports one core worker - echo "-- NOTICE: setting MAX_CORES to 1" - MAX_CORES=1 -} - -# Callback to call before starting nginx when using OFP-DPDK -# First parameter of callback is the number of cores -function odp_dpdk_pre_cb { - # clean hugepages - rm -rf /dev/hugepages/* -} - -function odp_dpdk_post_cb { - local cores=$1 - - systemctl status nginx - echo "-- AFFINITY $cores" - for pid in $(pgrep nginx); do - taskset -p "$pid" - done -} - -function linux_ip_pre_cb { - local cores=$1 - - ethtool -L "$VLAND_IFACE" combined "$cores" -} - -function linux_ip_post_cb { - local cores=$1 - - systemctl status nginx - echo "-- AFFINITY $cores" - for pid in $(pgrep nginx); do - taskset -p "$pid" - done - echo "-- INTERRUPTS $cores" - grep "$VLAND_IFACE" /proc/interrupts - echo "--- MPSTAT $cores" - mpstat -P ALL | cat -} - function configure_ramdisk { local ROOT=/www mkdir "$ROOT" @@ -119,14 +42,16 @@ function configure_ramdisk { } function write_config { + local cores=${1} + local config_file=${2:-/etc/nginx/nginx.conf} + # Simple configuration file for NGiNX - cat > /etc/nginx/nginx.conf <<-EOF + cat > "$config_file" <<-EOF user www-data; - worker_processes $1; + worker_processes $cores; timer_resolution 1s; worker_rlimit_nofile 4096; error_log /dev/null crit; - pid /run/nginx.pid; ${WRITE_CONFIG_CORE} events { @@ -143,7 +68,7 @@ function write_config { open_file_cache max=10; server { # TODO: investigate backlog value - listen ${SERVER_IP}:80 default_server ${REUSEPORT} so_keepalive=off; + ${WRITE_CONFIG_LISTEN:-listen 80 default_server;} location / { root /www; } @@ -151,7 +76,7 @@ function write_config { } EOF echo "-- CONFIG FOR $1:" - cat /etc/nginx/nginx.conf + cat "$config_file" echo "-- END --" } @@ -170,32 +95,19 @@ function get_num_real_cores { echo "$num_cores" } -configure_ramdisk - -case ${CONFIG_TYPE} in - linux-ip) - echo "-- CONFIGURING Linux Kernel IP stack" - config_linux_ip - PRE_TEST_CB="linux_ip_pre_cb" - POST_TEST_CB="linux_ip_post_cb" - WRITE_CONFIG_CORE="worker_cpu_affinity auto;" - WRITE_CONFIG_EVENTS="" - REUSEPORT="reuseport" - ;; - odp-dpdk) - echo "-- CONFIGURING OFP IP Stack" - config_dpdk_dev igb_uio - PRE_TEST_CB="odp_dpdk_pre_cb" - POST_TEST_CB="odp_dpdk_post_cb" - WRITE_CONFIG_CORE="" - WRITE_CONFIG_EVENTS="use select;" - REUSEPORT="" - ;; - *) - echo "Invalid CONFIG_TYPE: $CONFIG_TYPE" - exit 1 - ;; -esac +test_functions="${THIS_DIR}/${CONFIG_TYPE}.sh" + +if [ ! -f "$test_functions" ]; then + echo "Invalid CONFIG_TYPE: $CONFIG_TYPE" + exit 1 +fi + +echo "-- Sourcing $test_functions" >&2 + +# shellcheck disable=SC1090 +. "$test_functions" + +do_configure_system "$(get_num_real_cores)" "$VLAND_IFACE" "$SERVER_IP" NUM_CORES=$(get_num_real_cores) echo ">> SEND num_cores cores=$NUM_CORES" @@ -207,22 +119,24 @@ lava-wait client_ready for num_cores in 1 $(seq 2 2 "$NUM_CORES"); do echo "-- BEGIN $num_cores" echo "-- Stopping NGiNX" - systemctl stop nginx + do_stop_nginx echo "-- Writing configuration file for $num_cores" - write_config "$num_cores" + do_write_nginx_config "$num_cores" "$VLAND_IFACE" "$SERVER_IP" echo "-- CALLING PRE-TEST CALLBACK $PRE_TEST_CB" - $PRE_TEST_CB "$num_cores" + do_pre_test_cb "$num_cores" "$VLAND_IFACE" "$SERVER_IP" echo "-- STARTING NGiNX for test $num_cores" - systemctl start nginx + do_start_nginx echo ">> SEND server_num_cores_${num_cores}_ready" lava-send "server_num_cores_${num_cores}_ready" echo "<< WAIT client_num_cores_${num_cores}_done" lava-wait "client_num_cores_${num_cores}_done" echo "-- CALLING POST-TEST CALLBACK $POST_TEST_CB" - $POST_TEST_CB "$num_cores" + do_post_test_cb "$num_cores" "$VLAND_IFACE" "$SERVER_IP" echo "-- END $num_cores" done +do_stop_nginx + echo "<< WAIT client_done" lava-wait client_done echo "A10" diff --git a/automated/linux/nginx-server/odp-dpdk.sh b/automated/linux/nginx-server/odp-dpdk.sh new file mode 100644 index 0000000..f078e05 --- /dev/null +++ b/automated/linux/nginx-server/odp-dpdk.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +exit_error() { + echo "-- SERVER ERROR" + journalctl -u nginx + lava-test-case server_up --status fail +} +trap exit_error ERR + +do_configure_system() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + local driver=${DPDK_DRIVER:-igb_uio} + local pci_dev= + + pci_dev=$(basename "$(readlink -f "/sys/class/net/${vland_iface}/device")") + + if ! which dpdk-devbind &>/dev/null; then + echo "ERROR: dpdk not installed" + exit 1 + fi + + modprobe "$driver" + + dpdk-devbind -u "$pci_dev" + dpdk-devbind -b "$driver" "$pci_dev" + dpdk-devbind -s + + apt-get install -y nginx + systemctl stop nginx + + # when using write_config, we need to have /www mounted + configure_ramdisk + + WRITE_CONFIG_CORE="" + WRITE_CONFIG_EVENTS="use select;" + WRITE_CONFIG_LISTEN="listen $server_ip:80 default_server so_keepalive=off;" + + # FIXME: for now NGiNX for OFP only supports one core worker + echo "-- NOTICE: setting MAX_CORES to 1" + MAX_CORES=1 + + echo <<-EOF + WRITE_CONFIG_CORE=$WRITE_CONFIG_CORE + WRITE_CONFIG_EVENTS=$WRITE_CONFIG_EVENTS + WRITE_CONFIG_LISTEN=$WRITE_CONFIG_LISTEN + MAX_CORES=$MAX_CORES + EOF + +} + +do_stop_nginx() { + systemctl stop nginx +} + +do_start_nginx() { + systemctl start nginx +} + +do_write_nginx_config() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + + # when using write_config, we need to have /www mounted + write_config "$cores" /etc/nginx/nginx.conf +} + +do_pre_test_cb() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + + rm -rf /dev/hugepages/* +} + +do_post_test_cb() { + local cores=$1 + local vland_iface=$2 + local server_ip=$3 + local pid + + systemctl status nginx + echo "-- AFFINITY $cores" + for pid in $(pgrep nginx); do + taskset -p "$pid" + done +} + +echo "-- odp-dpdk NGiNX initialized" >&2 -- cgit v1.2.3