OFP event signal for handling socket operations such as accept and get

Signed-off-by: Kapil Hali <kapil.hali@linaro.org>

8 files changed, 340 insertions, 13 deletions

diff --git a/README b/README
new file mode 100644
index 00000000..949311a7
--- /dev/null
+++ b/README
@@ -0,0 +1,188 @@
+				README
+			      ==========
+		
+
+0. Prerequisites
+1. Preparing build
+2. Configuring NGINX 
+3. Running NGINX
+4. Limitations or Known Issues
+5. Changelog
+
+
+0. Prerequisites
+================
+
+Download ODP (Version 1.3 or 1.4) and latest OFP code. Have these compiled and installed in some out directory
+
+1. Preparing build
+==================
+
+set build variables with respective OFP and odp build path
+export OFP_PATH=/home/kapilhali/odp/ofp/ofp/out
+export OFP_ODP=/home/kapilhali/odp/odp/out
+
+./configure --sbin-path=/usr/local/nginx/nginx --conf-path=/usr/local/nginx/nginx.conf --pid-path=/usr/local/nginx/nginx.pid --with-http_ssl_module --with-debug --with-select_module
+
+
+make install
+
+Note: make clean will remove main Makefile and obj dir. So configure shall be run again. 
+It would be useful to save the above build steps in a script and run the script.
+
+2. Configuring NGINX 
+====================
+
+After build and install, NGINX configuration file will be at /usr/local/nginx/nginx.conf 
+Change as mentioned below.
+
+-----
+diff --git a/conf/nginx.conf b/conf/nginx.conf
+index 29bc085..8421146 100644
+--- a/conf/nginx.conf
++++ b/conf/nginx.conf
+@@ -5,12 +5,14 @@ worker_processes  1;
+ #error_log  logs/error.log;
+ #error_log  logs/error.log  notice;
+ #error_log  logs/error.log  info;
++#error_log  logs/error.log  debug;
+ 
+ #pid        logs/nginx.pid;
+-
++daemon off;
+ 
+ events {
+     worker_connections  1024;
++    use select;
+ }
+
+
+@@ -27,8 +30,8 @@ http {
+     sendfile        on;
+     #tcp_nopush     on;
+ 
+-    #keepalive_timeout  0;
+-    keepalive_timeout  65;
++    keepalive_timeout  0;
++    #keepalive_timeout  65;
+ 
+     #gzip  on;
+
+-----
+
+To get NGINX debug logs uncomment the debug line
+ error_log  logs/error.log  debug
+
+Debug logs will be in /usr/local/nginx/logs/error.log
+
+
+3. Running NGINX
+================
+
+In conf/ofp.conf file, configure the ip address for fp0 interface.
+
+Enable hugepages for ODP using the command below
+
+echo 1024 > /proc/sys/vm/nr_hugepages
+
+Below is start_nginx.sh script
+
+-----
+
+root@Linaro:/home/nginx# cat start_nginx.sh 
+#!/bin/bash
+
+app="nginx"
+
+if [ "$#" -ne 1 ]; then
+    echo "requires interface parameter. Use: ./start_$app.sh ethX"
+    exit -1
+fi
+
+intf=$1
+echo Starting $app on interface $intf
+
+/usr/local/nginx/nginx &
+
+#sleep 1
+#ifconfig fp0 $2
+#sleep 1
+
+iptables -A FORWARD -i $intf -j DROP
+iptables -A INPUT -i $intf -j DROP
+ip6tables -A FORWARD -i $intf -j DROP
+ip6tables -A INPUT -i $intf -j DROP
+ifconfig $intf -arp
+ip addr flush dev $intf
+
+-----
+
+
+Below is stop_nginx.sh script
+
+-----
+
+root@Linaro:/home/nginx# cat stop_nginx.sh 
+#!/bin/bash
+if [ "$#" -ne 1 ]; then
+    echo "requires an interface parameter. Use: ./start_webserver.sh ethX"
+    exit -1
+fi
+
+intf=$1
+if test "X$intf" = "X"; then intf=eth0; fi
+
+app="nginx"
+echo "Stopping $app"
+
+kill -9 `cat /usr/local/nginx/nginx.pid`
+
+ps -eL | grep nginx | awk '{print $1}' | xargs kill -9
+
+ifconfig $intf down
+iptables -D FORWARD -i $intf -j DROP
+iptables -D INPUT - i $intf -j DROP
+ip6tables -D FORWARD -i $intf -j DROP
+ip6tables -D INPUT -i $intf -j DROP
+ifconfig $intf arp
+
+ifconfig eth0 192.168.1.4/24 up
+mv /usr/local/nginx/nginx.pid /usr/local/nginx/nginx.pid.bk
+
+----
+
+To start webserver
+
+root@Linaro:/home/nginx# sh start_nginx.sh eth0
+
+To stop webserver 
+
+root@Linaro:/home/nginx# sh stop_nginx.sh eth0
+
+
+From some other machine try to ping the IP we have set for fp0 interface.
+If ping works, then open a web browser and try to access the same ip address.
+If web page loads successfully, webserver with OFP is working.
+
+
+
+4. Limitations or Known Issues
+==============================
+
+a) Current NGINX/OFP implementation assumes interface as eth0. 
+It will not work for other interfaces.
+
+b) Running with keepalive timeout other than zero causing issue
+while more connection requests are triggered. To avoid this
+Configure keepalive_timeout to zero in conf file.
+
+
+5. Changelog
+============
+
+Made changes to utilise the OFP event signaling support for handling
+socket operation like accept and recieve calls on a fd. 
+In ngx_select_module.c file, macro OFP_NOTIFY is enabled to use this 
+implementation. 
+If macro is set to zero, can use older select() implementation.
+
diff --git a/conf/nginx.conf b/conf/nginx.conf
index 29bc085f..43e8917d 100644
--- a/conf/nginx.conf
+++ b/conf/nginx.conf
@@ -5,12 +5,15 @@ worker_processes  1;
 #error_log  logs/error.log;
 #error_log  logs/error.log  notice;
 #error_log  logs/error.log  info;
+#error_log   logs/error.log  debug;
+daemon off;
 
 #pid        logs/nginx.pid;
 
 
 events {
     worker_connections  1024;
+    use select;
 }
 
 
@@ -27,8 +30,8 @@ http {
     sendfile        on;
     #tcp_nopush     on;
 
-    #keepalive_timeout  0;
-    keepalive_timeout  65;
+    keepalive_timeout  0;
+    #keepalive_timeout  65;
 
     #gzip  on;
 
diff --git a/src/event/modules/ngx_ofp_module.c b/src/event/modules/ngx_ofp_module.c
index 3dd9a14d..e7d30b09 100644
--- a/src/event/modules/ngx_ofp_module.c
+++ b/src/event/modules/ngx_ofp_module.c
@@ -140,7 +140,7 @@ int my_webserver(int if_count, char **if_name)
 	if (core_count > 1)
 		num_workers--;
 
-	num_workers = odp_cpumask_def_worker(&cpumask, num_workers);
+	num_workers = odp_cpumask_default_worker(&cpumask, num_workers);
 	odp_cpumask_to_str(&cpumask, cpumaskstr, sizeof(cpumaskstr));
 
 	printf("Num worker threads: %i\n", num_workers);
diff --git a/src/event/modules/ngx_select_module.c b/src/event/modules/ngx_select_module.c
index 186b88ed..eed6f9c8 100644
--- a/src/event/modules/ngx_select_module.c
+++ b/src/event/modules/ngx_select_module.c
@@ -70,22 +70,80 @@ ngx_module_t  ngx_select_module = {
     NGX_MODULE_V1_PADDING
 };
 
-ofp_fd_set ofp_readfd;
+#define OFP_NOTIFY  1
 #define ODP_FD_BITS 30
 #undef FD_SET
+#define CHK_FD_BIT(fd)          (fd & (1 << ODP_FD_BITS))
+#define CLR_FD_BIT(fd)          (fd & ~(1 << ODP_FD_BITS))
 #define FD_SET(fd, fdsetp)      do { \
-				if (fd & (1 << ODP_FD_BITS)) {  \
-				   OFP_FD_SET ((fd & ~(1 << ODP_FD_BITS)) , (ofp_fd_set *)fdsetp) ; \
+				if (CHK_FD_BIT(fd)) {  \
+				   OFP_FD_SET (CLR_FD_BIT(fd) , (ofp_fd_set *)fdsetp) ; \
 				} else { \
 				   OFP_FD_SET (fd , (ofp_fd_set *)fdsetp) ; \
 				} \
 				} while (0)
 
 #undef FD_ISSET
-#define FD_ISSET(fd, fdsetp)    OFP_FD_ISSET((fd & ~(1 << ODP_FD_BITS)), (ofp_fd_set *)fdsetp)
+#define FD_ISSET(fd, fdsetp)    OFP_FD_ISSET(CLR_FD_BIT(fd), (ofp_fd_set *)fdsetp)
 
 #undef FD_CLR
-#define FD_CLR(fd, fdsetp)      OFP_FD_CLR ((fd & ~(1 << ODP_FD_BITS)), (ofp_fd_set *)fdsetp)
+#define FD_CLR(fd, fdsetp)      OFP_FD_CLR (CLR_FD_BIT(fd), (ofp_fd_set *)fdsetp)
+#if OFP_NOTIFY
+static void sigev_notify(union ofp_sigval sv)
+{
+    struct ofp_sock_sigval *ss = sv.sival_ptr;
+    if (!ss)
+       return ;
+    int s = ss->sockfd;
+    int event = ss->event;
+    odp_packet_t pkt = ss->pkt;
+    ngx_uint_t         i = 0;
+    ngx_queue_t       *queue = NULL;
+    ngx_event_t       *ev;
+    ngx_connection_t  *c;
+
+    if (event == OFP_EVENT_ACCEPT)  {
+        for (i = 0; i < nevents; i++) {
+            ev = event_index[i];
+            c  = ev->data;
+
+            if (ev->accept) {
+                ev->ready = 1;
+                ev->handler(ev);
+                OFP_DBG("%s: posted on ACCEPT EVENT", __func__);
+                break;
+            }
+        }
+    }
+
+    if (event != OFP_EVENT_RECV) {
+        goto end;
+    }
+
+    int r = odp_packet_len(pkt);
+
+    if (r > 0) {
+        for (i = 0; i < nevents; i++) {
+            ev = event_index[i];
+            c  = ev->data;
+            if (s == CLR_FD_BIT(c->fd))  {
+                queue = &ngx_posted_events;
+                ngx_post_event(ev, queue);
+                OFP_DBG("%s: posted on RECV EVENT", __func__);
+                ev->ready = 1;
+                break;
+            }
+        }
+    } else if (r == 0) {
+        odp_packet_free(pkt);
+        ss->pkt = ODP_PACKET_INVALID;
+    }
+
+end:
+    return;
+}
+#endif
+
 
 static ngx_int_t
 ngx_select_init(ngx_cycle_t *cycle, ngx_msec_t timer)
@@ -124,6 +182,27 @@ ngx_select_init(ngx_cycle_t *cycle, ngx_msec_t timer)
 
     max_fd = -1;
 
+#if OFP_NOTIFY
+    ngx_uint_t        i;
+    ngx_listening_t  *ls;
+    struct ofp_sigevent sigev;
+    struct ofp_sock_sigval socksig;
+    ls = cycle->listening.elts;
+    for (i = 0; i < cycle->listening.nelts; i++) {
+        if ((ls[i].listen) && (CHK_FD_BIT(ls[i].fd))) {
+            socksig.sockfd = CLR_FD_BIT(ls[i].fd);
+            socksig.event = 0;
+            socksig.pkt = ODP_PACKET_INVALID;
+            sigev.ofp_sigev_notify = 1;
+            sigev.ofp_sigev_notify_function = sigev_notify;
+            sigev.ofp_sigev_value.sival_ptr = &socksig;
+            ofp_socket_sigevent(&sigev);
+            ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0,
+                            "Set ofp notity on fd : %d",
+                                        socksig.sockfd);
+        }
+    }
+#endif
     return NGX_OK;
 }
 
@@ -167,13 +246,15 @@ ngx_select_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
                       "select %s event fd:%d  event:%d",
                       ev->write ? "write" : "read", c->fd, event);
 
+#if !(OFP_NOTIFY)
     if (event == NGX_READ_EVENT) {
-	if (c->fd & (1<<ODP_FD_BITS)) {
-           FD_SET(c->fd, (ofp_fd_set *)&master_read_fd_set);
+	if (CHK_FD_BIT(c->fd)) {
+            FD_SET(c->fd, (ofp_fd_set *)&master_read_fd_set);
 	}
     } else if (event == NGX_WRITE_EVENT) {
         FD_SET(c->fd, &master_write_fd_set);
     }
+#endif
 
     if (max_fd != -1 && max_fd < c->fd) {
         max_fd = c->fd;
@@ -209,12 +290,14 @@ ngx_select_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags)
     ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0,
                    "select del event fd:%d ev:%i", c->fd, event);
 
+#if !OFP_NOTIFY
     if (event == NGX_READ_EVENT) {
         FD_CLR(c->fd, &master_read_fd_set);
 
     } else if (event == NGX_WRITE_EVENT) {
         FD_CLR(c->fd, &master_write_fd_set);
     }
+#endif
 
     if (max_fd == c->fd) {
         max_fd = -1;
@@ -236,6 +319,9 @@ static ngx_int_t
 ngx_select_process_events(ngx_cycle_t *cycle, ngx_msec_t timer,
     ngx_uint_t flags)
 {
+#if OFP_NOTIFY
+    return NGX_OK;
+#endif
     int                ready, nready;
     ngx_err_t          err;
     ngx_uint_t         i, found;
diff --git a/src/event/ngx_event.c b/src/event/ngx_event.c
index 15da2134..c88335e3 100644
--- a/src/event/ngx_event.c
+++ b/src/event/ngx_event.c
@@ -243,8 +243,8 @@ ngx_process_events_and_timers(ngx_cycle_t *cycle)
 
     delta = ngx_current_msec - delta;
 
-    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
-                   "timer delta: %M", delta);
+//    ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
+//                   "timer delta: %M", delta);
 
     ngx_event_process_posted(cycle, &ngx_posted_accept_events);
 
diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c
index 5da09116..990e5748 100644
--- a/src/os/unix/ngx_process_cycle.c
+++ b/src/os/unix/ngx_process_cycle.c
@@ -748,7 +748,7 @@ ngx_worker_process_cycle(ngx_cycle_t *cycle, void *data)
             }
         }
 
-        ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "worker cycle");
+        //ngx_log_debug0(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "worker cycle");
 
         ngx_process_events_and_timers(cycle);
 
diff --git a/start_nginx.sh b/start_nginx.sh
new file mode 100644
index 00000000..e6d98e0f
--- /dev/null
+++ b/start_nginx.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+app="nginx" 
+
+if [ "$#" -ne 1 ]; then
+    echo "requires interface parameter. Use: ./start_$app.sh ethX"
+    exit -1
+fi
+
+intf=$1
+echo Starting $app on interface $intf
+
+/usr/local/nginx/nginx &
+
+#sleep 1
+#ifconfig fp0 $2
+#sleep 1
+
+iptables -A FORWARD -i $intf -j DROP
+iptables -A INPUT -i $intf -j DROP
+ip6tables -A FORWARD -i $intf -j DROP
+ip6tables -A INPUT -i $intf -j DROP
+ifconfig $intf -arp
+ip addr flush dev $intf
+
diff --git a/stop_nginx.sh b/stop_nginx.sh
new file mode 100644
index 00000000..a8ed3d9b
--- /dev/null
+++ b/stop_nginx.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+if [ "$#" -ne 1 ]; then
+    echo "requires an interface parameter. Use: ./start_webserver.sh ethX"
+    exit -1
+fi
+
+intf=$1
+if test "X$intf" = "X"; then intf=eth0; fi
+
+app="nginx"
+echo "Stopping $app"
+
+kill -9 `cat /usr/local/nginx/nginx.pid`
+
+ps -eL | grep nginx | awk '{print $1}' | xargs kill -9
+
+ifconfig $intf down
+iptables -D FORWARD -i $intf -j DROP
+iptables -D INPUT -i $intf -j DROP
+ip6tables -D FORWARD -i $intf -j DROP
+ip6tables -D INPUT -i $intf -j DROP
+ifconfig $intf arp
+
+ifconfig eth0 192.168.1.4/24 up
+mv /usr/local/nginx/nginx.pid /usr/local/nginx/nginx.pid.bk