1. Test environment

1.1 Hardware/OS

2 Linux boxes in a gigabit ethernet LAN, 1 server and 1 test client
Linux Centos 5.2 64bit
Intel(R) Xeon(R) CPU E5410  @ 2.33GHz (L2 cache: 6M), Quad-Core * 2
8G memory
SCSI disk (standalone disk, no other access)

1.2 Software version

nginx, nginx-0.7.63.tar.gz
Erlang, otp_src_R13B02-1.tar.gz
Java, jdk-6u17-linux-x64.bin, mina-2.0.0-RC1.tar.gz, netty-3.2.0.ALPHA1-dist.tar.bz2
Go, hg clone -r release https://go.googlecode.com/hg/ $GOROOT (Nov 12, 2009)

1.3 Source code and configuration

Linux, run sysctl -p

net.ipv4.ip_forward = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.conf.default.accept_source_route = 0
kernel.sysrq = 0
kernel.core_uses_pid = 1
net.ipv4.tcp_syncookies = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.shmmax = 68719476736
kernel.shmall = 4294967296
kernel.panic = 1
net.ipv4.tcp_rmem = 8192	873800	8738000
net.ipv4.tcp_wmem = 4096	655360	6553600
net.ipv4.ip_local_port_range = 1024	65000
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216

# ulimit -n
150000

C: ngnix hello world module, copy the code ngx_http_hello_module.c from http://timyang.net/web/nginx-module/

in nginx.conf, set “worker_processes  1; worker_connections 10240″ for 1 cpu test, set “worker_processes  4; worker_connections 2048″ for multi-core cpu test. Turn off all access or debug log in nginx.conf, as follows

worker_processes  1;
worker_rlimit_nofile 10240;
events {
    worker_connections  10240;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
    keepalive_timeout  0;
    server {
        listen       8080;
        server_name  localhost;
        location / {
            root   html;
            index  index.html index.htm;
        }
          location /hello {
            ngx_hello_module;
            hello 1234;
        }

        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }
    }
}

$ taskset -c 1 ./nginx or $ taskset -c 1-7 ./nginx

Erlang hello world server
The source code is available at yufeng’s blog, see http://blog.yufeng.info/archives/105
Just copy the code after “cat ehttpd.erl”, and compile it.

$ erlc ehttpd.erl
$ taskset -c 1 erl +K true +h 99999 +P 99999 -smp enable +S 2:1 -s ehttpd
$ taskset -c 1-7 erl +K true -s ehttpd
We use taskset to limit erlang vm to use only 1 CPU/core or use all CPU cores. The 2nd line is run in single CPU mode, and the 3rd line is run in multi-core CPU mode.

Java source code, save the 2 class as HttpServer.java and HttpProtocolHandler.java, and do necessary import.

public class HttpServer {
    public static void main(String[] args) throws Exception {
        SocketAcceptor acceptor = new NioSocketAcceptor(4);
        acceptor.setReuseAddress( true );

		int port = 8080;
		String hostname = null;
		if (args.length > 1) {
			hostname = args[0];
			port = Integer.parseInt(args[1]);
		}

        // Bind
        acceptor.setHandler(new HttpProtocolHandler());
        if (hostname != null)
        	acceptor.bind(new InetSocketAddress(hostname, port));
        else
        	acceptor.bind(new InetSocketAddress(port));

        System.out.println("Listening on port " + port);
        Thread.currentThread().join();
    }
}

public class HttpProtocolHandler extends IoHandlerAdapter {
    public void sessionCreated(IoSession session) {
        session.getConfig().setIdleTime(IdleStatus.BOTH_IDLE, 10);
        session.setAttribute(SslFilter.USE_NOTIFICATION);
    }

    public void sessionClosed(IoSession session) throws Exception {}
    public void sessionOpened(IoSession session) throws Exception {}
    public void sessionIdle(IoSession session, IdleStatus status) {}
    public void exceptionCaught(IoSession session, Throwable cause) {
        session.close(true);
    }

    static IoBuffer RESULT = null;
	public static String HTTP_200 = "HTTP/1.1 200 OK\r\nContent-Length: 13\r\n\r\n" +
			"hello world\r\n";
	static {
    	RESULT = IoBuffer.allocate(32).setAutoExpand(true);
    	RESULT.put(HTTP_200.getBytes());
    	RESULT.flip();
    }
    public void messageReceived(IoSession session, Object message)
            throws Exception {
        if (message instanceof IoBuffer) {
        	IoBuffer buf = (IoBuffer) message;
        	int c = buf.get();
        	if (c == 'G' || c == 'g') {
        		session.write(RESULT.duplicate());
        	}
        	session.close(false);
        }
    }
}

Nov 24 update Because the above Mina code doesn’t parse HTTP request and handle the necessary HTTP protocol, replaced with org.jboss.netty.example.http.snoop.HttpServer from Netty example, but removed all the string builder code from HttpRequestHandler.messageReceived() and just return a “hello world” result in HttpRequestHandler.writeResponse(). Please read the source code and the Netty documentation for more information.

$ taskset -c 1-7 \
java -server -Xmx1024m -Xms1024m -XX:+UseConcMarkSweepGC -classpath . test.HttpServer 192.168.10.1 8080

We use taskset to limit java only use cpu1-7, and not use cpu0, because we want cpu0 dedicate for system call(Linux use CPU0 for network interruptions).

Go language, source code

package main
import (
   "http";
    "io";
)
func HelloServer(c *http.Conn, req *http.Request) {
    io.WriteString(c, "hello, world!\n");
}
func main() {
     runtime.GOMAXPROCS(8); // 8 cores
     http.Handle("/", http.HandlerFunc(HelloServer));
     err := http.ListenAndServe(":8080", nil);
    if err != nil {
        panic("ListenAndServe: ", err.String())
    }
}

$ 6g httpd2.go
$ 6l httpd2.6
$ taskset -c 1-7 ./6.out

1.4 Performance test client

ApacheBench client, for 30, 100, 1,000, 5,000 concurrent threads
ab -c 30 -n 1000000 http://192.168.10.1:8080/
ab -c 100 -n 1000000 http://192.168.10.1:8080/
1000 thread, 334 from 3 different machine
ab -c 334 -n 334000 http://192.168.10.1:8080/
5000 thread, 1667 from 3 different machine
ab -c 1667 -n 334000 http://192.168.10.1:8080/

2. Test results

2.1 request per second

2.2 latency, 99% requests within(ms)

3. Notes

* On large concurrent connections, C, Erlang, Java no big difference on their performance, results are very close.
* Java runs better on small connections, but the code in this test doesn’t parse the HTTP request header (the MINA code).
* Although Mr. Yu Feng (the Erlang guru in China) mentioned that Erlang performance better on single CPU(prevent context switch), but the result tells that Erlang has big latency(> 1S) under 1,000 or 5,000 connections.
* Go language is very close to Erlang, but still not good under heavy load (5,000 threads)
After redo 1,000 and 5,000 tests on Nov 18
* Nginx module is the winner on 5,000 concurrent requests.
* Although there is improvement space for Go, Go has the same performance from 30-5,000 threads.
* Erlang process is impressive on large concurrent request, still as good as nginx (5,000 threads).

4. Update Log

Nov 12, change nginx.conf work_connections from 1024 to 10240
Nov 13, add runtime.GOMAXPROCS(8); to go’s code, add sysctl -p env
Nov 18, realized that ApacheBench itself is a bottleneck under 1,000 or 5,000 threads, so use 3 clients from 3 different machines to redo all tests of 1,000 and 5,000 concurrent tests.
Nov 24, use Netty with full HTTP implementation to replace Mina 2 for the Java web server. Still very fast and low latency after added HTTP handle code.

• 如何写nginx module
• MemcacheDB, Tokyo Tyrant, Redis performance test
• Thrift, Protocol Buffers installation and Java code howto
• Thrift and Protocol Buffers performance in Java
• Thrift and Protocol Buffers performance in Java Round 2

1. nginx 配置添加

./configure --add-module=/path/to/module1/source

2. 添加 /path/to/module1/source/config 文件，内容

ngx_addon_name=ngx_http_hello_module
HTTP_MODULES="$HTTP_MODULES ngx_http_hello_module"
NGX_ADDON_SRCS="$NGX_ADDON_SRCS $ngx_addon_dir/ngx_http_hello_module.c"
CORE_LIBS="$CORE_LIBS -lfoo"

最后一行如果没有使用其他library, 可以去掉

3. 源代码 /path/to/module1/source/ngx_http_hello_module.c, 主要的业务逻辑在make_http_get_body 中完善。典型的hello world源代码如下

#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>

#define OUT_BUFSIZE 256

static char *ngx_http_hello_set(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);
static char *ngx_http_foo_set(ngx_conf_t *cf, ngx_command_t *cmd, void *conf);

static ngx_int_t ngx_http_hello_process_init(ngx_cycle_t *cycle);
static void ngx_http_hello_process_exit(ngx_cycle_t *cycle);

static ngx_int_t make_http_header(ngx_http_request_t *r);
static ngx_int_t make_http_get_body(ngx_http_request_t *r, char *out_buf);

static char g_foo_settings[64] = {0};

/* Commands */
static ngx_command_t  ngx_http_hello_commands[] = {
    { ngx_string("ngx_hello_module"),
      NGX_HTTP_LOC_CONF|NGX_CONF_NOARGS,
      ngx_http_hello_set,
      NGX_HTTP_LOC_CONF_OFFSET,
      0,
      NULL },

    { ngx_string("hello"),
      NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
      ngx_http_foo_set,
      NGX_HTTP_LOC_CONF_OFFSET,
      0,
      NULL },  

      ngx_null_command
};

static ngx_http_module_t  ngx_http_hello_module_ctx = {
    NULL,                                  /* preconfiguration */
    NULL,                                     /* postconfiguration */

    NULL,                                  /* create main configuration */
    NULL,                                  /* init main configuration */

    NULL,                                  /* create server configuration */
    NULL,                                  /* merge server configuration */

    NULL,                                  /* create location configuration */
    NULL                                   /* merge location configuration */
};

/* hook */
ngx_module_t  ngx_http_hello_module = {
    NGX_MODULE_V1,
    &ngx_http_hello_module_ctx,              /* module context */
    ngx_http_hello_commands,                 /* module directives */
    NGX_HTTP_MODULE,                       /* module type */
    NULL,                                  /* init master */
    NULL,                                  /* init module */
    ngx_http_hello_process_init,             /* init process */
    NULL,                                  /* init thread */
    NULL,                                  /* exit thread */
    ngx_http_hello_process_exit,             /* exit process */
    NULL,                                  /* exit master */
    NGX_MODULE_V1_PADDING
};

/* setting header for no-cache */
static ngx_int_t make_http_header(ngx_http_request_t *r){
    ngx_uint_t        i;
    ngx_table_elt_t  *cc, **ccp;

    r->headers_out.content_type.len = sizeof("text/html") - 1;
    r->headers_out.content_type.data = (u_char *) "text/html";
    ccp = r->headers_out.cache_control.elts;
    if (ccp == NULL) {

        if (ngx_array_init(&r->headers_out.cache_control, r->pool,
                           1, sizeof(ngx_table_elt_t *))
            != NGX_OK)
        {
            return NGX_ERROR;
        }

        ccp = ngx_array_push(&r->headers_out.cache_control);
        if (ccp == NULL) {
            return NGX_ERROR;
        }

        cc = ngx_list_push(&r->headers_out.headers);
        if (cc == NULL) {
            return NGX_ERROR;
        }

        cc->hash = 1;
        cc->key.len = sizeof("Cache-Control") - 1;
        cc->key.data = (u_char *) "Cache-Control";

        *ccp = cc;

    } else {
        for (i = 1; i < r->headers_out.cache_control.nelts; i++) {
            ccp[i]->hash = 0;
        }

        cc = ccp[0];
    }

    cc->value.len = sizeof("no-cache") - 1;
    cc->value.data = (u_char *) "no-cache";

    return NGX_OK;
}

static ngx_int_t make_http_get_body(ngx_http_request_t *r, char *out_buf){
    char *qs_start = (char *)r->args_start;
    char *qs_end = (char *)r->uri_end;
    char uri[128] = {0};
    char *id;

    if (qs_start == NULL || qs_end == NULL){
        return NGX_HTTP_BAD_REQUEST;
    }
    if ((memcmp(qs_start, "id=", 3) == 0)){
        id = qs_start + 3;
        *qs_end = '\0';
    }else{
        return NGX_HTTP_BAD_REQUEST;
    }
    snprintf(uri, r->uri.len + 1, "%s", r->uri.data);
    sprintf(out_buf, "Author: http://timyang.net/ nconfig=%snid=%snuri=%snret=%lxn", g_foo_settings, id, uri, ngx_random());
    return NGX_OK;
}

static ngx_int_t
ngx_http_hello_handler(ngx_http_request_t *r)
{
    ngx_int_t     rc;
    ngx_buf_t    *b;
    ngx_chain_t   out;

    /* Http Output Buffer */
    char out_buf[OUT_BUFSIZE] = {0};

    if (!(r->method & (NGX_HTTP_GET|NGX_HTTP_HEAD))) {
        return NGX_HTTP_NOT_ALLOWED;
    }

    rc = ngx_http_discard_request_body(r);

    if (rc != NGX_OK && rc != NGX_AGAIN) {
        return rc;
    }

    /* make http header */
    rc = make_http_header(r);
    if (rc != NGX_OK) {
        return rc;
    }

    if (r->method == NGX_HTTP_HEAD) {
        r->headers_out.status = NGX_HTTP_OK;
        return ngx_http_send_header(r);
    } else if (r->method == NGX_HTTP_GET) {
        /* make http get body buffer */
        rc = make_http_get_body(r, out_buf);
        if (rc != NGX_OK) {
            return rc;
        }
    } else {
        return NGX_HTTP_NOT_ALLOWED;
    }

    b = ngx_pcalloc(r->pool, sizeof(ngx_buf_t));
    if (b == NULL) {
        return NGX_HTTP_INTERNAL_SERVER_ERROR;
    }

    out.buf = b;
    out.next = NULL;

    b->pos = (u_char *)out_buf;
    b->last = (u_char *)out_buf + strlen(out_buf);
    b->memory = 1;
    b->last_buf = 1;
    r->headers_out.status = NGX_HTTP_OK;
    r->headers_out.content_length_n = strlen(out_buf);

    rc = ngx_http_send_header(r);

    if (rc == NGX_ERROR || rc > NGX_OK || r->header_only) {
        return rc;
    }

    return ngx_http_output_filter(r, &out);
}

static char *
ngx_http_hello_set(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
{
    ngx_http_core_loc_conf_t *clcf = ngx_http_conf_get_module_loc_conf(cf, ngx_http_core_module);

    /* register hanlder */
    clcf->handler = ngx_http_hello_handler;

    return NGX_CONF_OK;
}

static char *
ngx_http_foo_set(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
{
    ngx_str_t *value = cf->args->elts;
    memcpy(g_foo_settings, value[1].data, value[1].len);
    g_foo_settings[value[1].len] = '�';

    return NGX_CONF_OK;
}

static ngx_int_t
ngx_http_hello_process_init(ngx_cycle_t *cycle)
{
    // do some init here
    return NGX_OK;
}

static void
ngx_http_hello_process_exit(ngx_cycle_t *cycle)
{
    return;
}

4. 配置文件 nginx.conf

        location /hello {
            ngx_hello_module;
            hello 1234;
        }

5. 访问 http://localhost/hello

也可参考更详细的英文说明：
Emiller’s Guide To Nginx Module Development

• C, Erlang, Java and Go Web Server performance test
• MemcacheDB, Tokyo Tyrant, Redis performance test
• python web.py使用flup lighttpd优化过程
• 利用Gearman来实现远程监控与管理
• Twitter架构图(cache篇)