jetty8+jdk6+repcachedでセッションクラスタリング

検証環境

CentOS release 6.5
サーバ1: 192.168.0.1
サーバ2: 192.168.0.2

ソースファイル

# ls /usr/local/src
jetty-distribution-8.1.14.v20131031.tar.gz
memcached-1.2.8-repcached-2.2.1.tar.gz
jdk-6u45-linux-x64-rpm.bin
nginx-1.5.8.tar.gz

java

2台のサーバで実施

インストール

# cd /usr/local/src
# bash jdk-6u45-linux-x64-rpm.bin
# java -version
java version "1.6.0_45" 
Java(TM) SE Runtime Environment (build 1.6.0_45-b06)
Java HotSpot(TM) 64-Bit Server VM (build 20.45-b01, mixed mode)

jetty

2台のサーバで実施

インストール

# cd /usr/local/src
# tar zxvf jetty-distribution-8.1.14.v20131031.tar.gz
# mv jetty-distribution-8.1.14.v20131031 /usr/local/
# ln -s /usr/local/jetty-distribution-8.1.14.v20131031 /usr/local/jetty

# useradd jetty
# chown -R jetty.jetty /usr/local/jetty-distribution-8.1.14.v20131031

起動
# su - jetty
# /usr/local/jetty/bin/jetty.sh start

終了
# /usr/local/jetty/bin/jetty.sh stop

nginx

192.168.1.1側だけでよい

インストール

# cd /usr/local/src
# useradd nginx
# yum -y install gcc zlib-devel pcre-devel
# yum -y install openssl-devel
# yum -y install libxslt-devel
# yum -y install git

# tar zxvf nginx-1.5.8.tar.gz

# git clone https://github.com/agentzh/headers-more-nginx-module.git

# cd nginx-1.5.8
# ./configure --prefix=/usr/local/nginx-1.5.8 \
  --user=nginx \
  --group=nginx \
  --with-http_ssl_module \
  --with-http_realip_module \
  --with-http_addition_module \
  --with-http_xslt_module \
  --with-http_sub_module \
  --with-http_dav_module \
  --with-http_flv_module \
  --with-http_gzip_static_module \
  --with-http_random_index_module \
  --with-http_secure_link_module \
  --with-http_stub_status_module \
  --add-module=/usr/local/src/headers-more-nginx-module
# make && make install

# ln -s /usr/local/nginx-1.5.8 /usr/local/nginx
# mkdir  /usr/local/nginx/conf/virtualhost

起動スクリプト(/etc/init.d/nginx)

#!/bin/sh
#
# nginx - this script starts and stops the nginx daemon
#
# chkconfig:   - 85 15
# description:  Nginx is an HTTP(S) server, HTTP(S) reverse \
#               proxy and IMAP/POP3 proxy server
# processname: nginx
# config:      /etc/nginx/nginx.conf
# config:      /etc/sysconfig/nginx
# pidfile:     /var/run/nginx.pid

# Source function library.
. /etc/rc.d/init.d/functions

# Source networking configuration.
. /etc/sysconfig/network

# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0

nginx="/usr/local/nginx/sbin/nginx" 
prog=$(basename $nginx)

NGINX_CONF_FILE="/usr/local/nginx/conf/nginx.conf" 

[ -f /etc/sysconfig/nginx ] && . /etc/sysconfig/nginx

lockfile=/var/lock/subsys/nginx

make_dirs() {
   # make required directories
   user=`$nginx -V 2>&1 | grep "configure arguments:" | sed 's/[^*]*--user=\([^ ]*\).*/\1/g' -`
   user="nginx" 
   if [ -z "`grep $user /etc/passwd`" ]; then
       useradd -M -s /bin/nologin $user
   fi
   options=`$nginx -V 2>&1 | grep 'configure arguments:'`
   for opt in $options; do
       if [ `echo $opt | grep '.*-temp-path'` ]; then
           value=`echo $opt | cut -d "=" -f 2`
           if [ ! -d "$value" ]; then
               # echo "creating" $value
               mkdir -p $value && chown -R $user $value
           fi
       fi
   done
}

start() {
    [ -x $nginx ] || exit 5
    [ -f $NGINX_CONF_FILE ] || exit 6
    make_dirs
    echo -n $"Starting $prog: " 
    daemon $nginx -c $NGINX_CONF_FILE
    retval=$?
    echo
    [ $retval -eq 0 ] && touch $lockfile
    return $retval
}

stop() {
    echo -n $"Stopping $prog: " 
    killproc $prog -QUIT
    retval=$?
    echo
    [ $retval -eq 0 ] && rm -f $lockfile
    return $retval
}

restart() {
    configtest || return $?
    stop
    sleep 1
    start
}

reload() {
    configtest || return $?
    echo -n $"Reloading $prog: " 
    killproc $nginx -HUP
    RETVAL=$?
    echo
}

force_reload() {
    restart
}

configtest() {
  $nginx -t -c $NGINX_CONF_FILE
}

rh_status() {
    status $prog
}

rh_status_q() {
    rh_status >/dev/null 2>&1
}

case "$1" in
    start)
        rh_status_q && exit 0
        $1
        ;;
    stop)
        rh_status_q || exit 0
        $1
        ;;
    restart|configtest)
        $1
        ;;
    reload)
        rh_status_q || exit 7
        $1
        ;;
    force-reload)
        force_reload
        ;;
    status)
        rh_status
        ;;
    condrestart|try-restart)
        rh_status_q || exit 0
            ;;
    *)
        echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}" 
        exit 2
esac

自動起動

# chmod 755 /etc/init.d/nginx
# chkconfig add nginx
# chkconfig nginx on

/usr/local/nginx/conf/nginx.conf

user nginx;
worker_processes  2;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;
error_log  logs/error.log  debug;

pid        logs/nginx.pid;

events {
    worker_connections  1024;
}

http {
    include       mime.types;
    default_type  application/octet-stream;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    client_max_body_size 256M;

    server {
        listen       80;
        server_name  localhost;

        #charset koi8-r;

        access_log  /usr/local/nginx/logs/localhost.access.log  main;
        #access_log  logs/host.access.log  main;

        try_files $uri $uri.html $uri/index.html /;

        #nginxで処理するファイル
        location ~ .*\.(jpg|JPG|gif|GIF|png|PNG|swf|SWF|css|CSS|js|JS|inc|INC|ico|ICO) {
            root    /usr/local/nginx/html;
            index   index.html;
            ssi     on;
            expires 1h;
            break;
        }

        #エラーページのカスタマイズ
        location / {
            error_page  400 401 402 403 404 500 501 502 503 504  /error.html;
            location = /error.html {
                root   /usr/local/nginx/html;
            }
        }

    }

    #各バーチャルホストの設定
    include /usr/local/nginx/conf/virtualhost/*.conf;
}

/usr/local/nginx/conf/virtualhost/192.168.1.1.conf

upstream test-node {
  server 192.168.1.1:8080;
  server 192.168.1.2:8080;
}
server {
  listen 80;
  server_name 192.168.1.1;
  access_log /usr/local/nginx/logs/192.168.1.1.access.log  main;
  try_files $uri $uri.html $uri/index.html /;

  #とりあえず全部jettyに渡してしまう設定
  location / {
    proxy_pass http://test-node;
    break;
  }

  error_page   400 401 402 403 404 500 501 502 503 504  /error.html;
  location = /error.html {
    root /usr/local/nginx/html;
  }
}

起動

/etc/init.d/nginx start

repcached

2台のサーバで実施

インストール

# yum install -y libevent-devel memcached

# tar zxvf memcached-1.2.8-repcached-2.2.1.tar.gz
# cd memcached-1.2.8-repcached-2.2.1
# ./configure --enable-replication --prefix=/usr/local/memcached-1.2.8-repcached-2.2
# make
# make install
# ln -s /usr/local/memcached-1.2.8-repcached-2.2 /usr/local/repcached

以下のrepcached起動スクリプト内REPHOSTの設定は相手側のIPを記載すること

起動スクリプト(/etc/init.d/repcached)

#! /bin/sh
#
# chkconfig: - 55 45
# description:  The memcached daemon is a network memory cache service.
# processname: memcached
# config: /etc/sysconfig/memcached

# Source function library.
. /etc/rc.d/init.d/functions

PORT=11211
USER=memcached
MAXCONN=1024
CACHESIZE=640
OPTIONS=""
REPHOST=192.168.1.1
#REPHOST=192.168.1.2

if [ -f /etc/sysconfig/memcached ];then
        . /etc/sysconfig/memcached
fi

# Check that networking is up.
if [ "$NETWORKING" = "no" ]
then
        exit 0
fi

RETVAL=0
prog="memcached"

start () {
        echo -n $"Starting $prog: "
        # insure that /var/run/memcached has proper permissions
        chown $USER /var/run/memcached
        daemon /usr/local/repcached/bin/memcached -d -x $REPHOST -p $PORT -u $USER  -m $CACHESIZE -c $MAXCONN -P /var/run/memcached/memcached.pid $OPTIONS
        RETVAL=$?
        echo
        [ $RETVAL -eq 0 ] && touch /var/lock/subsys/memcached
}
stop () {
        echo -n $"Stopping $prog: "
        killproc memcached
        RETVAL=$?
        echo
        if [ $RETVAL -eq 0 ] ; then
            rm -f /var/lock/subsys/memcached
            rm -f /var/run/memcached.pid
        fi
}

restart () {
        stop
        start
}
# See how we were called.
case "$1" in
  start)
        start
        ;;
  stop)
        stop
        ;;
  status)
        status memcached
        ;;
  restart|reload)
        restart
        ;;
  condrestart)
        [ -f /var/lock/subsys/memcached ] && restart || :
        ;;
  *)
        echo $"Usage: $0 {start|stop|status|restart|reload|condrestart}"
        exit 1
esac

exit $?

自動起動

# chmod 755 /etc/init.d/repcached
# chkconfig add repcached
# chkconfig repcached on

起動

/etc/init.d/repcached start

jettyセッションのmemcached保存

以下のURLを参考に設定する
https://github.com/yyuu/jetty-nosql-memcached/tree/jetty8

jarの配置

ls /usr/local/jetty/lib/ext
jetty-nosql-kvs-0.3.1.jar
jetty-nosql-memcached-0.3.1-jar-with-dependencies.jar

/usr/local/jetty/etc/jetty.xmlに追記

<!-- Memcached -->
    <Set name="sessionIdManager">
      <New id="memcachedSessionIdManager" class="org.eclipse.jetty.nosql.memcached.MemcachedSessionIdManager">
        <Arg><Ref id="Server" /></Arg>
        <Set name="serverString">localhost:11211</Set>
        <Set name="keyPrefix">session:</Set>
      </New>
    </Set>
    <Call name="setAttribute">
      <Arg>memcachedSessionIdManager</Arg>
      <Arg><Ref id="memcachedSessionIdManager" /></Arg>
    </Call>

以下はアプリケーションがROOT.warにある想定
各自の環境によって読み替える。

/usr/local/jetty/webapps/ROOT.war/WEB-INF/jetty-web.xmlを作成

<?xml version="1.0"  encoding="ISO-8859-1"?>
<!DOCTYPE Configure PUBLIC "-//Mort Bay Consulting//DTD Configure//EN" "http://www.eclipse.org/jetty/configure.dtd">

<!--
This is the jetty specific web application configuration file.  When starting
a Web Application, the WEB-INF/web-jetty.xml file is looked for and if found, treated
as a org.eclipse.jetty.server.server.xml.XmlConfiguration file and is applied to the
org.eclipse.jetty.servlet.WebApplicationContext objet
-->

<Configure class="org.eclipse.jetty.webapp.WebAppContext">
  <Call class="org.eclipse.jetty.util.log.Log" name="debug"><Arg>executing jetty-web.xml</Arg></Call>
  <!-- <Set name="contextPath">/mycontext</Set> -->

  <Get name="server">
    <Get id="memcachedSessionIdManager" name="sessionIdManager" />
  </Get>
  <Set name="sessionHandler">
    <New class="org.eclipse.jetty.server.session.SessionHandler">
      <Arg>
        <New class="org.eclipse.jetty.nosql.memcached.MemcachedSessionManager">
          <Set name="sessionIdManager">
            <Ref id="memcachedSessionIdManager" />
          </Set>
        </New>
      </Arg>
    </New>
  </Set>

</Configure>

起動

# su - jetty
# /usr/local/jetty/bin/jetty.sh start

アクセスしてみる