Commit 64b9bc3f authored by Wandenberg Peixoto's avatar Wandenberg Peixoto

Initial commit

parents
build/
.cproject
.project
h1. Nginx Push Stream Module
A pure stream http push technology for your Nginx setup.
Comet made easy and *really scalable*.
h2. Installing
<pre>
<code>
# clone
git clone http://github.com/wandenberg/nginx-push-stream-module.git
cd nginx-push-stream-module
# build 0.7.67
./build.sh master 0.7.67
cd build/nginx-0.7.67
# or build 0.8.53
./build.sh master 0.8.53
cd build/nginx-0.8.53
# finish
sudo make install
# checking
sudo /usr/local/nginx/sbin/nginx -v
nginx version: nginx/0.8.53
sudo /usr/local/nginx/sbin/nginx -c nginx-push-stream-module/misc/nginx.conf -t
the configuration file nginx-push-stream-module/misc/nginx.conf syntax is ok
configuration file nginx-push-stream-module/misc/nginx.conf test is successful
# running
sudo /usr/local/nginx/sbin/nginx -c nginx-push-stream-module/misc/nginx.conf
</code>
</pre>
h2. Basic Configuration
<pre>
<code>
location /pub {
# activate publisher mode for this location
push_stream_publisher;
# query string based channel id
set $push_stream_channel_id $arg_id;
# message template
push_stream_message_template "<script>p(~id~,'~channel~','~text~');</script>";
# max messages to store in memory
push_stream_max_message_buffer_length 20;
# message ttl
push_stream_min_message_buffer_timeout 5m;
# client_max_body_size MUST be equal to client_body_buffer_size or
# you will be sorry.
client_max_body_size 32k;
client_body_buffer_size 32k;
}
location ~ /sub/(.*) {
# activate subscriber mode for this location
push_stream_subscriber;
# positional channel path
set $push_stream_channels_path $1;
# header to be sent when receiving new subscriber connection
push_stream_header_template "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\r\n<meta http-equiv=\"Cache-Control\" content=\"no-store\">\r\n<meta http-equiv=\"Cache-Control\" content=\"no-cache\">\r\n<meta http-equiv=\"Pragma\" content=\"no-cache\">\r\n<meta http-equiv=\"Expires\" content=\"Thu, 1 Jan 1970 00:00:00 GMT\">\r\n<script type=\"text/javascript\">\r\nwindow.onError = null;\r\ndocument.domain = 'localhost';\r\nparent.PushStream.register(this);\r\n</script>\r\n</head>\r\n<body onload=\"try { parent.PushStream.reset(this) } catch (e) {}\">";
# message template
push_stream_message_template "<script>p(~id~,'~channel~','~text~');</script>";
# content-type
push_stream_content_type "text/html; charset=utf-8";
# subscriber may create channels on demand or only authorized
# (publisher) may do it?
push_stream_authorized_channels_only off;
# ping frequency
push_stream_ping_message_interval 10s;
# disconnection candidates test frequency
push_stream_subscriber_disconnect_interval 30s;
# connection ttl to enable recycle
push_stream_subscriber_connection_timeout 15m;
# solving some leakage problem with persistent connections in
# Nginx's chunked filter (ngx_http_chunked_filter_module.c)
chunked_transfer_encoding off;
}
</code>
</pre>
h2. Basic Usage
You can feel the flavor right now at the command line. Try using more than
one terminal and start playing http pubsub:
<pre>
<code>
# Pub
curl -s -v -X POST "http://localhost/pub?id=my_channel_1" -d "Hello World!"
# Sub
curl -s -v "http://localhost/sub/my_channel_1.b20"
# Channel Stats (text format)
curl -s -v "http://localhost/pub?id=my_channel_1"
# All Channels Stats (json format)
curl -s -v "http://localhost/pub?id=ALL"
</code>
</pre>
#!/bin/bash
TAG="$1"
NGINX_VERSION="$2"
PREFIX="nginx-push-stream-module"
CONFIGURE_OPTIONS="--without-select_module \
--without-poll_module \
--without-http_charset_module \
--without-http_ssi_module \
--without-http_auth_basic_module \
--without-http_autoindex_module \
--without-http_geo_module \
--without-http_map_module \
--without-http_referer_module \
--without-http_fastcgi_module \
--without-http_memcached_module \
--without-http_limit_zone_module \
--without-http_limit_req_module \
--without-http_empty_gif_module \
--without-http_browser_module \
--without-http_upstream_ip_hash_module \
--without-mail_pop3_module \
--without-mail_imap_module \
--without-mail_smtp_module \
--with-http_stub_status_module \
--add-module=nginx-push-stream-module"
if [[ -z "$TAG" || -z "$NGINX_VERSION" ]]
then
echo "Usage: $0 <tag> <nginx_version>"
exit 1
fi
(./pack.sh $TAG && \
cd build && \
rm -rf nginx-${NGINX_VERSION}* && \
wget http://sysoev.ru/nginx/nginx-${NGINX_VERSION}.tar.gz && \
tar xzvf nginx-${NGINX_VERSION}.tar.gz && \
cd nginx-$NGINX_VERSION && \
tar xzvf ../$PREFIX-$TAG.tar.gz && \
./configure $CONFIGURE_OPTIONS && \
make && \
echo "
##############################################################
Build generated: build/nginx-$NGINX_VERSION
Configure options used:
$CONFIGURE_OPTIONS
To finish the process:
cd build/nginx-$NGINX_VERSION
sudo make install") || \
(echo "There was a problem building the module" ; exit 1)
echo "##############################################################"
ngx_feature="http_push_stream_module"
ngx_feature_name=
ngx_feature_run=no
ngx_feature_incs=
ngx_feature_path=
ngx_feature_libs=
ngx_feature_test=
ngx_addon_name=ngx_http_push_stream_module
HTTP_MODULES="$HTTP_MODULES ngx_http_push_stream_module"
CORE_INCS="$CORE_INCS \
$ngx_addon_dir/src"
NGX_ADDON_SRCS="$NGX_ADDON_SRCS \
${ngx_addon_dir}/src/ngx_http_push_stream_module.c"
have=NGX_HTTP_HEADERS . auto/have
. auto/feature
types {
text/html html htm shtml;
text/css css;
text/xml xml;
image/gif gif;
image/jpeg jpeg jpg;
application/x-javascript js;
application/atom+xml atom;
application/rss+xml rss;
text/mathml mml;
text/plain txt;
text/vnd.sun.j2me.app-descriptor jad;
text/vnd.wap.wml wml;
text/x-component htc;
image/png png;
image/tiff tif tiff;
image/vnd.wap.wbmp wbmp;
image/x-icon ico;
image/x-jng jng;
image/x-ms-bmp bmp;
image/svg+xml svg;
application/java-archive jar war ear;
application/mac-binhex40 hqx;
application/msword doc;
application/pdf pdf;
application/postscript ps eps ai;
application/rtf rtf;
application/vnd.ms-excel xls;
application/vnd.ms-powerpoint ppt;
application/vnd.wap.wmlc wmlc;
application/vnd.wap.xhtml+xml xhtml;
application/vnd.google-earth.kml+xml kml;
application/vnd.google-earth.kmz kmz;
application/x-cocoa cco;
application/x-java-archive-diff jardiff;
application/x-java-jnlp-file jnlp;
application/x-makeself run;
application/x-perl pl pm;
application/x-pilot prc pdb;
application/x-rar-compressed rar;
application/x-redhat-package-manager rpm;
application/x-sea sea;
application/x-shockwave-flash swf;
application/x-stuffit sit;
application/x-tcl tcl tk;
application/x-x509-ca-cert der pem crt;
application/x-xpinstall xpi;
application/zip zip;
application/octet-stream bin exe dll;
application/octet-stream deb;
application/octet-stream dmg;
application/octet-stream eot;
application/octet-stream iso img;
application/octet-stream msi msp msm;
audio/midi mid midi kar;
audio/mpeg mp3;
audio/x-realaudio ra;
video/3gpp 3gpp 3gp;
video/mpeg mpeg mpg;
video/quicktime mov;
video/x-flv flv;
video/x-mng mng;
video/x-ms-asf asx asf;
video/x-ms-wmv wmv;
video/x-msvideo avi;
}
pid logs/nginx.pid;
error_log logs/nginx-main_error.log;
worker_processes 2;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
access_log logs/nginx-http_access.log;
error_log logs/nginx-http_error.log;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 10;
send_timeout 10;
client_body_timeout 10;
client_header_timeout 10;
sendfile on;
client_header_buffer_size 1k;
large_client_header_buffers 2 4k;
client_max_body_size 1k;
client_body_buffer_size 1k;
ignore_invalid_headers on;
client_body_in_single_buffer on;
server {
listen 80;
server_name localhost;
location /pub {
# activate publisher mode for this location
push_stream_publisher;
# query string based channel id
set $push_stream_channel_id $arg_id;
# message template
push_stream_message_template "<script>p(~id~,'~channel~','~text~');</script>";
# max messages to store in memory
push_stream_max_message_buffer_length 20;
# message ttl
push_stream_min_message_buffer_timeout 5m;
# client_max_body_size MUST be equal to client_body_buffer_size or
# you will be sorry.
client_max_body_size 32k;
client_body_buffer_size 32k;
}
location ~ /sub/(.*) {
# activate subscriber mode for this location
push_stream_subscriber;
# positional channel path
set $push_stream_channels_path $1;
# header to be sent when receiving new subscriber connection
push_stream_header_template "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\r\n<meta http-equiv=\"Cache-Control\" content=\"no-store\">\r\n<meta http-equiv=\"Cache-Control\" content=\"no-cache\">\r\n<meta http-equiv=\"Pragma\" content=\"no-cache\">\r\n<meta http-equiv=\"Expires\" content=\"Thu, 1 Jan 1970 00:00:00 GMT\">\r\n<script type=\"text/javascript\">\r\nwindow.onError = null;\r\ndocument.domain = 'localhost';\r\nparent.PushStream.register(this);\r\n</script>\r\n</head>\r\n<body onload=\"try { parent.PushStream.reset(this) } catch (e) {}\">";
# message template
push_stream_message_template "<script>p(~id~,'~channel~','~text~');</script>";
# content-type
push_stream_content_type "text/html; charset=utf-8";
# subscriber may create channels on demand or only authorized
# (publisher) may do it?
push_stream_authorized_channels_only off;
# ping frequency
push_stream_ping_message_interval 10s;
# disconnection candidates test frequency
push_stream_subscriber_disconnect_interval 30s;
# connection ttl to enable recycle
push_stream_subscriber_connection_timeout 15m;
# solving some leakage problem with persitent connections in
# Nginx's chunked filter (ngx_http_chunked_filter_module.c)
chunked_transfer_encoding off;
}
}
}
#!/bin/bash
TAG="$1"
PREFIX="nginx-push-stream-module"
if [[ -z "$TAG" ]]
then
echo "Usage: $0 <tag>"
exit 1
fi
mkdir -p build
git archive --format=tar --prefix=$PREFIX/ $TAG src config | gzip > build/$PREFIX-$TAG.tar.gz
echo "Package generated: build/$PREFIX-$TAG.tar.gz"
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
#include <ngx_http_push_stream_module.h>
static void ngx_rbtree_generic_insert(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel, int (*compare) (const ngx_rbtree_node_t *left, const ngx_rbtree_node_t *right));
static void ngx_http_push_stream_rbtree_insert(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel);
static int ngx_http_push_stream_compare_rbtree_node(const ngx_rbtree_node_t *v_left, const ngx_rbtree_node_t *v_right);
static ngx_int_t ngx_http_push_stream_delete_node_locked(ngx_rbtree_t *tree, ngx_rbtree_node_t *trash, ngx_slab_pool_t *shpool);
static ngx_http_push_stream_channel_t *
ngx_http_push_stream_clean_channel_locked(ngx_http_push_stream_channel_t *channel)
{
ngx_queue_t *sentinel = &channel->message_queue->queue;
time_t now = ngx_time();
ngx_http_push_stream_msg_t *msg = NULL;
while (!ngx_queue_empty(sentinel)) {
msg = ngx_queue_data(ngx_queue_head(sentinel), ngx_http_push_stream_msg_t, queue);
if (msg != NULL && msg->expires != 0 && now > msg->expires) {
ngx_http_push_stream_delete_message_locked(channel, msg, ngx_http_push_stream_shpool);
} else { // definitely a message left to send
return NULL;
}
}
// at this point, the queue is empty
return channel->subscribers == 0 ? channel : NULL; // if no waiting requests, return this channel to be deleted
}
static ngx_int_t
ngx_http_push_stream_delete_channel_locked(ngx_http_push_stream_channel_t *trash)
{
ngx_int_t res;
res = ngx_http_push_stream_delete_node_locked(&((ngx_http_push_stream_shm_data_t *) ngx_http_push_stream_shm_zone->data)->tree, (ngx_rbtree_node_t *) trash, ngx_http_push_stream_shpool);
if (res == NGX_OK) {
((ngx_http_push_stream_shm_data_t *) ngx_http_push_stream_shm_zone->data)->channels--;
return NGX_OK;
}
return res;
}
static ngx_int_t
ngx_http_push_stream_delete_node_locked(ngx_rbtree_t *tree, ngx_rbtree_node_t *trash, ngx_slab_pool_t *shpool)
{
// assume the shm zone is already locked
if (trash != NULL) { // take out the trash
ngx_rbtree_delete(tree, trash);
// delete the worker-subscriber queue
ngx_queue_t *sentinel = (ngx_queue_t *) (&((ngx_http_push_stream_channel_t *) trash)->workers_with_subscribers);
ngx_queue_t *cur = ngx_queue_head(sentinel);
ngx_queue_t *next;
while (cur != sentinel) {
next = ngx_queue_next(cur);
ngx_slab_free_locked(shpool, cur);
cur = next;
}
ngx_slab_free_locked(shpool, trash);
return NGX_OK;
}
return NGX_DECLINED;
}
static ngx_http_push_stream_channel_t *
ngx_http_push_stream_find_channel(ngx_str_t *id, ngx_log_t *log)
{
ngx_rbtree_t *tree = &((ngx_http_push_stream_shm_data_t *) ngx_http_push_stream_shm_zone->data)->tree;
uint32_t hash;
ngx_rbtree_node_t *node, *sentinel;
ngx_int_t rc;
ngx_http_push_stream_channel_t *up = NULL;
ngx_http_push_stream_channel_t *trash[] = { NULL, NULL, NULL };
ngx_uint_t i, trashed = 0;
if (tree == NULL) {
return NULL;
}
hash = ngx_crc32_short(id->data, id->len);
node = tree->root;
sentinel = tree->sentinel;
while (node != sentinel) {
// every search is responsible for deleting a couple of empty, if it comes across them
if (trashed < (sizeof(trash) / sizeof(*trash))) {
if ((trash[trashed] = ngx_http_push_stream_clean_channel_locked((ngx_http_push_stream_channel_t *) node)) != NULL) {
trashed++;
}
}
if (hash < node->key) {
node = node->left;
continue;
}
if (hash > node->key) {
node = node->right;
continue;
}
/* hash == node->key */
do {
up = (ngx_http_push_stream_channel_t *) node;
rc = ngx_memn2cmp(id->data, up->id.data, id->len, up->id.len);
if (rc == 0) {
// found
for(i=0; i<trashed; i++) {
if (trash[i] != up) { // take out the trash
ngx_http_push_stream_delete_channel_locked(trash[i]);
}
}
ngx_http_push_stream_clean_channel_locked(up);
return up;
}
node = (rc < 0) ? node->left : node->right;
} while (node != sentinel && hash == node->key);
break;
}
// not found
for(i=0; i<trashed; i++) {
ngx_http_push_stream_delete_channel_locked(trash[i]);
}
return NULL;
}
// find a channel by id. if channel not found, make one, insert it, and return that.
static ngx_http_push_stream_channel_t *
ngx_http_push_stream_get_channel(ngx_str_t *id, ngx_log_t *log)
{
ngx_rbtree_t *tree;
ngx_http_push_stream_channel_t *up = ngx_http_push_stream_find_channel(id, log);
if (up != NULL) { // we found our channel
return up;
}
tree = &((ngx_http_push_stream_shm_data_t *) ngx_http_push_stream_shm_zone->data)->tree;
if ((up = ngx_http_push_stream_slab_alloc_locked(sizeof(*up) + id->len + 1 + sizeof(ngx_http_push_stream_msg_t))) == NULL) {
return NULL;
}
up->id.data = (u_char *) (up+1); // contiguous piggy
up->message_queue = (ngx_http_push_stream_msg_t *) (up->id.data + id->len + 1);
up->id.len = (u_char) id->len;
ngx_memzero(up->id.data, up->id.len + 1);
ngx_memcpy(up->id.data, id->data, up->id.len);
up->node.key = ngx_crc32_short(id->data, id->len);
ngx_rbtree_insert(tree, (ngx_rbtree_node_t *) up);
// initialize queues
ngx_queue_init(&up->message_queue->queue);
up->last_message_id = 0;
up->stored_messages = 0;
ngx_queue_init(&up->workers_with_subscribers.queue);
up->subscribers = 0;
((ngx_http_push_stream_shm_data_t *) ngx_http_push_stream_shm_zone->data)->channels++;
return up;
}
static void
ngx_rbtree_generic_insert(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel, int (*compare) (const ngx_rbtree_node_t *left, const ngx_rbtree_node_t *right))
{
for (;;) {
if (node->key < temp->key) {
if (temp->left == sentinel) {
temp->left = node;
break;
}
temp = temp->left;
} else if (node->key > temp->key) {
if (temp->right == sentinel) {
temp->right = node;
break;
}
temp = temp->right;
} else { /* node->key == temp->key */
if (compare(node, temp) < 0) {
if (temp->left == sentinel) {
temp->left = node;
break;
}
temp = temp->left;
} else {
if (temp->right == sentinel) {
temp->right = node;
break;
}
temp = temp->right;
}
}
}
node->parent = temp;
node->left = sentinel;
node->right = sentinel;
ngx_rbt_red(node);
}
#define ngx_http_push_stream_walk_rbtree(apply) \
ngx_http_push_stream_rbtree_walker(&((ngx_http_push_stream_shm_data_t *) ngx_http_push_stream_shm_zone->data)->tree, (ngx_slab_pool_t *) ngx_http_push_stream_shm_zone->shm.addr, apply, ((ngx_http_push_stream_shm_data_t *) ngx_http_push_stream_shm_zone->data)->tree.root)
static void
ngx_http_push_stream_rbtree_walker(ngx_rbtree_t *tree, ngx_slab_pool_t *shpool, ngx_int_t (*apply) (ngx_http_push_stream_channel_t *channel, ngx_slab_pool_t *shpool), ngx_rbtree_node_t *node)
{
ngx_rbtree_node_t *sentinel = tree->sentinel;
if (node != sentinel) {
apply((ngx_http_push_stream_channel_t *) node, shpool);
if (node->left != NULL) {
ngx_http_push_stream_rbtree_walker(tree, shpool, apply, node->left);
}
if (node->right != NULL) {
ngx_http_push_stream_rbtree_walker(tree, shpool, apply, node->right);
}
}
}
static void
ngx_http_push_stream_rbtree_insert(ngx_rbtree_node_t *temp, ngx_rbtree_node_t *node, ngx_rbtree_node_t *sentinel)
{
ngx_rbtree_generic_insert(temp, node, sentinel, ngx_http_push_stream_compare_rbtree_node);
}
static int
ngx_http_push_stream_compare_rbtree_node(const ngx_rbtree_node_t *v_left, const ngx_rbtree_node_t *v_right)
{
ngx_http_push_stream_channel_t *left = (ngx_http_push_stream_channel_t *) v_left, *right = (ngx_http_push_stream_channel_t *) v_right;
return ngx_memn2cmp(left->id.data, right->id.data, left->id.len, right->id.len);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment