using distributed lock with different responsibilities

include/ngx_http_push_stream_module.h

@@ -134,7 +134,6 @@ typedef struct {
 typedef struct {
     ngx_rbtree_node_t                   node;
     ngx_queue_t                         queue;
-    ngx_queue_t                        *queue_sentinel;
     ngx_str_t                           id;
     ngx_uint_t                          last_message_id;
     time_t                              last_message_time;
@@ -147,6 +146,7 @@ typedef struct {
     ngx_flag_t                          deleted;
     ngx_flag_t                          wildcard;
     ngx_http_push_stream_msg_t         *channel_deleted_message;
+    ngx_shmtx_t                        *mutex;
 } ngx_http_push_stream_channel_t;
 
 typedef struct {
@@ -242,9 +242,17 @@ struct ngx_http_push_stream_shm_data_s {
     ngx_uint_t                              stored_messages;    // # of messages being stored
     ngx_uint_t                              subscribers;        // # of subscribers in all channels
     ngx_queue_t                             messages_trash;
+    ngx_shmtx_t                             messages_trash_mutex;
+    ngx_shmtx_sh_t                          messages_trash_lock;
     ngx_queue_t                             channels_queue;
+    ngx_shmtx_t                             channels_queue_mutex;
+    ngx_shmtx_sh_t                          channels_queue_lock;
     ngx_queue_t                             channels_trash;
+    ngx_shmtx_t                             channels_trash_mutex;
+    ngx_shmtx_sh_t                          channels_trash_lock;
     ngx_queue_t                             channels_to_delete;
+    ngx_shmtx_t                             channels_to_delete_mutex;
+    ngx_shmtx_sh_t                          channels_to_delete_lock;
     ngx_uint_t                              channels_in_trash;  // # of channels in trash queue
     ngx_uint_t                              messages_in_trash;  // # of messages in trash queue
     ngx_http_push_stream_worker_data_t      ipc[NGX_MAX_PROCESSES]; // interprocess stuff
@@ -256,6 +264,11 @@ struct ngx_http_push_stream_shm_data_s {
     ngx_shm_zone_t                         *shm_zone;
     ngx_slab_pool_t                        *shpool;
     ngx_uint_t                              slots_for_census;
+    ngx_uint_t                              mutex_round_robin;
+    ngx_shmtx_t                             channels_mutex[10];
+    ngx_shmtx_sh_t                          channels_lock[10];
+    ngx_shmtx_t                             cleanup_mutex;
+    ngx_shmtx_sh_t                          cleanup_lock;
 };
 
 ngx_shm_zone_t     *ngx_http_push_stream_global_shm_zone = NULL;
@@ -389,6 +402,9 @@ static const ngx_str_t  NGX_HTTP_PUSH_STREAM_ALLOWED_HEADERS = ngx_string("If-Mo
 #define NGX_HTTP_PUSH_STREAM_DECREMENT_COUNTER(counter) \
     (counter = (counter > 1) ? counter - 1 : 0)
 
+#define NGX_HTTP_PUSH_STREAM_DECREMENT_COUNTER_BY(counter, qtd) \
+    (counter = (counter > qtd) ? counter - qtd : 0)
+
 #define NGX_HTTP_PUSH_STREAM_TIME_FMT_LEN   30 //sizeof("Mon, 28 Sep 1970 06:00:00 GMT")
 
 

include/ngx_http_push_stream_module_ipc.h

@@ -58,7 +58,7 @@ static ngx_int_t        ngx_http_push_stream_alert_worker(ngx_pid_t pid, ngx_int
 #define ngx_http_push_stream_alert_worker_delete_channel(pid, slot, log) ngx_http_push_stream_alert_worker(pid, slot, log, NGX_CMD_HTTP_PUSH_STREAM_DELETE_CHANNEL)
 #define ngx_http_push_stream_alert_worker_shutting_down_cleanup(pid, slot, log) ngx_http_push_stream_alert_worker(pid, slot, log, NGX_CMD_HTTP_PUSH_STREAM_CLEANUP_SHUTTING_DOWN)
 
-static ngx_int_t        ngx_http_push_stream_send_worker_message_locked(ngx_http_push_stream_channel_t *channel, ngx_queue_t *subscriptions_sentinel, ngx_pid_t pid, ngx_int_t worker_slot, ngx_http_push_stream_msg_t *msg, ngx_flag_t *queue_was_empty, ngx_log_t *log, ngx_http_push_stream_main_conf_t *mcf);
+static ngx_int_t        ngx_http_push_stream_send_worker_message(ngx_http_push_stream_channel_t *channel, ngx_queue_t *subscriptions_sentinel, ngx_pid_t pid, ngx_int_t worker_slot, ngx_http_push_stream_msg_t *msg, ngx_flag_t *queue_was_empty, ngx_log_t *log, ngx_http_push_stream_main_conf_t *mcf);
 
 static ngx_int_t        ngx_http_push_stream_init_ipc(ngx_cycle_t *cycle, ngx_int_t workers);
 static void             ngx_http_push_stream_ipc_exit_worker(ngx_cycle_t *cycle);

include/ngx_http_push_stream_module_utils.h

@@ -226,7 +226,7 @@ ngx_event_t         ngx_http_push_stream_memory_cleanup_event;
 ngx_event_t         ngx_http_push_stream_buffer_cleanup_event;
 
 // general request handling
-ngx_http_push_stream_msg_t *ngx_http_push_stream_convert_char_to_msg_on_shared_locked(ngx_http_push_stream_main_conf_t *mcf, u_char *data, size_t len, ngx_http_push_stream_channel_t *channel, ngx_int_t id, ngx_str_t *event_id, ngx_str_t *event_type, ngx_pool_t *temp_pool);
+ngx_http_push_stream_msg_t *ngx_http_push_stream_convert_char_to_msg_on_shared(ngx_http_push_stream_main_conf_t *mcf, u_char *data, size_t len, ngx_http_push_stream_channel_t *channel, ngx_int_t id, ngx_str_t *event_id, ngx_str_t *event_type, ngx_pool_t *temp_pool);
 static ngx_int_t            ngx_http_push_stream_send_only_added_headers(ngx_http_request_t *r);
 static void                 ngx_http_push_stream_add_polling_headers(ngx_http_request_t *r, time_t last_modified_time, ngx_int_t tag, ngx_pool_t *temp_pool);
 static void                 ngx_http_push_stream_get_last_received_message_values(ngx_http_request_t *r, time_t *if_modified_since, ngx_int_t *tag, ngx_str_t **last_event_id);
@@ -266,17 +266,17 @@ static void                 ngx_http_push_stream_timer_reset(ngx_msec_t timer_in
 #define ngx_http_push_stream_memory_cleanup_timer_set(void) ngx_http_push_stream_timer_set(NGX_HTTP_PUSH_STREAM_DEFAULT_SHM_MEMORY_CLEANUP_INTERVAL, &ngx_http_push_stream_memory_cleanup_event, ngx_http_push_stream_memory_cleanup_timer_wake_handler, 1);
 #define ngx_http_push_stream_buffer_cleanup_timer_set(pslcf) ngx_http_push_stream_timer_set(NGX_HTTP_PUSH_STREAM_MESSAGE_BUFFER_CLEANUP_INTERVAL, &ngx_http_push_stream_buffer_cleanup_event, ngx_http_push_stream_buffer_timer_wake_handler, pslcf->store_messages);
 
-static void                 ngx_http_push_stream_worker_subscriber_cleanup_locked(ngx_http_push_stream_subscriber_t *worker_subscriber);
+static void                 ngx_http_push_stream_worker_subscriber_cleanup(ngx_http_push_stream_subscriber_t *worker_subscriber);
 static ngx_str_t *          ngx_http_push_stream_create_str(ngx_pool_t *pool, uint len);
 
-static void                 ngx_http_push_stream_mark_message_to_delete_locked(ngx_http_push_stream_msg_t *msg, ngx_http_push_stream_shm_data_t *data);
+static void                 ngx_http_push_stream_throw_the_message_away(ngx_http_push_stream_msg_t *msg, ngx_http_push_stream_shm_data_t *data);
 static ngx_flag_t           ngx_http_push_stream_delete_channel(ngx_http_push_stream_main_conf_t *mcf, ngx_http_push_stream_channel_t *channel, u_char *text, size_t len, ngx_pool_t *temp_pool);
 static void                 ngx_http_push_stream_collect_expired_messages_data(ngx_http_push_stream_shm_data_t *data, ngx_flag_t force);
 static void                 ngx_http_push_stream_collect_expired_messages_and_empty_channels(ngx_flag_t force);
-static void                 ngx_http_push_stream_free_message_memory_locked(ngx_slab_pool_t *shpool, ngx_http_push_stream_msg_t *msg);
-static void                 ngx_http_push_stream_free_worker_message_memory_locked(ngx_slab_pool_t *shpool, ngx_http_push_stream_worker_msg_t *worker_msg);
+static void                 ngx_http_push_stream_free_message_memory(ngx_slab_pool_t *shpool, ngx_http_push_stream_msg_t *msg);
+static void                 ngx_http_push_stream_free_worker_message_memory(ngx_slab_pool_t *shpool, ngx_http_push_stream_worker_msg_t *worker_msg);
 static ngx_int_t            ngx_http_push_stream_free_memory_of_expired_messages_and_channels(ngx_flag_t force);
-static ngx_inline void      ngx_http_push_stream_ensure_qtd_of_messages_locked(ngx_http_push_stream_shm_data_t *data, ngx_http_push_stream_channel_t *channel, ngx_uint_t max_messages, ngx_flag_t expired);
+ngx_uint_t                  ngx_http_push_stream_ensure_qtd_of_messages(ngx_http_push_stream_shm_data_t *data, ngx_http_push_stream_channel_t *channel, ngx_uint_t max_messages, ngx_flag_t expired);
 static ngx_inline void      ngx_http_push_stream_delete_worker_channel(void);
 
 static ngx_http_push_stream_content_subtype_t *     ngx_http_push_stream_match_channel_info_format_and_content_type(ngx_http_request_t *r, ngx_uint_t default_subtype);
@@ -298,4 +298,6 @@ static ngx_int_t            ngx_http_push_stream_set_expires(ngx_http_request_t
 
 ngx_http_push_stream_requested_channel_t *ngx_http_push_stream_parse_channels_ids_from_path(ngx_http_request_t *r, ngx_pool_t *pool);
 
+ngx_int_t                   ngx_http_push_stream_create_shmtx(ngx_shmtx_t *mtx, ngx_shmtx_sh_t *addr, u_char *name);
+
 #endif /* NGX_HTTP_PUSH_STREAM_MODULE_UTILS_H_ */

misc/spec/mix/channel_statistics_spec.rb

@@ -648,7 +648,7 @@ shared_examples_for "statistics location" do
           pub.callback do
             pub.should be_http_status(200).without_body
 
-            sleep(2)
+            sleep(5)
 
             pub_3 = EventMachine::HttpRequest.new(nginx_address + '/channels-stats').get :head => headers
             pub_3.callback do

misc/spec/spec_helper.rb

@@ -12,7 +12,7 @@ Signal.trap("CLD", "IGNORE")
 
 RSpec.configure do |config|
   config.after(:each) do
-    non_time_wait_connections = `netstat -an | grep ":#{nginx_port} " | grep -v TIME_WAIT | grep -v LISTEN`.chomp.split("\n")
+    non_time_wait_connections = `netstat -an | grep ":#{nginx_port} " | grep -v TIME_WAIT | grep -v LISTEN | grep -v ESTABLISHED`.chomp.split("\n")
     abort "There are sockects on non time wait state: #{non_time_wait_connections.join("\n")}" if non_time_wait_connections.count > 0
     NginxTestHelper::Config.delete_config_and_log_files(config_id) if has_passed?
   end

src/ngx_http_push_stream_module.c

@@ -200,13 +200,12 @@ ngx_http_push_stream_send_response_all_channels_info_detailed(ngx_http_request_t
     ngx_http_push_stream_main_conf_t         *mcf = ngx_http_get_module_main_conf(r, ngx_http_push_stream_module);
     ngx_queue_t                               queue_channel_info;
     ngx_http_push_stream_shm_data_t          *data = mcf->shm_data;
-    ngx_slab_pool_t                          *shpool = mcf->shpool;
     ngx_queue_t                              *q;
     ngx_http_push_stream_channel_t           *channel;
 
     ngx_queue_init(&queue_channel_info);
 
-    ngx_shmtx_lock(&shpool->mutex);
+    ngx_shmtx_lock(&data->channels_queue_mutex);
     for (q = ngx_queue_head(&data->channels_queue); q != ngx_queue_sentinel(&data->channels_queue); q = ngx_queue_next(q)) {
         channel = ngx_queue_data(q, ngx_http_push_stream_channel_t, queue);
 
@@ -226,7 +225,7 @@ ngx_http_push_stream_send_response_all_channels_info_detailed(ngx_http_request_t
 
         }
     }
-    ngx_shmtx_unlock(&shpool->mutex);
+    ngx_shmtx_unlock(&data->channels_queue_mutex);
 
     return ngx_http_push_stream_send_response_channels_info(r, &queue_channel_info);
 }
@@ -235,8 +234,6 @@ static ngx_int_t
 ngx_http_push_stream_send_response_channels_info_detailed(ngx_http_request_t *r, ngx_http_push_stream_requested_channel_t *requested_channels) {
     ngx_str_t                                *text;
     ngx_queue_t                               queue_channel_info;
-    ngx_http_push_stream_main_conf_t         *mcf = ngx_http_get_module_main_conf(r, ngx_http_push_stream_module);
-    ngx_slab_pool_t                          *shpool = mcf->shpool;
     ngx_http_push_stream_content_subtype_t   *subtype = ngx_http_push_stream_match_channel_info_format_and_content_type(r, 1);
     ngx_http_push_stream_channel_info_t      *channel_info;
     ngx_http_push_stream_requested_channel_t *requested_channel;
@@ -245,7 +242,6 @@ ngx_http_push_stream_send_response_channels_info_detailed(ngx_http_request_t *r,
 
     ngx_queue_init(&queue_channel_info);
 
-    ngx_shmtx_lock(&shpool->mutex);
     for (q = ngx_queue_head(&requested_channels->queue); q != ngx_queue_sentinel(&requested_channels->queue); q = ngx_queue_next(q)) {
         requested_channel = ngx_queue_data(q, ngx_http_push_stream_requested_channel_t, queue);
 
@@ -260,7 +256,6 @@ ngx_http_push_stream_send_response_channels_info_detailed(ngx_http_request_t *r,
             qtd_channels++;
         }
     }
-    ngx_shmtx_unlock(&shpool->mutex);
 
     if (qtd_channels == 0) {
         return ngx_http_push_stream_send_only_header_response(r, NGX_HTTP_NOT_FOUND, NULL);

src/ngx_http_push_stream_module_publisher.c

@@ -99,7 +99,6 @@ ngx_http_push_stream_publisher_handler(ngx_http_request_t *r)
             // create the channel if doesn't exist
             requested_channel->channel = ngx_http_push_stream_get_channel(requested_channel->id, r->connection->log, cf, mcf);
             if (requested_channel->channel == NULL) {
-                ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "push stream module: unable to allocate memory for new channel");
                 return ngx_http_push_stream_send_only_header_response(r, NGX_HTTP_INTERNAL_SERVER_ERROR, NULL);
             }
 

src/ngx_http_push_stream_module_setup.c

@@ -1077,5 +1077,35 @@ ngx_http_push_stream_init_shm_zone(ngx_shm_zone_t *shm_zone, void *data)
 
     ngx_queue_insert_tail(&global_shm_data->shm_datas_queue, &d->shm_data_queue);
 
+    if (ngx_http_push_stream_create_shmtx(&d->messages_trash_mutex, &d->messages_trash_lock, (u_char *) "push_stream_messages_trash") != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    if (ngx_http_push_stream_create_shmtx(&d->channels_queue_mutex, &d->channels_queue_lock, (u_char *) "push_stream_channels_queue") != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    if (ngx_http_push_stream_create_shmtx(&d->channels_to_delete_mutex, &d->channels_to_delete_lock, (u_char *) "push_stream_channels_to_delete") != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    if (ngx_http_push_stream_create_shmtx(&d->channels_trash_mutex, &d->channels_trash_lock, (u_char *) "push_stream_channels_trash") != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    if (ngx_http_push_stream_create_shmtx(&d->cleanup_mutex, &d->cleanup_lock, (u_char *) "push_stream_cleanup") != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    u_char lock_name[25];
+    for (i = 0; i < 10; i++) {
+        ngx_sprintf(lock_name, "push_stream_channels_%d", i);
+        if (ngx_http_push_stream_create_shmtx(&d->channels_mutex[i], &d->channels_lock[i], lock_name) != NGX_OK) {
+            return NGX_ERROR;
+        }
+    }
+
+    d->mutex_round_robin = 0;
+
     return NGX_OK;
 }

src/ngx_http_push_stream_module_websocket.c

@@ -38,7 +38,6 @@ ngx_http_push_stream_websocket_handler(ngx_http_request_t *r)
 #endif
     ngx_http_push_stream_main_conf_t               *mcf = ngx_http_get_module_main_conf(r, ngx_http_push_stream_module);
     ngx_http_push_stream_loc_conf_t                *cf = ngx_http_get_module_loc_conf(r, ngx_http_push_stream_module);
-    ngx_slab_pool_t                                *shpool = mcf->shpool;
     ngx_http_push_stream_subscriber_t              *worker_subscriber;
     ngx_http_push_stream_requested_channel_t       *requested_channels, *requested_channel;
     ngx_queue_t                                    *q;
@@ -46,7 +45,6 @@ ngx_http_push_stream_websocket_handler(ngx_http_request_t *r)
     ngx_int_t                                       tag;
     time_t                                          if_modified_since;
     ngx_str_t                                      *last_event_id = NULL;
-    ngx_int_t                                       rc;
     ngx_int_t                                       status_code;
     ngx_str_t                                      *explain_error_message;
     ngx_str_t                                      *upgrade_header, *connection_header, *sec_key_header, *sec_version_header, *sec_accept_header;
@@ -131,11 +129,7 @@ ngx_http_push_stream_websocket_handler(ngx_http_request_t *r)
         return ngx_http_push_stream_send_websocket_close_frame(r, NGX_HTTP_INTERNAL_SERVER_ERROR, &NGX_HTTP_PUSH_STREAM_EMPTY);
     }
 
-    ngx_shmtx_lock(&shpool->mutex);
-    rc = ngx_http_push_stream_registry_subscriber_locked(r, worker_subscriber);
-    ngx_shmtx_unlock(&shpool->mutex);
-
-    if (rc == NGX_ERROR) {
+    if (ngx_http_push_stream_registry_subscriber(r, worker_subscriber) == NGX_ERROR) {
         return ngx_http_push_stream_send_websocket_close_frame(r, NGX_HTTP_INTERNAL_SERVER_ERROR, &NGX_HTTP_PUSH_STREAM_EMPTY);
     }
 

src/ngx_http_push_stream_rbtree_util.c

@@ -76,7 +76,6 @@ ngx_http_push_stream_find_channel_on_tree(ngx_str_t *id, ngx_log_t *log, ngx_rbt
 static ngx_http_push_stream_channel_t *
 ngx_http_push_stream_find_channel(ngx_str_t *id, ngx_log_t *log, ngx_http_push_stream_main_conf_t *mcf)
 {
-    ngx_slab_pool_t                    *shpool = mcf->shpool;
     ngx_http_push_stream_shm_data_t    *data = mcf->shm_data;
     ngx_http_push_stream_channel_t     *channel = NULL;
 
@@ -85,12 +84,9 @@ ngx_http_push_stream_find_channel(ngx_str_t *id, ngx_log_t *log, ngx_http_push_s
         return NULL;
     }
 
-    ngx_shmtx_lock(&shpool->mutex);
+    ngx_shmtx_lock(&data->channels_queue_mutex);
     channel = ngx_http_push_stream_find_channel_on_tree(id, log, &data->tree);
-    ngx_shmtx_unlock(&shpool->mutex);
-    if ((channel == NULL) || channel->deleted) {
-        return NULL;
-    }
+    ngx_shmtx_unlock(&data->channels_queue_mutex);
 
     return channel;
 }
@@ -110,12 +106,12 @@ ngx_http_push_stream_get_channel(ngx_str_t *id, ngx_log_t *log, ngx_http_push_st
         return NULL;
     }
 
-    ngx_shmtx_lock(&shpool->mutex);
+    ngx_shmtx_lock(&data->channels_queue_mutex);
 
     // check again to see if any other worker didn't create the channel
     channel = ngx_http_push_stream_find_channel_on_tree(id, log, &data->tree);
     if (channel != NULL) { // we found our channel
-        ngx_shmtx_unlock(&shpool->mutex);
+        ngx_shmtx_unlock(&data->channels_queue_mutex);
         return channel;
     }
 
@@ -125,18 +121,21 @@ ngx_http_push_stream_get_channel(ngx_str_t *id, ngx_log_t *log, ngx_http_push_st
 
     if (((!is_wildcard_channel) && (mcf->max_number_of_channels != NGX_CONF_UNSET_UINT) && (mcf->max_number_of_channels == data->channels)) ||
         ((is_wildcard_channel) && (mcf->max_number_of_wildcard_channels != NGX_CONF_UNSET_UINT) && (mcf->max_number_of_wildcard_channels == data->wildcard_channels))) {
-        ngx_shmtx_unlock(&shpool->mutex);
+        ngx_shmtx_unlock(&data->channels_queue_mutex);
+        ngx_log_error(NGX_LOG_ERR, log, 0, "push stream module: number of channels were exceeded");
         return NGX_HTTP_PUSH_STREAM_NUMBER_OF_CHANNELS_EXCEEDED;
     }
 
-    if ((channel = ngx_slab_alloc_locked(shpool, sizeof(ngx_http_push_stream_channel_t))) == NULL) {
-        ngx_shmtx_unlock(&shpool->mutex);
+    if ((channel = ngx_slab_alloc(shpool, sizeof(ngx_http_push_stream_channel_t))) == NULL) {
+        ngx_shmtx_unlock(&data->channels_queue_mutex);
+        ngx_log_error(NGX_LOG_ERR, log, 0, "push stream module: unable to allocate memory for new channel");
         return NULL;
     }
 
-    if ((channel->id.data = ngx_slab_alloc_locked(shpool, id->len + 1)) == NULL) {
-        ngx_slab_free_locked(shpool, channel);
-        ngx_shmtx_unlock(&shpool->mutex);
+    if ((channel->id.data = ngx_slab_alloc(shpool, id->len + 1)) == NULL) {
+        ngx_slab_free(shpool, channel);
+        ngx_shmtx_unlock(&data->channels_queue_mutex);
+        ngx_log_error(NGX_LOG_ERR, log, 0, "push stream module: unable to allocate memory for new channel id");
         return NULL;
     }
 
@@ -160,10 +159,11 @@ ngx_http_push_stream_get_channel(ngx_str_t *id, ngx_log_t *log, ngx_http_push_st
     channel->node.key = ngx_crc32_short(channel->id.data, channel->id.len);
     ngx_rbtree_insert(&data->tree, &channel->node);
     ngx_queue_insert_tail(&data->channels_queue, &channel->queue);
-    channel->queue_sentinel = &data->channels_queue;
     (channel->wildcard) ? data->wildcard_channels++ : data->channels++;
 
-    ngx_shmtx_unlock(&shpool->mutex);
+    channel->mutex = &data->channels_mutex[data->mutex_round_robin++ % 9];
+
+    ngx_shmtx_unlock(&data->channels_queue_mutex);
     return channel;
 }