adding basic load test tools

.gitignore

@@ -5,3 +5,6 @@ build/
 test/.bundle/
 test/tmp/
 misc/*.textile*
+*.o
+misc/tools/publisher
+misc/tools/subscriber

misc/tools/Makefile

+all: publisher subscriber 
+
+subscriber: subscriber.o util.o
+	gcc -g -Oo subscriber.o util.o -o subscriber -largtable2
+
+publisher: publisher.o util.o
+	gcc -g -Oo publisher.o util.o -o publisher -largtable2
+
+subscriber.o: subscriber.c
+	gcc -g -c subscriber.c
+
+publisher.o: publisher.c
+	gcc -g -c publisher.c
+
+util.o: util.c
+	gcc -g -c util.c
+
+clean:
+	rm -rf *o publisher subscriber

misc/tools/README

+Copyright (C) 2011 Michael Costello, Wandenberg Peixoto <wandenberg@gmail.com>
+
+These tools, publisher and subscriber, were developed only to do some load tests on push stream module.
+Their use is very restricted and is not intended to cover all possible configuration for the module.
+The first version was developed by Michael Costello and I made some improvements to distribute it.
+Feel free to help continuous improvement.
+
+Any feedbacks will be welcome.
+
+=============
+Requirements:
+=============
+
+lib argtable2
+GCC, make, the usual guys
+epoll event support
+
+================
+Developer Guide:
+================
+
+The basic configuration used on the load tests are listed bellow.
+To compile the tools only execute a make.
+
+To see all options use:
+  ./publisher --help
+  ./subscriber --help
+
+Pay attention on default values to run your tests.
+
+====================
+Basic Configuration:
+====================
+
+pid         logs/nginx.pid;
+error_log   logs/nginx-main_error.log debug;
+
+worker_rlimit_core  500M;
+working_directory /tmp/nginx;
+
+worker_processes    2;
+
+events {
+  worker_connections  1024;
+  use                 epoll;
+}
+
+http {
+  include         mime.types;
+  default_type    application/octet-stream;
+
+  access_log      logs/nginx-http_access.log;
+  error_log       logs/nginx-http_error.log debug;
+
+  push_stream_shared_memory_size               500M;
+
+  server {
+    listen           9080 default_server;
+    server_name     localhost;
+
+    location /channels-stats {
+      push_stream_channels_statistics;
+      set $push_stream_channel_id             $arg_id;
+    }
+
+    location /pub {
+      push_stream_publisher admin;
+      set $push_stream_channel_id             $arg_id;
+      push_stream_store_messages              off;
+      push_stream_keepalive                   on;
+    }
+
+    location ~ /sub/(.*) {
+      push_stream_subscriber;
+      set $push_stream_channels_path              $1;
+      push_stream_message_template                "~text~:~id~:~channel~";
+    }
+  }
+}

misc/tools/util.c

+#include "util.h"
+
+
+int
+fill_server_address(const char *server_hostname, int server_port, struct sockaddr_in *server_address)
+{
+    struct hostent *server = NULL;
+    if ((server = gethostbyname(server_hostname)) == NULL) {
+        return EXIT_FAILURE;
+    }
+
+    bzero((char *) server_address, sizeof(struct sockaddr_in));
+    server_address->sin_family = AF_INET;
+    memcpy((char *) &server_address->sin_addr.s_addr, (const char *) server->h_addr, server->h_length);
+    server_address->sin_port = htons(server_port);
+
+    return EXIT_SUCCESS;
+}
+
+
+Connection *
+init_connections(int num_connections, struct sockaddr_in *server_address, int main_sd)
+{
+    Connection *connections;
+    int         i;
+
+    if ((connections = (Connection *) malloc(sizeof(Connection) * num_connections)) == NULL) {
+        return NULL;
+    }
+
+    for (i = 0; i < num_connections; ++i) {
+        connections[i].index = i;
+        connections[i].server_address = server_address;
+        connections[i].main_sd = main_sd;
+        if (open_connection(&connections[i]) != 0) {
+            error("Opening connection %d\n", i);
+            return NULL;
+        }
+    }
+
+    info("Added %d connections.\n", num_connections);
+
+    return connections;
+}
+
+
+int
+open_connection(Connection *connection)
+{
+    struct epoll_event anEvent;
+    int exitcode = EXIT_SUCCESS;
+
+    connection->state = CONNECTING;
+    connection->channel_id = -1;
+    connection->content_length = 0;
+    connection->channel_count = 0;
+
+    if ((connection->sd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
+        error3("ERROR %d opening socket for connection %d\n", errno, connection->index);
+    }
+
+//    // set nonblocking
+//    int flags = fcntl(connection->sd, F_GETFL, 0);
+//    fcntl(connection->sd, F_SETFL, flags | O_NONBLOCK);
+//
+//    int rc = connect(connection->sd, (struct sockaddr *) connection->server_address, sizeof(struct sockaddr_in));
+//    if ((rc < 0) && (errno != EINPROGRESS))  {
+//        error3("ERROR connecting to server on connection %d\n", connection->index);
+//    }
+
+    if (connect(connection->sd, (struct sockaddr *) connection->server_address, sizeof(struct sockaddr_in)) < 0)  {
+        error3("ERROR connecting to server on connection %d\n", connection->index);
+    }
+
+    debug("Adding connection %d\n", connection->index);
+
+    anEvent.events = EPOLLIN | EPOLLOUT | EPOLLHUP;
+    anEvent.data.ptr = (void *) connection;
+    if (epoll_ctl(connection->main_sd, EPOLL_CTL_ADD, connection->sd, &anEvent) < 0)	{
+        error3("ERROR %d Failed creating socket for connection %d\n", errno, connection->index);
+    }
+
+    debug("Connection opening for index %d\n", connection->index);
+
+exit:
+    return exitcode;
+}
+
+
+void
+close_connection(Connection *connection)
+{
+    connection->state = CLOSED;
+    close(connection->sd);
+}
+
+
+
+int
+reopen_connection(Connection *connection)
+{
+    close_connection(connection);
+    return open_connection(connection);
+}
+
+
+int
+change_connection(Connection *connection, uint32_t events)
+{
+    struct epoll_event anEvent;
+    anEvent.events = events;
+    anEvent.data.ptr = (void *) connection;
+    return epoll_ctl(connection->main_sd, EPOLL_CTL_MOD, connection->sd, &anEvent);
+}
+
+
+int
+write_connection(Connection *connection, Statistics *stats, char *buffer, int buffer_len)
+{
+    int bytes_written = 0;
+
+    bytes_written = write(connection->sd, buffer, buffer_len);
+
+    if (bytes_written != buffer_len) {
+        error4("Error %d writing bytes (wrote=%d, wanted=%d) for connection %d\n", errno, bytes_written, buffer_len, connection->index);
+        return EXIT_FAILURE;
+    }
+    stats->bytes_written += bytes_written;
+    trace("Wrote %s\n", buffer);
+
+    return EXIT_SUCCESS;
+}
+
+
+float
+calc_message_per_second(int num_messages, int start_time)
+{
+    float ret_val = 0.0;
+    int now = time(NULL);
+    int diff = now - start_time;
+
+    if (diff == 0) {
+        diff = 1;
+    }
+
+    ret_val = (float) num_messages/diff;
+
+    info("CALC TIME.  Messages=%d, Time=%d Avg=%0.2f\n", num_messages, diff, ret_val);
+
+    return ret_val;
+}
+
+
+int
+count_strinstr(const char *big, const char *little)
+{
+    const char *p;
+    int count = 0;
+    size_t lil_len = strlen(little);
+
+    /* you decide what to do here */
+    if (lil_len == 0)
+        return -1;
+
+    p = strstr(big, little);
+    while (p) {
+    count++;
+    p = strstr(p + lil_len, little);
+    }
+    return count;
+}
+

misc/tools/util.h

+#ifndef _UTIL_H_
+#define _UTIL_H_
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/socket.h>
+#include <sys/epoll.h>
+#include <netdb.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#define error(...)  {fprintf(stderr, __VA_ARGS__);}
+#define error2(...) {error(__VA_ARGS__); exitcode = EXIT_FAILURE; goto exit;}
+#define error3(...) {perror("ERROR");error(__VA_ARGS__); exitcode = EXIT_FAILURE; goto exit;}
+#define error4(...) {perror("ERROR");error(__VA_ARGS__);}
+
+#define info(...) {if (verbose_messages >= 1) {fprintf(stdout, __VA_ARGS__);}}
+#define debug(...) {if (verbose_messages >= 2) {fprintf(stdout, __VA_ARGS__);}}
+#define trace(...) {if (verbose_messages >= 3) {fprintf(stdout, __VA_ARGS__);}}
+
+#define summary(...) fprintf(stdout, __VA_ARGS__)
+
+#define VERSION   "0.1"
+#define COPYRIGHT "Copyright (C) 2011 Michael Costello, Wandenberg Peixoto <wandenberg@gmail.com>"
+#define DESCRIPTION_PUBLISHER "'%s' v%s - program to publish messages to test Push Stream Module.\n%s\n"
+#define DESCRIPTION_SUBSCRIBER "'%s' v%s - program to subscribe channels to test Push Stream Module.\n%s\n"
+
+#define DEFAULT_NUM_MESSAGES    1
+#define DEFAULT_CONCURRENT_CONN 1
+#define DEFAULT_NUM_CHANNELS    1
+#define DEFAULT_SERVER_HOSTNAME "127.0.0.1"
+#define DEFAULT_SERVER_PORT     9080
+#define DEFAULT_TIMEOUT         1000
+
+#define MAX_EVENTS (60000 * 8)
+#define ITERATIONS_TILL_SUMMARY_PER_TIMEOUT 10000 //timeout: 1000 -> summary each 10 seconds
+#define BUFFER_SIZE 1024
+#define BIG_BUFFER_SIZE 640000
+
+typedef struct
+{
+    long requested_connections;
+    long connections;
+    long messages;
+    long bytes_written;
+    long bytes_read;
+} Statistics;
+
+enum State {INIT=0, CONNECTING, CONNECTED, CLOSED};
+
+// store per connection state here
+typedef struct
+{
+    int		   index;
+    int        main_sd;
+    int		   sd;
+    int		   message_count;
+    int		   num_messages;
+    int		   channel_count;
+    long	   channel_id;
+    long	   channel_start;
+    long	   channel_end;
+    char       content_buffer[BUFFER_SIZE];
+    int        content_length;
+    enum State state;
+    struct sockaddr_in *server_address;
+} Connection;
+
+static int verbose_messages = 0;
+
+int fill_server_address(const char *server_hostname, int server_port, struct sockaddr_in *server_address);
+Connection *init_connections(int count, struct sockaddr_in *server_address, int main_sd);
+int open_connection(Connection *connection);
+void close_connection(Connection *connection);
+int reopen_connection(Connection *connection);
+int write_connection(Connection *connection, Statistics *stats, char *buffer, int buffer_len);
+int change_connection(Connection *connection, uint32_t events);
+float calc_message_per_second(int num_messages, int start_time);
+
+
+#endif /* _UTIL_H_ */