OF-205: Added configurable startup delay/retry for Hazelcast plugin

git-svn-id: http://svn.igniterealtime.org/svn/repos/openfire/branches/pubsub_clustering@13298 b35dd754-fafc-0310-a699-88a17e54d16e

src/plugins/hazelcast/src/java/com/jivesoftware/openfire/HazelcastPlugin.java

@@ -27,6 +27,7 @@ import java.text.ParseException;
 import java.util.Date;
 import java.util.Locale;
 import java.util.Map;
+import java.util.TimerTask;
 
 import org.jivesoftware.openfire.XMPPServer;
 import org.jivesoftware.openfire.cluster.ClusterManager;
@@ -35,6 +36,7 @@ import org.jivesoftware.openfire.container.PluginManager;
 import org.jivesoftware.util.JiveGlobals;
 import org.jivesoftware.util.PropertyEventDispatcher;
 import org.jivesoftware.util.PropertyEventListener;
+import org.jivesoftware.util.TaskEngine;
 import org.jivesoftware.util.cache.ExternalizableUtil;
 import org.jivesoftware.util.cache.ExternalizableUtilStrategy;
 import org.slf4j.Logger;
@@ -51,10 +53,13 @@ import com.jivesoftware.util.cluster.ClusterPacketRouter;
  * @author Tom Evans
  * @author Matt Tucker
  */
-public class HazelcastPlugin implements Plugin, PropertyEventListener {
+public class HazelcastPlugin extends TimerTask implements Plugin, PropertyEventListener {
 
     private static Logger logger = LoggerFactory.getLogger(HazelcastPlugin.class);
 
+    private static final long CLUSTER_STARTUP_DELAY_TIME = 
+    		JiveGlobals.getLongProperty("hazelcast.startup.delay.seconds", 5);
+    
     /**
      * Keep serialization strategy the server was using before we set our strategy. We will
      * restore old strategy when plugin is unloaded.
@@ -62,7 +67,14 @@ public class HazelcastPlugin implements Plugin, PropertyEventListener {
     private ExternalizableUtilStrategy serializationStrategy;
 
     public void initializePlugin(PluginManager manager, File pluginDirectory) {
-        System.out.println("Starting Clustering Plugin");
+    	// start cluster using a separate thread after a short delay
+    	// this will allow other plugins to initialize during startup
+    	TaskEngine.getInstance().schedule(this, CLUSTER_STARTUP_DELAY_TIME*1000);
+    }
+
+	@Override
+	public void run() {
+        System.out.println("Starting Hazelcast Clustering Plugin");
 
         // Check if another cluster is installed and stop loading this plugin if found
         File pluginDir = new File(JiveGlobals.getHomeDirectory(), "plugins");
@@ -88,10 +100,9 @@ public class HazelcastPlugin implements Plugin, PropertyEventListener {
             // Start up or join the cluster and initialize caches
             ClusterManager.startup();
         }
+	}
 
-    }
-
-    private void initForClustering() {
+	private void initForClustering() {
         // Set the serialization strategy to use for transmitting objects between node clusters
         serializationStrategy = ExternalizableUtil.getInstance().getStrategy();
         ExternalizableUtil.getInstance().setStrategy(new ClusterExternalizableUtil());
@@ -151,4 +162,5 @@ public class HazelcastPlugin implements Plugin, PropertyEventListener {
     public void xmlPropertyDeleted(String property, Map<String, Object> params) {
         // Do nothing
     }
+
 }

src/plugins/hazelcast/src/java/com/jivesoftware/util/cache/ClusteredCacheFactory.java

@@ -65,6 +65,10 @@ public class ClusteredCacheFactory implements CacheFactoryStrategy {
 
     private static final long MAX_CLUSTER_EXECUTION_TIME = 
     		JiveGlobals.getLongProperty("hazelcast.max.execution.seconds", 30);
+    private static final long CLUSTER_STARTUP_RETRY_TIME = 
+    		JiveGlobals.getLongProperty("hazelcast.startup.retry.seconds", 10);
+    private static final long CLUSTER_STARTUP_RETRY_COUNT = 
+    		JiveGlobals.getLongProperty("hazelcast.startup.retry.count", 1);
     private static final String HAZELCAST_CONFIG_FILE = 
     		JiveGlobals.getProperty("hazelcast.config.xml.filename", "hazelcast-cache-config.xml");
 
@@ -85,45 +89,48 @@ public class ClusteredCacheFactory implements CacheFactoryStrategy {
     private State state = State.stopped;
 
     public boolean startCluster() {
-        ClassLoader oldLoader = null;
-        // Set that we are starting up the cluster service
         state = State.starting;
-        try {
-            // Store previous class loader (in case we change it)
-            oldLoader = Thread.currentThread().getContextClassLoader();
-            ClassLoader loader = new ClusterClassLoader();
-            Thread.currentThread().setContextClassLoader(loader);
-            Config config = new ClasspathXmlConfig(HAZELCAST_CONFIG_FILE);
-            config.setInstanceName("openfire");
-        	hazelcast = Hazelcast.newHazelcastInstance(config);
-            cluster = hazelcast.getCluster();
-
-            // Update the running state of the cluster
-            state = cluster != null ? State.started : State.stopped;
-
-            Member localMember = cluster.getLocalMember();
-
-            // Set the ID of this cluster node
-            XMPPServer.getInstance().setNodeID(NodeID.getInstance(getClusterMemberID()));
-            // CacheFactory is now using clustered caches. We can add our listeners.
-            clusterListener = new ClusterListener(cluster);
-            hazelcast.getLifecycleService().addLifecycleListener(clusterListener);
-            cluster.addMembershipListener(clusterListener);
-
-            return cluster != null;
-        }
-        catch (Exception e) {
-            logger.error("Unable to start clustering - continuing in local mode", e);
-        }
-        finally {
-            if (oldLoader != null) {
-                // Restore previous class loader
-                Thread.currentThread().setContextClassLoader(oldLoader);
-            }
+        ClassLoader oldLoader = null;
+        // Store previous class loader (in case we change it)
+        oldLoader = Thread.currentThread().getContextClassLoader();
+        ClassLoader loader = new ClusterClassLoader();
+        Thread.currentThread().setContextClassLoader(loader);
+        int retry = 0;
+        do {
+            try {
+	            Config config = new ClasspathXmlConfig(HAZELCAST_CONFIG_FILE);
+	            config.setInstanceName("openfire");
+	        	hazelcast = Hazelcast.newHazelcastInstance(config);
+	            cluster = hazelcast.getCluster();
+	
+	            // Update the running state of the cluster
+	            state = cluster != null ? State.started : State.stopped;
+	
+	            // Set the ID of this cluster node
+	            XMPPServer.getInstance().setNodeID(NodeID.getInstance(getClusterMemberID()));
+	            // CacheFactory is now using clustered caches. We can add our listeners.
+	            clusterListener = new ClusterListener(cluster);
+	            hazelcast.getLifecycleService().addLifecycleListener(clusterListener);
+	            cluster.addMembershipListener(clusterListener);
+	            break;
+	        } catch (Exception e) {
+	            if (retry < CLUSTER_STARTUP_RETRY_COUNT) {
+	            	logger.warn("Failed to start clustering (" +  e.getMessage() + "); " +
+	            			"will retry in " + CLUSTER_STARTUP_RETRY_TIME + " seconds");
+	                try { Thread.sleep(CLUSTER_STARTUP_RETRY_TIME*1000); }
+	                catch (InterruptedException ie) { /* ignore */ }
+	            } else {
+	            	logger.error("Unable to start clustering - continuing in local mode", e);
+		            state = State.stopped;
+	            }
+	        }
+    	} while (retry++ < CLUSTER_STARTUP_RETRY_COUNT);
+        
+        if (oldLoader != null) {
+            // Restore previous class loader
+            Thread.currentThread().setContextClassLoader(oldLoader);
         }
-        // For some reason the cluster was not started so update the status
-        state = State.stopped;
-        return false;
+        return cluster != null;
     }
 
     public void stopCluster() {