OF-602: Hazelcast clustering plugin improvements (install/upgrade/docs)

git-svn-id: http://svn.igniterealtime.org/svn/repos/openfire/trunk@13396 b35dd754-fafc-0310-a699-88a17e54d16e

src/java/org/jivesoftware/openfire/pubsub/PubSubModule.java

@@ -34,8 +34,6 @@ import org.jivesoftware.openfire.PacketRouter;
 import org.jivesoftware.openfire.RoutableChannelHandler;
 import org.jivesoftware.openfire.RoutingTable;
 import org.jivesoftware.openfire.XMPPServer;
-import org.jivesoftware.openfire.cluster.ClusterEventListener;
-import org.jivesoftware.openfire.cluster.ClusterManager;
 import org.jivesoftware.openfire.commands.AdHocCommandManager;
 import org.jivesoftware.openfire.component.InternalComponentManager;
 import org.jivesoftware.openfire.container.BasicModule;
@@ -68,7 +66,7 @@ import org.xmpp.packet.Presence;
  * @author Matt Tucker
  */
 public class PubSubModule extends BasicModule implements ServerItemsProvider, DiscoInfoProvider,
-        DiscoItemsProvider, RoutableChannelHandler, PubSubService, ClusterEventListener, PropertyEventListener {
+        DiscoItemsProvider, RoutableChannelHandler, PubSubService, PropertyEventListener {
 
 	private static final Logger Log = LoggerFactory.getLogger(PubSubModule.class);
 
@@ -434,8 +432,6 @@ public class PubSubModule extends BasicModule implements ServerItemsProvider, Di
         else {
             rootCollectionNode = (CollectionNode) getNode(rootNodeID);
         }
-        // Listen to cluster events
-        ClusterManager.addListener(this);
     }
 
     @Override
@@ -503,24 +499,6 @@ public class PubSubModule extends BasicModule implements ServerItemsProvider, Di
         return serviceEnabled;
     }
 
-    public void joinedCluster() {
-        // Disable the service until we know that we are the senior cluster member
-		// enableService(false);
-    }
-
-    public void joinedCluster(byte[] nodeID) {
-        // Do nothing
-    }
-
-    public void leftCluster() {
-        // Offer the service when not running in a cluster
-		// enableService(true);
-    }
-
-    public void leftCluster(byte[] nodeID) {
-        // Do nothing
-    }
-
     public void markedAsSeniorClusterMember() {
         // Offer the service since we are the senior cluster member
 		// enableService(true);

src/java/org/jivesoftware/util/cache/CacheFactory.java

@@ -435,7 +435,16 @@ public class CacheFactory {
      * this JVM to join a cluster.
      */
     public static boolean isClusteringAvailable() {
-        return getMaxClusterNodes() > 1;
+    	if (clusteredCacheFactoryStrategy == null) {
+	        try {
+	        	clusteredCacheFactoryStrategy = (CacheFactoryStrategy) Class.forName(
+	        			clusteredCacheFactoryClass, true,
+	        			getClusteredCacheStrategyClassLoader()).newInstance();
+	        } catch (Exception e) {
+	        	log.warn("Clustered cache factory strategy " + clusteredCacheFactoryClass + " not found");
+	        }
+    	}
+    	return (clusteredCacheFactoryStrategy != null);
     }
 
     /**
@@ -609,13 +618,9 @@ public class CacheFactory {
     }
 
     public static void startClustering() {
-        try {
-        	clusteredCacheFactoryStrategy = (CacheFactoryStrategy) Class.forName(clusteredCacheFactoryClass, true,
-                    							getClusteredCacheStrategyClassLoader()).newInstance();
-            clusteringStarting = clusteredCacheFactoryStrategy.startCluster();
-        } catch (Exception e) {
-        	log.error("Clustered cache factory strategy " + clusteredCacheFactoryClass + " not found", e);
-        }
+    	if (isClusteringAvailable()) {
+    		clusteringStarting = clusteredCacheFactoryStrategy.startCluster();
+    	}
         if (clusteringStarting) {
             if (statsThread == null) {
                 // Start a timing thread with 1 second of accuracy.
@@ -677,6 +682,7 @@ public class CacheFactory {
     public static void stopClustering() {
         // Stop the cluster
     	clusteredCacheFactoryStrategy.stopCluster();
+    	clusteredCacheFactoryStrategy = null;
         // Set the strategy to local
         cacheFactoryStrategy = localCacheFactoryStrategy;
     }
@@ -691,7 +697,6 @@ public class CacheFactory {
         for (Cache cache : getAllCaches()) {
             // skip local-only caches
             if (localOnly.contains(cache.getName())) continue;
-            cache.clear();
             CacheWrapper cacheWrapper = ((CacheWrapper) cache);
             Cache clusteredCache = cacheFactoryStrategy.createCache(cacheWrapper.getName());
             cacheWrapper.setWrappedCache(clusteredCache);
@@ -713,7 +718,6 @@ public class CacheFactory {
         for (Cache cache : getAllCaches()) {
             // skip local-only caches
             if (localOnly.contains(cache.getName())) continue;
-            cache.clear();
             CacheWrapper cacheWrapper = ((CacheWrapper) cache);
             Cache standaloneCache = cacheFactoryStrategy.createCache(cacheWrapper.getName());
             cacheWrapper.setWrappedCache(standaloneCache);

src/plugins/clustering/src/java/com/jivesoftware/openfire/ClusteringPlugin.java

@@ -20,10 +20,10 @@
 
 package com.jivesoftware.openfire;
 
-import com.jivesoftware.openfire.session.RemoteSessionLocator;
-import com.jivesoftware.util.cache.CoherenceExternalizableUtil;
-import com.jivesoftware.util.cluster.CoherencePacketRouter;
-import com.tangosol.net.CacheFactory;
+import java.io.File;
+import java.io.FileFilter;
+import java.net.MalformedURLException;
+import java.net.URL;
 
 import org.jivesoftware.openfire.XMPPServer;
 import org.jivesoftware.openfire.cluster.ClusterManager;
@@ -31,36 +31,18 @@ import org.jivesoftware.openfire.container.Plugin;
 import org.jivesoftware.openfire.container.PluginManager;
 import org.jivesoftware.util.JiveGlobals;
 import org.jivesoftware.util.Log;
-import org.jivesoftware.util.PropertyEventDispatcher;
-import org.jivesoftware.util.PropertyEventListener;
-import org.jivesoftware.util.cache.ExternalizableUtil;
-import org.jivesoftware.util.cache.ExternalizableUtilStrategy;
 
-import java.io.File;
-import java.io.FileFilter;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.util.Date;
-import java.util.Locale;
-import java.util.Map;
+import com.tangosol.net.CacheFactory;
 
 /**
  * Clustering Enterprise plugin.
  *
  * @author Matt Tucker
  */
-public class ClusteringPlugin implements Plugin, PropertyEventListener {
+public class ClusteringPlugin implements Plugin {
     private static final String COHERENCE_CONFIG = "tangosol-coherence-override";
     private static final String COHERENCE_CACHE_CONFIG = "coherence-cache-config";
 
-    /**
-     * Keep serialization strategy the server was using before we set our strategy. We will
-     * restore old strategy when plugin is unloaded.
-     */
-    private ExternalizableUtilStrategy serializationStrategy;
-
     public void initializePlugin(PluginManager manager, File pluginDirectory) {
         System.out.println("Starting Clustering Plugin");
 
@@ -101,9 +83,6 @@ public class ClusteringPlugin implements Plugin, PropertyEventListener {
 //                    "tangosol.jar, coherence.jar and coherence-work.jar files to [OPENFIRE_HOME]/lib and restart the server.");
 //        }
 
-        // List for clustering setting events (e.g. enabled/disabled)
-        PropertyEventDispatcher.addListener(this);
-
         // Delete no longer used COHERENCE_CONFIG file. Java system properties should be used
         // to customize coherence
         File configFile = new File(enterpriseDir, COHERENCE_CONFIG + ".xml");
@@ -127,76 +106,14 @@ public class ClusteringPlugin implements Plugin, PropertyEventListener {
         catch (MalformedURLException e) {
             Log.error("Error adding openfireHome/enterprise to the classpath of the enterprise plugin", e);
         }
-
-        if (ClusterManager.isClusteringEnabled()) {
-            initForClustering();
-
-            // Start up or join the cluster and initialize caches
-            ClusterManager.startup();
-        }
-
-    }
-
-    private void initForClustering() {
-        // Set the serialization strategy to use for transmitting objects between node clusters
-        serializationStrategy = ExternalizableUtil.getInstance().getStrategy();
-        ExternalizableUtil.getInstance().setStrategy(new CoherenceExternalizableUtil());
-        // Set session locator to use when in a cluster
-        XMPPServer.getInstance().setRemoteSessionLocator(new RemoteSessionLocator());
-        // Set packet router to use to deliver packets to remote cluster nodes
-        XMPPServer.getInstance().getRoutingTable().setRemotePacketRouter(new CoherencePacketRouter());
-        // Initialize the Coherence cluster configuration
         CacheFactory.getClusterConfig();
-    }
-
-    /**
-     * Returns the date when this release of Openfire Enterprise was released.
-     *
-     * @return the date when this release of Openfire Enterprise was released.
-     */
-    public static Date getReleaseDate() {
-        try {
-            // @DATE@ should be replaced with a date with the following format: Jan 31, 2007
-            // Automaticly set by ANT build tasks
-            return DateFormat.getDateInstance(DateFormat.MEDIUM, Locale.US).parse("@DATE@");
-        }
-        catch (ParseException e) {
-            Log.error("Error parsing date", e);
-            return null;
-        }
+        ClusterManager.startup();
     }
 
     public void destroyPlugin() {
-        ClusterManager.shutdown();
-
-        // Set the old serialization strategy was using before enterprise was loaded
-        ExternalizableUtil.getInstance().setStrategy(serializationStrategy);
-
-        // Stop listing for clustering setting events (e.g. enabled/disabled)
-        PropertyEventDispatcher.removeListener(this);
-    }
-
-    public void propertySet(String property, Map<String, Object> params) {
-        // Ignore
-    }
-
-    public void propertyDeleted(String property, Map<String, Object> params) {
-        // Ignore
-    }
-
-    public void xmlPropertySet(String property, Map<String, Object> params) {
-        if (ClusterManager.CLUSTER_PROPERTY_NAME.equals(property)) {
-            if (Boolean.parseBoolean((String) params.get("value"))) {
-                // Clustering was enabled
-                initForClustering();
-            }
-            else {
-                // Clustering was disabled
-            }
-        }
-    }
-
-    public void xmlPropertyDeleted(String property, Map<String, Object> params) {
-        // Do nothing
+        // Shutdown is initiated by XMPPServer before unloading plugins
+    	if (!XMPPServer.getInstance().isShuttingDown()) {
+    		ClusterManager.shutdown();
+    	}
     }
 }

src/plugins/clustering/src/java/com/jivesoftware/util/cache/ClusterListener.java

@@ -252,8 +252,6 @@ public class ClusterListener implements MemberListener {
             // Clean up all traces. This will set all remote sessions as unavailable
             List<NodeID> nodeIDs = new ArrayList<NodeID>(nodeSessions.keySet());
 
-            // Revert cluster caches to local caches
-            CacheFactory.leftCluster();
             // Trigger event. Wait until the listeners have processed the event. Caches will be populated
             // again with local content.
             ClusterManager.fireLeftCluster();

src/plugins/clustering/src/java/com/jivesoftware/util/cache/ClusteredCacheFactory.java

@@ -19,8 +19,19 @@
 
 package com.jivesoftware.util.cache;
 
-import com.jivesoftware.util.cluster.CoherenceClusterNodeInfo;
-import com.tangosol.net.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+
 import org.jivesoftware.openfire.XMPPServer;
 import org.jivesoftware.openfire.cluster.ClusterManager;
 import org.jivesoftware.openfire.cluster.ClusterNodeInfo;
@@ -30,11 +41,17 @@ import org.jivesoftware.util.cache.Cache;
 import org.jivesoftware.util.cache.CacheFactoryStrategy;
 import org.jivesoftware.util.cache.CacheWrapper;
 import org.jivesoftware.util.cache.ClusterTask;
+import org.jivesoftware.util.cache.ExternalizableUtil;
+import org.jivesoftware.util.cache.ExternalizableUtilStrategy;
 
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
+import com.jivesoftware.openfire.session.RemoteSessionLocator;
+import com.jivesoftware.util.cluster.CoherenceClusterNodeInfo;
+import com.jivesoftware.util.cluster.CoherencePacketRouter;
+import com.tangosol.net.AbstractInvocable;
+import com.tangosol.net.Cluster;
+import com.tangosol.net.Invocable;
+import com.tangosol.net.InvocationService;
+import com.tangosol.net.Member;
 
 /**
  * CacheFactory implementation to use when using Coherence in cluster mode.
@@ -43,6 +60,12 @@ import java.util.concurrent.locks.Lock;
  */
 public class ClusteredCacheFactory implements CacheFactoryStrategy {
 
+    /**
+     * Keep serialization strategy the server was using before we set our strategy. We will
+     * restore old strategy when plugin is unloaded.
+     */
+    private ExternalizableUtilStrategy serializationStrategy;
+
     /**
      * Storage for cache statistics
      */
@@ -69,6 +92,14 @@ public class ClusteredCacheFactory implements CacheFactoryStrategy {
         ClassLoader oldLoader = null;
         // Set that we are starting up the cluster service
         state = State.starting;
+        // Set the serialization strategy to use for transmitting objects between node clusters
+        serializationStrategy = ExternalizableUtil.getInstance().getStrategy();
+        ExternalizableUtil.getInstance().setStrategy(new CoherenceExternalizableUtil());
+        // Set session locator to use when in a cluster
+        XMPPServer.getInstance().setRemoteSessionLocator(new RemoteSessionLocator());
+        // Set packet router to use to deliver packets to remote cluster nodes
+        XMPPServer.getInstance().getRoutingTable().setRemotePacketRouter(new CoherencePacketRouter());
+        // Initialize the Coherence cluster configuration
         try {
             // Store previous class loader (in case we change it)
             oldLoader = Thread.currentThread().getContextClassLoader();
@@ -158,6 +189,13 @@ public class ClusteredCacheFactory implements CacheFactoryStrategy {
         }
         // Reset the node ID
         XMPPServer.getInstance().setNodeID(null);
+
+        // Reset packet router to use to deliver packets to remote cluster nodes
+        XMPPServer.getInstance().getRoutingTable().setRemotePacketRouter(null);
+        // Reset the session locator to use
+        XMPPServer.getInstance().setRemoteSessionLocator(null);
+        // Set the old serialization strategy was using before clustering was loaded
+        ExternalizableUtil.getInstance().setStrategy(serializationStrategy);
     }
 
     public Cache createCache(String name) {

src/plugins/hazelcast/changelog.html

@@ -44,7 +44,16 @@
 Hazelcast Clustering Plugin Changelog
 </h1>
 
-<p><b>1.0.2</b> -- January 8, 2013</p>
+<p><b>1.0.3</b> -- January 15, 2013</p>
+<p>Minor improvements for plugin installation and upgrade:</p>
+<ul>
+     <li>Modify timing for cluster startup/shutdown operations to improve plugin installation and upgrade.</li>
+     <li>Defer initialization of cluster event dispatcher thread until cluster is actually enabled.</li>
+     <li>Improved <code>readme.html</code> to describe upgrade procedures, DNS round-robin configuration, etc.</li>
+     <li>Upgraded Hazelcast to version 2.5.</li>
+</ul>
+
+<p><b>1.0.2</b> -- January 9, 2013</p>
 <p>This release addresses a number of issues and other feedback received via the 
 <a href="http://community.igniterealtime.org/message/224947#224947">Hazelcast announcement</a>
 posted in the Openfire community forum:</p>

src/plugins/hazelcast/plugin.xml

@@ -5,7 +5,7 @@
     <name>${plugin.name}</name>
     <description>${plugin.description}</description>
     <author>Tom Evans</author>
-    <version>1.0.2</version>
-    <date>01/09/2013</date>
+    <version>1.0.3</version>
+    <date>01/15/2013</date>
     <minServerVersion>3.7.2</minServerVersion>
 </plugin>

src/plugins/hazelcast/readme.html

@@ -12,9 +12,13 @@
             font-size : 0.8em;
         }
         H2 {
-             font-size : 10pt;
+             font-size : 11pt;
              font-weight : bold;
         }
+        H3 {
+             font-size : 10pt;
+             font-style : italic;
+        }
         A:hover {
             text-decoration : none;
         }
@@ -53,7 +57,7 @@
 <h2>Overview</h2>
 <p>
 The Hazelcast plugin adds support for running multiple redundant Openfire
-servers together in a cluster.  By running Openfire in a cluster, you can
+servers together in a cluster.  By running Openfire as a cluster, you can
 distribute the connection load among several servers, while also providing
 failover in the event that one of your servers fails. This plugin is a
 drop-in replacement for the original Openfire clustering plugin, using the
@@ -61,32 +65,98 @@ open source <a href="http://www.hazelcast.com">Hazelcast</a> data distribution
 framework in lieu of an expensive proprietary third-party product.
 </p>
 <p>
-The current Hazelcast release is version 2.4.1.
+The current Hazelcast release is version 2.5.
 </p>
 <h2>Installation</h2>
 <p>
-To install Hazelcast, simply drop the hazelcast.jar into $OPENFIRE_HOME/plugins along
-with any other plugins you may have installed. Note that Hazelcast and the original
-Openfire clustering plugin (clustering.jar) are mutually exclusive. You will need to
-remove the clustering plugin before installing Hazelcast into your Openfire instance.
+To create an Openfire cluster, you should have at least two Openfire servers,
+and each server must have the Hazelcast plugin installed. To install Hazelcast, 
+simply drop the hazelcast.jar into $OPENFIRE_HOME/plugins along with any other 
+plugins you may have installed. You may also use the Plugins page from the 
+admin console to install the plugin. Note that all servers in a given cluster 
+must be configured to share a single external database (not the Embedded DB). 
 </p>
 <p>
-To create an Openfire cluster, you will need at least two separate Openfire servers,
-and each server must have the Hazelcast plugin installed. By default, the servers
+By default during the Openfire startup/initialization process, the servers
 will discover each other by exchanging UDP (multicast) packets via a configurable
-IP address and port, but other initialization options are available if your network
-does not support multicast communication (see "Configuration" below).
+IP address and port. However, be advised that many other initialization options 
+are available and may be used if your network does not support multicast 
+communication (see <a href="#config">Configuration</a> below).
+</p>
+<p>After the Hazelcast plugin has been deployed to each of the servers, use the 
+radio button controls located on the Clustering page in the admin console to 
+activate/enable the cluster. You only need to enable clustering once; the change 
+will be propagated to the other servers automatically. After refreshing the 
+Clustering page you will be able to see all the servers that have successfully 
+joined the cluster.
+</p>
+<p>
+Note that Hazelcast and the earlier clustering plugins (clustering.jar and enterprise.jar) 
+are mutually exclusive. You will need to remove any existing older clustering plugin(s) 
+before installing Hazelcast into your Openfire server(s).
 </p>
 <p>
-In addition, you will need some form of load balancer to distribute the connection
-load among the members of your Openfire cluster. There are several commercial and
-open source alternatives for this, including the Apache web server (httpd) plus
-<a href="http://httpd.apache.org/docs/current/mod/mod_proxy_balancer.html">mod_proxy_balancer</a>
-(if you are using the HTTP/BOSH Openfire connector). Some popular solutions include the
+With your cluster up and running, you will now want some form of load balancer to 
+distribute the connection load among the members of your Openfire cluster. There 
+are several commercial and open source alternatives for this. For example, 
+if you are using the HTTP/BOSH Openfire connector to connect to Openfire, 
+the Apache web server (httpd) plus the corresponding proxy balancer module
+(<a href="http://httpd.apache.org/docs/current/mod/mod_proxy_balancer.html">mod_proxy_balancer</a>)
+could provide a workable solution. Some other popular options include the
 <a href="http://www.f5.com/products/big-ip/big-ip-local-traffic-manager/overview/">F5 LTM</a>
 (commercial) and <a href="http://haproxy.1wt.eu/">HAProxy</a> (open source), among
-<a href="http://en.wikipedia.org/wiki/Load_balancing_%28computing%29">many others</a>.
+<a href="http://en.wikipedia.org/wiki/Load_balancing_%28computing%29">many more</a>.
+<p>
+A simple round-robin DNS configuration can help distribute XMPP connections across multiple
+Openfire servers in a cluster. While popular as a lightweight and low-cost way to provide
+basic scalability, note that this approach is not considered adequate for true load balancing 
+nor does it provide high availability (HA) from a client perspective. If you are evaluating 
+these options, you can <a href="http://en.wikipedia.org/wiki/Round-robin_DNS">read more here</a>. 
+</p>
+<h2>Upgrading the Hazelcast Plugin</h2>
+<p>
+The process of upgrading the Hazelcast plugin requires a few additional steps when
+compared with a traditional plugin due to the cross-server dependencies within a running
+cluster. Practically speaking, all the members of the cluster need to be running the
+same version of the plugin to prevent various errors and data synchronization issues.
+</p>
+<h3>Option 1: Offline</h3>
+<p><b>NOTE:</b> This upgrade procedure is neat and tidy, but will incur a brief service outage.
 </p>
+<ol>
+<li>Shut down Openfire on all servers in the cluster.</li>
+<li>For the first server in the cluster, perform the following steps:</li>
+<ol type="a">
+<li>Remove the existing <code>plugins/hazelcast.jar</code></li>
+<li>Remove (recursively) the <code>plugins/hazelcast</code> directory</li>
+<li>Copy the updated <code>hazelcast.jar</code> into the <code>plugins</code> directory</li>
+<li>Restart Openfire to unpack and install the updated plugin</li>
+</ol>
+<li>Repeat these steps for the remaining servers in the cluster.</li>
+</ol>
+<h3>Option 2: Online</h3> 
+<p><b>NOTE:</b> Using this approach you should be able to continue servicing
+XMPP connections during the upgrade.
+</p>
+<ol>
+<li>Shut down Openfire on all servers <b>except one</b>.</li>
+<li>Using the Plugins page from the online server, remove the existing Hazelcast plugin.</li>
+<li>Upload the new Hazelcast plugin and confirm it is installed (refresh the page if necessary)</li>
+<li>Use the "Offline" steps above to upgrade and restart the remaining servers.</li>
+</ol>
+<h3>Option 3: Split-Brain</h3> 
+<p><b>NOTE:</b> Use this approach if you only have access to the Openfire console.
+Note however that users may not be able to communicate with each other during the upgrade 
+(if they are connected to different servers).
+</p>
+<ol>
+<li>From the Clustering page in the Openfire admin console, disable clustering. This will disable
+clustering for all members of the cluster.</li>
+<li>For each server, update the Hazelcast plugin using the Plugins page.</li>
+<li>After upgrading the plugin on all servers, use the Clustering page to enable clustering.
+This will activate clustering for all members of the cluster.</li>
+</ol>
+<a name="config" />
 <h2>Configuration</h2>
 <p>
 There are several configuration options built into the Hazelcast plugin
@@ -112,7 +182,7 @@ configuration file to be located outside the Openfire home directory.</li>
 </ol>
 </p>
 <p>
-The Hazelcast plugin uses the <a href="http://www.hazelcast.com/docs/2.3/manual/single_html/#Config">
+The Hazelcast plugin uses the <a href="http://www.hazelcast.com/docs/2.5/manual/single_html/#Config">
 XML configuration builder</a> to initialize the cluster from the XML file described above.
 By default the cluster members will attempt to discover each other via multicast at the
 following location:
@@ -137,7 +207,7 @@ following alternative:
 &lt;/join&gt;
 ...
 </pre>
-Please refer to the <a href="http://www.hazelcast.com/docs/2.3/manual/single_html/">
+Please refer to the <a href="http://www.hazelcast.com/docs/2.5/manual/single_html/">
 Hazelcast reference manual</a> for more information.
 </p>
 </body>

src/plugins/hazelcast/src/java/com/jivesoftware/openfire/HazelcastPlugin.java

@@ -22,11 +22,6 @@ package com.jivesoftware.openfire;
 
 import java.io.File;
 import java.io.FileFilter;
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.util.Date;
-import java.util.Locale;
-import java.util.Map;
 import java.util.TimerTask;
 
 import org.jivesoftware.openfire.XMPPServer;
@@ -34,18 +29,10 @@ import org.jivesoftware.openfire.cluster.ClusterManager;
 import org.jivesoftware.openfire.container.Plugin;
 import org.jivesoftware.openfire.container.PluginManager;
 import org.jivesoftware.util.JiveGlobals;
-import org.jivesoftware.util.PropertyEventDispatcher;
-import org.jivesoftware.util.PropertyEventListener;
 import org.jivesoftware.util.TaskEngine;
-import org.jivesoftware.util.cache.ExternalizableUtil;
-import org.jivesoftware.util.cache.ExternalizableUtilStrategy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.jivesoftware.openfire.session.RemoteSessionLocator;
-import com.jivesoftware.util.cache.ClusterExternalizableUtil;
-import com.jivesoftware.util.cluster.ClusterPacketRouter;
-
 /**
  * Hazelcast clustering plugin. This implementation is based upon
  * (and borrows heavily from) the original Openfire clustering plugin.
@@ -54,19 +41,13 @@ import com.jivesoftware.util.cluster.ClusterPacketRouter;
  * @author Tom Evans
  * @author Matt Tucker
  */
-public class HazelcastPlugin extends TimerTask implements Plugin, PropertyEventListener {
+public class HazelcastPlugin extends TimerTask implements Plugin {
 
     private static Logger logger = LoggerFactory.getLogger(HazelcastPlugin.class);
 
     private static final long CLUSTER_STARTUP_DELAY_TIME = 
     		JiveGlobals.getLongProperty("hazelcast.startup.delay.seconds", 5);
     
-    /**
-     * Keep serialization strategy the server was using before we set our strategy. We will
-     * restore old strategy when plugin is unloaded.
-     */
-    private ExternalizableUtilStrategy serializationStrategy;
-
     public void initializePlugin(PluginManager manager, File pluginDirectory) {
     	// start cluster using a separate thread after a short delay
     	// this will allow other plugins to initialize during startup
@@ -91,78 +72,14 @@ public class HazelcastPlugin extends TimerTask implements Plugin, PropertyEventL
             logger.warn("Conflicting clustering plugins found; remove Coherence and/or Enterprise jar files");
             throw new IllegalStateException("Clustering plugin configuration conflict (Coherence)");
         }
-
-        // List for clustering setting events (e.g. enabled/disabled)
-        PropertyEventDispatcher.addListener(this);
-
-        if (ClusterManager.isClusteringEnabled()) {
-            initForClustering();
-
-            // Start up or join the cluster and initialize caches
-            ClusterManager.startup();
-        }
+        ClusterManager.startup();
 	}
 
-	private void initForClustering() {
-        // Set the serialization strategy to use for transmitting objects between node clusters
-        serializationStrategy = ExternalizableUtil.getInstance().getStrategy();
-        ExternalizableUtil.getInstance().setStrategy(new ClusterExternalizableUtil());
-        // Set session locator to use when in a cluster
-        XMPPServer.getInstance().setRemoteSessionLocator(new RemoteSessionLocator());
-        // Set packet router to use to deliver packets to remote cluster nodes
-        XMPPServer.getInstance().getRoutingTable().setRemotePacketRouter(new ClusterPacketRouter());
-    }
-
-    /**
-     * Returns the date when this release of Openfire clustering plugin was released.
-     *
-     * @return the date when this release of Openfire clustering plugin was released.
-     */
-    public static Date getReleaseDate() {
-        try {
-            // @DATE@ should be replaced with a date with the following format: Jan 31, 2007
-            // Automatically set by ANT build tasks
-            return DateFormat.getDateInstance(DateFormat.MEDIUM, Locale.US).parse("@DATE@");
-        }
-        catch (ParseException e) {
-            logger.error("Error parsing date", e);
-            return null;
-        }
-    }
-
     public void destroyPlugin() {
         // Shutdown is initiated by XMPPServer before unloading plugins
-    	// ClusterManager.shutdown();
-
-        // Set the old serialization strategy was using before clustering was loaded
-        ExternalizableUtil.getInstance().setStrategy(serializationStrategy);
-
-        // Stop listing for clustering setting events (e.g. enabled/disabled)
-        PropertyEventDispatcher.removeListener(this);
-    }
-
-    public void propertySet(String property, Map<String, Object> params) {
-        // Ignore
-    }
-
-    public void propertyDeleted(String property, Map<String, Object> params) {
-        // Ignore
-    }
-
-    public void xmlPropertySet(String property, Map<String, Object> params) {
-        if (ClusterManager.CLUSTER_PROPERTY_NAME.equals(property)) {
-            if (Boolean.parseBoolean((String) params.get("value"))) {
-                // Clustering was enabled
-                initForClustering();
-            }
-            else {
-                // Clustering was disabled
-            }
-        }
-    }
-
-    public void xmlPropertyDeleted(String property, Map<String, Object> params) {
-        // Do nothing
+    	if (!XMPPServer.getInstance().isShuttingDown()) {
+    		ClusterManager.shutdown();
+    	}
     }
 
 }

src/plugins/hazelcast/src/java/com/jivesoftware/util/cache/ClusterListener.java

@@ -585,8 +585,6 @@ public class ClusterListener implements MembershipListener, LifecycleListener {
         // Clean up all traces. This will set all remote sessions as unavailable
         List<NodeID> nodeIDs = new ArrayList<NodeID>(nodeSessions.keySet());
 
-        // Revert cluster caches to local caches
-        CacheFactory.leftCluster();
         // Trigger event. Wait until the listeners have processed the event. Caches will be populated
         // again with local content.
         ClusterManager.fireLeftCluster();

src/plugins/hazelcast/src/java/com/jivesoftware/util/cache/ClusteredCacheFactory.java

@@ -43,6 +43,8 @@ import org.jivesoftware.util.cache.Cache;
 import org.jivesoftware.util.cache.CacheFactoryStrategy;
 import org.jivesoftware.util.cache.CacheWrapper;
 import org.jivesoftware.util.cache.ClusterTask;
+import org.jivesoftware.util.cache.ExternalizableUtil;
+import org.jivesoftware.util.cache.ExternalizableUtilStrategy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -54,6 +56,8 @@ import com.hazelcast.core.Hazelcast;
 import com.hazelcast.core.HazelcastInstance;
 import com.hazelcast.core.Member;
 import com.hazelcast.core.MultiTask;
+import com.jivesoftware.openfire.session.RemoteSessionLocator;
+import com.jivesoftware.util.cluster.ClusterPacketRouter;
 import com.jivesoftware.util.cluster.HazelcastClusterNodeInfo;
 
 /**
@@ -75,6 +79,12 @@ public class ClusteredCacheFactory implements CacheFactoryStrategy {
 
 	private static Logger logger = LoggerFactory.getLogger(ClusteredCacheFactory.class);
 
+    /**
+     * Keep serialization strategy the server was using before we set our strategy. We will
+     * restore old strategy when plugin is unloaded.
+     */
+    private ExternalizableUtilStrategy serializationStrategy;
+
     /**
      * Storage for cache statistics
      */
@@ -91,6 +101,15 @@ public class ClusteredCacheFactory implements CacheFactoryStrategy {
 
     public boolean startCluster() {
         state = State.starting;
+        
+        // Set the serialization strategy to use for transmitting objects between node clusters
+        serializationStrategy = ExternalizableUtil.getInstance().getStrategy();
+        ExternalizableUtil.getInstance().setStrategy(new ClusterExternalizableUtil());
+        // Set session locator to use when in a cluster
+        XMPPServer.getInstance().setRemoteSessionLocator(new RemoteSessionLocator());
+        // Set packet router to use to deliver packets to remote cluster nodes
+        XMPPServer.getInstance().getRoutingTable().setRemotePacketRouter(new ClusterPacketRouter());
+
         ClassLoader oldLoader = null;
         // Store previous class loader (in case we change it)
         oldLoader = Thread.currentThread().getContextClassLoader();
@@ -152,6 +171,13 @@ public class ClusteredCacheFactory implements CacheFactoryStrategy {
         }
         // Reset the node ID
         XMPPServer.getInstance().setNodeID(null);
+
+        // Reset packet router to use to deliver packets to remote cluster nodes
+        XMPPServer.getInstance().getRoutingTable().setRemotePacketRouter(null);
+        // Reset the session locator to use
+        XMPPServer.getInstance().setRemoteSessionLocator(null);
+        // Set the old serialization strategy was using before clustering was loaded
+        ExternalizableUtil.getInstance().setStrategy(serializationStrategy);
     }
 
     public Cache createCache(String name) {