Jitsi Videobridge plugin: refresh with jitsi.org. More work on video...

Jitsi Videobridge plugin: refresh with jitsi.org. More work on video recording. Still not yet working.

src/plugins/jitsivideobridge/src/java/org/ebml/matroska/MatroskaFileWriter.java

@@ -37,6 +37,7 @@ public class MatroskaFileWriter
 {
   protected DataWriter ioDW;
   private MatroskaCluster clusterElem = null;
+  private MasterElement segmentElem = null;
   private long clusterTimecode;
   protected MatroskaDocType doc = new MatroskaDocType();
 
@@ -88,13 +89,23 @@ public class MatroskaFileWriter
 
   public void writeSegmentHeader()
   {
-    //MatroskaSegment segmentElem = (MatroskaSegment)doc.createElement(MatroskaDocType.Segment_Id);
-    //segmentElem.setSize(-1);
-    //segmentElem.setUnknownSize(false);
-    //segmentElem.writeHeaderData(ioDW);
 
-    MasterElement ebmlHeaderElem = (MasterElement)doc.createElement(MatroskaDocType.Segment_Id);
-    ebmlHeaderElem.writeElement(ioDW);
+    segmentElem = (MasterElement)doc.createElement(MatroskaDocType.Segment_Id);
+
+    MasterElement ebmlSeekHeadElem = (MasterElement)doc.createElement(MatroskaDocType.SeekHead_Id);
+
+    MasterElement ebmlSeekEntryElem1 = (MasterElement)doc.createElement(MatroskaDocType.SeekEntry_Id);
+    BinaryElement seekID1 = (BinaryElement)doc.createElement(MatroskaDocType.SeekID_Id);
+    seekID1.setData(MatroskaDocType.Tracks_Id);
+    UnsignedIntegerElement SeekPosition1 = (UnsignedIntegerElement)doc.createElement(MatroskaDocType.SeekPosition_Id);
+    SeekPosition1.setValue(0);
+    ebmlSeekEntryElem1.addChildElement(seekID1);
+    ebmlSeekEntryElem1.addChildElement(SeekPosition1);
+
+    ebmlSeekHeadElem.addChildElement(ebmlSeekEntryElem1);
+
+    segmentElem.addChildElement(ebmlSeekHeadElem);
+
   }
 
   public void writeSegmentInfo()
@@ -115,7 +126,8 @@ public class MatroskaFileWriter
     timecodescaleElem.setValue(TimecodeScale);
 
     FloatElement durationElem = (FloatElement)doc.createElement(MatroskaDocType.Duration_Id);
-    durationElem.setValue(Duration * 1000.0);
+    //durationElem.setValue(Duration * 1000.0);
+    durationElem.setValue(0);
 
     //segmentInfoElem.addChildElement(dateElem);
     segmentInfoElem.addChildElement(timecodescaleElem);
@@ -123,7 +135,9 @@ public class MatroskaFileWriter
     segmentInfoElem.addChildElement(writingAppElem);
     segmentInfoElem.addChildElement(durationElem);
 
-    segmentInfoElem.writeElement(ioDW);
+
+    //segmentInfoElem.writeElement(ioDW);
+    segmentElem.addChildElement(segmentInfoElem);
   }
 
   public void writeTracks()
@@ -227,7 +241,8 @@ public class MatroskaFileWriter
       tracksElem.addChildElement(trackEntryElem);
     }
 
-    tracksElem.writeElement(ioDW);
+    segmentElem.addChildElement(tracksElem);
+    segmentElem.writeElement(ioDW);
   }
 
   public void startCluster(long clusterTimecode)

src/plugins/jitsivideobridge/src/java/org/jitsi/impl/neomedia/RTPConnectorInputStream.java

@@ -64,6 +64,7 @@ public abstract class RTPConnectorInputStream
     protected boolean closed;
 
     public Participant videoRecorder;
+    public Participant audioScanner;
 
     /**
      * The <tt>DatagramPacketFilter</tt>s which allow dropping
@@ -412,6 +413,9 @@ public abstract class RTPConnectorInputStream
                         if (buffer != null)
                             buffer.setFlags(pkt.getFlags());
 
+                        if (videoRecorder != null) videoRecorder.recordData(pkt);
+                        if (audioScanner != null) audioScanner.scanData(pkt);
+
                     }
                 }
             }
@@ -551,9 +555,6 @@ public abstract class RTPConnectorInputStream
                                 poolRawPacket(oldPkt);
                             }
 
-
-                        	if (videoRecorder != null) videoRecorder.recordData(pkt);
-
                             if ((transferHandler != null) && !closed)
                             {
                                 try

src/plugins/jitsivideobridge/src/java/org/jitsi/videobridge/openfire/PluginImpl.java

@@ -13,7 +13,7 @@ import java.lang.reflect.*;
 import java.net.*;
 import java.util.*;
 import java.util.jar.*;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.*;
 import java.security.cert.Certificate;
 
 import javax.media.*;
@@ -156,19 +156,19 @@ public class PluginImpl  implements Plugin, PropertyEventListener
      * The Jabber component which has been added to {@link #componentManager}
      * i.e. Openfire.
      */
-    private Component component;
+    private Component component = null;
 
     /**
      * The <tt>ComponentManager</tt> to which the {@link #component} of this
      * <tt>Plugin</tt> has been added.
      */
-    private ComponentManager componentManager;
+    private ComponentManager componentManager = null;
 
     /**
      * The subdomain of the address of {@link #component} with which it has been
      * added to {@link #componentManager}.
      */
-    private String subdomain;
+    private String subdomain = null;
 
     /**
      * RAYO IQ Handler for colibri
@@ -202,11 +202,18 @@ public class PluginImpl  implements Plugin, PropertyEventListener
      */
     private static ComponentImpl componentImpl;
 
+    /**
+	 *
+     */
+    private ExecutorService executorService;
+
 
     public void destroyPlugin()
     {
         PropertyEventDispatcher.removeListener(this);
 
+        executorService.shutdown();
+
         if ((componentManager != null) && (subdomain != null))
         {
             try
@@ -234,7 +241,7 @@ public class PluginImpl  implements Plugin, PropertyEventListener
      * located
      * @see Plugin#initializePlugin(PluginManager, File)
      */
-    public void initializePlugin(PluginManager manager, File pluginDirectory)
+    public void initializePlugin(final PluginManager manager, final File pluginDirectory)
     {
         PropertyEventDispatcher.addListener(this);
 
@@ -242,94 +249,89 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 		System.setProperty("net.java.sip.communicator.SC_HOME_DIR_NAME", ".");
 		System.setProperty("org.jitsi.impl.neomedia.transform.srtp.SRTPCryptoContext.checkReplay", JiveGlobals.getProperty(CHECKREPLAY_PROPERTY_NAME, "false"));
 
-		// start video conference web application
+		executorService = Executors.newFixedThreadPool(2);
 
-		try {
-			String appName = JiveGlobals.getProperty(VIDEO_CONFERENCE_PROPERTY_NAME, "jitsi");
-			Log.info("Initialize Web App " + appName);
+		// start video conference web application
+		executorService.execute(new Runnable()
+		{
+			public void run()
+			{
+				try {
+					String appName = JiveGlobals.getProperty(VIDEO_CONFERENCE_PROPERTY_NAME, "jitsi");
+					Log.info("Initialize Web App " + appName);
 
-			ContextHandlerCollection contexts = HttpBindManager.getInstance().getContexts();
-			WebAppContext context = new WebAppContext(contexts, pluginDirectory.getPath(), "/" + appName);
-			context.setWelcomeFiles(new String[]{"index.html"});
+					ContextHandlerCollection contexts = HttpBindManager.getInstance().getContexts();
+					WebAppContext context = new WebAppContext(contexts, pluginDirectory.getPath(), "/" + appName);
+					context.setWelcomeFiles(new String[]{"index.html"});
 
-			String username = JiveGlobals.getProperty(USERNAME_PROPERTY_NAME, null);
-			String password = JiveGlobals.getProperty(PASSWORD_PROPERTY_NAME, "jitsi");
+					String username = JiveGlobals.getProperty(USERNAME_PROPERTY_NAME, null);
+					String password = JiveGlobals.getProperty(PASSWORD_PROPERTY_NAME, "jitsi");
 
-			if (username != null && "".equals(username) == false)
-			{
-				context.setSecurityHandler(basicAuth(username, password, "Videobridge"));
-			}
+					if (username != null && "".equals(username) == false)
+					{
+						context.setSecurityHandler(basicAuth(username, password, "Videobridge"));
+					}
 
-			createIQHandlers();
+					createIQHandlers();
 
-			Properties properties = new Properties();
-			String hostName = XMPPServer.getInstance().getServerInfo().getHostname();
-			String logDir = pluginDirectory.getAbsolutePath() + File.separator + ".." + File.separator + ".." + File.separator + "logs" + File.separator;
-			String port = JiveGlobals.getProperty(SIP_PORT_PROPERTY_NAME, "5070");
+					Properties properties = new Properties();
+					String hostName = XMPPServer.getInstance().getServerInfo().getHostname();
+					String logDir = pluginDirectory.getAbsolutePath() + File.separator + ".." + File.separator + ".." + File.separator + "logs" + File.separator;
+					String port = JiveGlobals.getProperty(SIP_PORT_PROPERTY_NAME, "5070");
 
-			properties.setProperty("com.voxbone.kelpie.hostname", hostName);
-			properties.setProperty("com.voxbone.kelpie.ip", hostName);
-			properties.setProperty("com.voxbone.kelpie.sip_port", port);
+					properties.setProperty("com.voxbone.kelpie.hostname", hostName);
+					properties.setProperty("com.voxbone.kelpie.ip", hostName);
+					properties.setProperty("com.voxbone.kelpie.sip_port", port);
 
-			properties.setProperty("javax.sip.IP_ADDRESS", hostName);
+					properties.setProperty("javax.sip.IP_ADDRESS", hostName);
 
-			properties.setProperty("gov.nist.javax.sip.TRACE_LEVEL", "99");
-			properties.setProperty("gov.nist.javax.sip.SERVER_LOG", logDir + "sip_server.log");
-			properties.setProperty("gov.nist.javax.sip.DEBUG_LOG", logDir + "sip_debug.log");
+					properties.setProperty("gov.nist.javax.sip.TRACE_LEVEL", "99");
+					properties.setProperty("gov.nist.javax.sip.SERVER_LOG", logDir + "sip_server.log");
+					properties.setProperty("gov.nist.javax.sip.DEBUG_LOG", logDir + "sip_debug.log");
 
-			new SipService(properties);
+					new SipService(properties);
 
-			Log.info("Initialize SIP Stack at " + hostName + ":" + port);
+					Log.info("Initialize SIP Stack at " + hostName + ":" + port);
 
-		}
-		catch(Exception e) {
-			Log.error( "Jitsi Videobridge web app initialize error", e);
-		}
+				}
+				catch(Exception e) {
+					Log.error( "Jitsi Videobridge web app initialize error", e);
+				}
 
 
-        // Let's check for custom configuration
-        String maxVal = JiveGlobals.getProperty(MAX_PORT_NUMBER_PROPERTY_NAME);
-        String minVal = JiveGlobals.getProperty(MIN_PORT_NUMBER_PROPERTY_NAME);
+				// Let's check for custom configuration
+				String maxVal = JiveGlobals.getProperty(MAX_PORT_NUMBER_PROPERTY_NAME);
+				String minVal = JiveGlobals.getProperty(MIN_PORT_NUMBER_PROPERTY_NAME);
 
-        if(maxVal != null)
-            setIntProperty(
-                DefaultStreamConnector.MAX_PORT_NUMBER_PROPERTY_NAME,
-                maxVal);
-        if(minVal != null)
-            setIntProperty(
-                DefaultStreamConnector.MIN_PORT_NUMBER_PROPERTY_NAME,
-                minVal);
+				if(maxVal != null)
+					setIntProperty(
+						DefaultStreamConnector.MAX_PORT_NUMBER_PROPERTY_NAME,
+						maxVal);
+				if(minVal != null)
+					setIntProperty(
+						DefaultStreamConnector.MIN_PORT_NUMBER_PROPERTY_NAME,
+						minVal);
 
-        checkNatives();
-        checkRecordingFolder(pluginDirectory);
+				checkNatives();
+				checkRecordingFolder(pluginDirectory);
 
-        ComponentManager componentManager = ComponentManagerFactory.getComponentManager();
-        String subdomain = ComponentImpl.SUBDOMAIN;
-        Component component = new ComponentImpl();
-        boolean added = false;
+				componentManager = ComponentManagerFactory.getComponentManager();
+				subdomain = ComponentImpl.SUBDOMAIN;
+				component = new ComponentImpl();
+				boolean added = false;
 
-        try
-        {
-            componentManager.addComponent(subdomain, component);
-            added = true;
-            componentImpl = (ComponentImpl) component;
-        }
-        catch (ComponentException ce)
-        {
-            ce.printStackTrace(System.err);
-        }
-        if (added)
-        {
-            this.componentManager = componentManager;
-            this.subdomain = subdomain;
-            this.component = component;
-        }
-        else
-        {
-            this.componentManager = null;
-            this.subdomain = null;
-            this.component = null;
-        }
+				try
+				{
+					componentManager.addComponent(subdomain, component);
+					added = true;
+					componentImpl = (ComponentImpl) component;
+				}
+				catch (ComponentException ce)
+				{
+					ce.printStackTrace(System.err);
+				}
+			}
+		});
     }
 
     /**
@@ -1039,7 +1041,12 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 		 *
 		 *
 		 */
-		public MediaStream mediaStream = null;
+		public MediaStream videoStream = null;
+		/**
+		 *
+		 *
+		 */
+		public MediaStream audioStream = null;
 		/**
 		 *
 		 *
@@ -1072,7 +1079,33 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 		 *
 		 *
 		 */
-		private PropertyChangeListener streamPropertyChangeListener = new PropertyChangeListener()
+		private PropertyChangeListener audioStreamPropertyChangeListener = new PropertyChangeListener()
+		{
+			public void propertyChange(PropertyChangeEvent ev)
+			{
+				String propertyName = ev.getPropertyName();
+				String prefix = MediaStreamImpl.class.getName() + ".rtpConnector.";
+
+				if (propertyName.startsWith(prefix))
+				{
+					Object newValue = ev.getNewValue();
+
+					if (newValue instanceof RTPConnectorInputStream)
+					{
+						String rtpConnectorPropertyName = propertyName.substring(prefix.length());
+
+						if (rtpConnectorPropertyName.equals("dataInputStream"))
+						{
+							Log.info("PropertyChangeListener " + rtpConnectorPropertyName);
+
+							((RTPConnectorInputStream) newValue).audioScanner = me;
+						}
+					}
+				}
+			}
+		};
+
+		private PropertyChangeListener videoStreamPropertyChangeListener = new PropertyChangeListener()
 		{
 			public void propertyChange(PropertyChangeEvent ev)
 			{
@@ -1101,57 +1134,106 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 		 *
 		 *
 		 */
-		public void recordData(RawPacket packet)
+		synchronized public void scanData(final RawPacket packet)
 		{
-			//if (snapshot < 1) Log.info("transferData " + packet.getPayloadLength() + " " + packet.getHeaderLength()  + " " + packet.getExtensionLength());
+			//if (snapshot < 10) Log.info("scanData " + packet.getPayloadLength() + " " + packet.getHeaderLength()  + " " + packet.getExtensionLength());
 
-			try {
+			if (packet != null)
+			{
+				final byte[] rtp = packet.getPayload();
+				final int sequenceNumber = packet.getSequenceNumber();
+				final boolean isMarked = packet.isPacketMarked();
+				final long timestamp = packet.getTimestamp();
+				final byte payloadType = packet.getPayloadType();
 
-				if (packet != null)
+				executorService.execute(new Runnable()
 				{
-					byte[] rtp = packet.getPayload();
+					public void run()
+					{
+						try {
 
-					if(!_sequenceNumberingViolated && _lastSequenceNumber.intValue() > -1 && Vp8Packet.getSequenceNumberDelta(packet.getSequenceNumber(), _lastSequenceNumber).intValue() > 1)
-						_sequenceNumberingViolated = true;
+							//Log.info("Audio packet type " + payloadType);
 
-					_lastSequenceNumber = packet.getSequenceNumber();
-					Vp8Packet packet2 = Vp8Packet.parseBytes(rtp);
 
-					if(packet2 == null)	return;
+						} catch (Exception e) {
+							Log.error("Error scanning audio", e);
+						}
+					}
+				});
 
-					_accumulator.add(packet2);
-					byte encodedFrame[] = null;
+			} else {
+				Log.error("scan audio cannot parse packet data " + packet);
+			}
+		}
 
-					if (packet.isPacketMarked())
-					{
-						encodedFrame = Vp8Packet.depacketize(_accumulator.getPackets());
-						boolean isKeyframe = encodedFrame != null && encodedFrame.length > 0 && (encodedFrame[0] & 1) == 0;
+		/**
+		 *
+		 *
+		 */
+		synchronized public void recordData(final RawPacket packet)
+		{
+			//if (snapshot < 10) Log.info("transferData " + packet.getPayloadLength() + " " + packet.getHeaderLength()  + " " + packet.getExtensionLength());
+
+			if (packet != null)
+			{
+				final byte[] rtp = packet.getPayload();
+				final int sequenceNumber = packet.getSequenceNumber();
+				final boolean isMarked = packet.isPacketMarked();
+				final long timestamp = packet.getTimestamp();
+
+				//executorService.execute(new Runnable()
+				//{
+				//	public void run()
+				//	{
+						try {
 
-						if(_sequenceNumberingViolated && isKeyframe)
-							_sequenceNumberingViolated = false;
+							synchronized (_accumulator)
+							{
+								if(!_sequenceNumberingViolated && _lastSequenceNumber.intValue() > -1 && Vp8Packet.getSequenceNumberDelta(sequenceNumber, _lastSequenceNumber).intValue() > 1)
+									_sequenceNumberingViolated = true;
 
-						_accumulator.reset();
+								_lastSequenceNumber = sequenceNumber;
+								Vp8Packet packet2 = Vp8Packet.parseBytes(rtp);
 
-						if (recorder != null && encodedFrame != null && _sequenceNumberingViolated == false)
-						{
-							byte[] full = Arrays.copyOf(encodedFrame, encodedFrame.length);
+								if(packet2 == null)	return;
 
-							recorder.write(full, 0, full.length, isKeyframe, packet.getTimestamp());
+								_accumulator.add(packet2);
+								byte encodedFrame[] = null;
 
-							if (isKeyframe && snapshot < 1)
-							{
-								Log.info("recordData " + " " + packet.getPayloadType() + " " + full + " " + packet.getSequenceNumber() + " " + packet.getTimestamp());
-								recorder.writeWebPImage(full, 0, full.length, packet.getTimestamp());
-								snapshot++;
+								if (isMarked)
+								{
+									encodedFrame = Vp8Packet.depacketize(_accumulator.getPackets());
+									boolean isKeyframe = encodedFrame != null && encodedFrame.length > 0 && (encodedFrame[0] & 1) == 0;
+
+									if(_sequenceNumberingViolated && isKeyframe)
+										_sequenceNumberingViolated = false;
+
+									_accumulator.reset();
+
+									if (recorder != null && encodedFrame != null && _sequenceNumberingViolated == false)
+									{
+										byte[] full = Arrays.copyOf(encodedFrame, encodedFrame.length);
+
+										recorder.write(full, 0, full.length, isKeyframe, timestamp);
+
+										if (isKeyframe && snapshot < 1)
+										{
+											Log.info("recordData " + full + " " + sequenceNumber + " " + timestamp);
+											recorder.writeWebPImage(full, 0, full.length, timestamp);
+											snapshot++;
+										}
+									}
+								}
 							}
+
+						} catch (Exception e) {
+							Log.error("Error writing video recording" , e);
 						}
-					}
-				} else {
-					Log.error("record video cannot parse packet data " + packet);
-				}
+				//	}
+				//});
 
-			} catch (Exception e) {
-				Log.error("Error writing video recording" , e);
+			} else {
+				Log.error("record video cannot parse packet data " + packet);
 			}
 		}
 		/**
@@ -1197,14 +1279,28 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 		 *
 		 *
 		 */
-		public void addMediaStream(MediaStream mediaStream)
+		public void setAudioStream(MediaStream mediaStream)
 		{
 			boolean recordMedia = "true".equals(JiveGlobals.getProperty(RECORD_PROPERTY_NAME, "false"));
 
 			if (recordMedia)
 			{
-				this.mediaStream = mediaStream;
-				mediaStream.addPropertyChangeListener(streamPropertyChangeListener);
+				audioStream = mediaStream;
+				audioStream.addPropertyChangeListener(audioStreamPropertyChangeListener);
+			}
+		}
+		/**
+		 *
+		 *
+		 */
+		public void setVideoStream(MediaStream mediaStream)
+		{
+			boolean recordMedia = "true".equals(JiveGlobals.getProperty(RECORD_PROPERTY_NAME, "false"));
+
+			if (recordMedia)
+			{
+				videoStream = mediaStream;
+				videoStream.addPropertyChangeListener(videoStreamPropertyChangeListener);
 
 				String recordingPath = JiveGlobals.getHomeDirectory() + File.separator + "resources" + File.separator + "spank" + File.separator + "rayo"  + File.separator + "video_recordings";
 				String fileName = "video-" + focusName + "-" + nickname + "-" + System.currentTimeMillis() + ".webm";
@@ -1224,9 +1320,14 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 		 */
 		public void removeMediaStream()
 		{
-			if (this.mediaStream != null)
+			if (audioStream != null)
+			{
+				audioStream.removePropertyChangeListener(audioStreamPropertyChangeListener);
+			}
+
+			if (videoStream != null)
 			{
-				mediaStream.removePropertyChangeListener(streamPropertyChangeListener);
+				videoStream.removePropertyChangeListener(videoStreamPropertyChangeListener);
 
 				if (recorder != null)
 				{
@@ -1392,8 +1493,7 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 
 							if (videoChannel != null)
 							{
-								// webm file creation not working yet
-								participant.addMediaStream(videoChannel.getMediaStream());
+								participant.setVideoStream(videoChannel.getMediaStream());
 							}
 						}
 				}
@@ -1401,6 +1501,19 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 				if ("audio".equals(content.attributeValue("name")))
 				{
 						participant.audioChannelId = channel.attributeValue("id");
+
+						String focusJid = XMPPServer.getInstance().createJID(focusName, focusName).toString();
+						Content vbContent = getVideoBridge().getConference(focusId, focusJid).getOrCreateContent("audio");
+
+						if (vbContent != null)
+						{
+							Channel audioChannel = vbContent.getChannel(participant.audioChannelId);
+
+							if (audioChannel != null)
+							{
+								participant.setAudioStream(audioChannel.getMediaStream());
+							}
+						}
 				}
 			}
 		}
@@ -1496,7 +1609,7 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 
 				router.route(iq);
 
-				if (participant.mediaStream != null)
+				if (participant.audioStream != null || participant.videoStream != null)
 				{
                 	participant.removeMediaStream();
 				}

src/plugins/jitsivideobridge/src/java/org/jitsi/videobridge/openfire/Recorder.java

@@ -427,6 +427,7 @@ public class Recorder extends Thread
 	    	{
 				if (recordWebm)
 				{
+           			Log.info("writeData " + d.timestamp);
 					long duration = 0;
 /*
 					if (d.keyframe || lastTimecode == 0)
@@ -487,7 +488,7 @@ public class Recorder extends Thread
 			{
 				if (recordWebm)
 				{
-					mFW.endCluster();
+					//mFW.endCluster();
 					iFW.close();
 
 				} else {