Jitsi Videobridge - Implemented conference audio recording, not working yet

git-svn-id: http://svn.igniterealtime.org/svn/repos/openfire/trunk@13854 b35dd754-fafc-0310-a699-88a17e54d16e

src/plugins/jitsivideobridge/src/java/com/rayo/core/verb/ColibriCommand.java

-package com.rayo.core.verb;
-
-import org.dom4j.Element;
-
-public class ColibriCommand extends AbstractVerbCommand {
-
-    private String videobridge;
-   	private String localRTPPort;
-   	private String localRTCPPort;
-   	private String remoteRTPPort;
-   	private String remoteRTCPPort;
-   	private String codec;
-
-	public ColibriCommand(String videobridge, String localRTPPort, String localRTCPPort, String remoteRTPPort, String remoteRTCPPort, String codec)
-	{
-		this.videobridge = videobridge;
-		this.localRTPPort = localRTPPort;
-		this.localRTCPPort = localRTCPPort;
-		this.remoteRTPPort = remoteRTPPort;
-		this.remoteRTCPPort = remoteRTCPPort;
-		this.codec = codec;
-	}
-
-	public String getVideobridge()
-	{
-		return this.videobridge;
-	}
-
-	public String getLocalRTPPort()
-	{
-		return this.localRTPPort;
-	}
-
-	public String getLocalRTCPPort()
-	{
-		return this.localRTCPPort;
-	}
-
-	public String getRemoteRTPPort()
-	{
-		return this.remoteRTPPort;
-	}
-
-	public String getRemoteRTCPPort()
-	{
-		return this.remoteRTCPPort;
-	}
-
-	public String getCodec()
-	{
-		return this.codec;
-	}
-}

src/plugins/jitsivideobridge/src/java/com/rayo/core/xml/providers/ColibriProvider.java

@@ -45,24 +45,13 @@ public class ColibriProvider extends BaseProvider {
     private Object buildColibriCommand(Element element) throws URISyntaxException
     {
 		String action = element.attributeValue("action");
-		String videobridge = element.attributeValue("videobridge");
-		String localRTPPort = element.attributeValue("localrtpport");
-		String localRTCPPort = element.attributeValue("localrtcpport");
-		String remoteRTPPort = element.attributeValue("remotertpport");
-		String remoteRTCPPort = element.attributeValue("remotertcpport");
-		String codec = element.attributeValue("codec");
-
 		Object command = null;
 
-		if ("register".equals(action))
-		{
+		if ("register".equals(action)) {
 			command = new RegisterCommand();
 
 		} else if ("unregister".equals(action)) {
 			command = new UnRegisterCommand();
-
-		} else if ("bridge".equals(action)) {
- 			command = new ColibriCommand(videobridge, localRTPPort, localRTCPPort, remoteRTPPort, remoteRTCPPort, codec);
 		}
 
         return command;

src/plugins/jitsivideobridge/src/java/org/jitsi/videobridge/AudioSilenceCaptureDevice.java

+/*
+ * Jitsi Videobridge, OpenSource video conferencing.
+ *
+ * Distributable under LGPL license.
+ * See terms of license at gnu.org.
+ */
+package org.jitsi.videobridge;
+
+import java.io.*;
+import java.util.*;
+
+import javax.media.*;
+import javax.media.control.*;
+import javax.media.format.*;
+import javax.media.protocol.*;
+
+import org.jitsi.impl.neomedia.codec.*;
+import org.jitsi.impl.neomedia.jmfext.media.protocol.*;
+import org.jitsi.impl.neomedia.jmfext.media.renderer.audio.*;
+
+/**
+ * Implements a <tt>CaptureDevice</tt> which provides silence in the form of
+ * audio media.
+ *
+ * @author Lyubomir Marinov
+ */
+public class AudioSilenceCaptureDevice
+    extends AbstractPushBufferCaptureDevice
+{
+    /**
+     * The compile-time flag which determines whether
+     * <tt>AudioSilenceCaptureDevice</tt> and, more specifically,
+     * <tt>AudioSilenceStream</tt> are to be used by <tt>AudioMixer</tt> for the
+     * mere purposes of ticking the clock which makes <tt>AudioMixer</tt> read
+     * media from its inputs, mix it, and write it to its outputs. The preferred
+     * value is <tt>true</tt> because it causes the <tt>AudioMixer</tt> to not
+     * push media unless at least one <tt>Channel</tt> is receiving actual
+     * media.
+     */
+    private static final boolean CLOCK_ONLY = true;
+
+    /**
+     * The interval of time in milliseconds between two consecutive ticks of the
+     * clock used by <tt>AudioSilenceCaptureDevice</tt> and, more specifically,
+     * <tt>AudioSilenceStream</tt>.
+     */
+    private static final long CLOCK_TICK_INTERVAL = 20;
+
+    /**
+     * The list of <tt>Format</tt>s supported by the
+     * <tt>AudioSilenceCaptureDevice</tt> instances.
+     */
+    private static final Format[] SUPPORTED_FORMATS
+        = new Format[]
+                {
+                    new AudioFormat(
+                            AudioFormat.LINEAR,
+                            48000,
+                            16,
+                            1,
+                            AudioFormat.LITTLE_ENDIAN,
+                            AudioFormat.SIGNED,
+                            Format.NOT_SPECIFIED,
+                            Format.NOT_SPECIFIED,
+                            Format.byteArray)
+                };
+
+    /**
+     * {@inheritDoc}
+     *
+     * Implements
+     * {@link AbstractPushBufferCaptureDevice#createStream(int, FormatControl)}.
+     */
+    protected AudioSilenceStream createStream(
+            int streamIndex,
+            FormatControl formatControl)
+    {
+        return new AudioSilenceStream(this, formatControl);
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * Overrides the super implementation in order to return the list of
+     * <tt>Format</tt>s hardcoded as supported in
+     * <tt>AudioSilenceCaptureDevice</tt> because the super looks them up by
+     * <tt>CaptureDeviceInfo</tt> and this instance does not have one.
+     */
+    @Override
+    protected Format[] getSupportedFormats(int streamIndex)
+    {
+        return SUPPORTED_FORMATS.clone();
+    }
+
+    /**
+     * Implements a <tt>PushBufferStream</tt> which provides silence in the form
+     * of audio media.
+     */
+    private static class AudioSilenceStream
+        extends AbstractPushBufferStream<AudioSilenceCaptureDevice>
+        implements Runnable
+    {
+        /**
+         * The indicator which determines whether {@link #start()} has been
+         * invoked on this instance without an intervening {@link #stop()}.
+         */
+        private boolean started;
+
+        /**
+         * The <tt>Thread</tt> which pushes available media data out of this
+         * instance to its consumer i.e. <tt>BufferTransferHandler</tt>.
+         */
+        private Thread thread;
+
+        /**
+         * Initializes a new <tt>AudioSilenceStream</tt> which is to be exposed
+         * by a specific <tt>AudioSilenceCaptureDevice</tt> and which is to have
+         * its <tt>Format</tt>-related information abstracted by a specific
+         * <tt>FormatControl</tt>.
+         *
+         * @param dataSource the <tt>AudioSilenceCaptureDevice</tt> which is
+         * initializing the new instance and which is to expose it in its array
+         * of <tt>PushBufferStream</tt>s
+         * @param formatControl the <tt>FormatControl</tt> which is to abstract
+         * the <tt>Format</tt>-related information of the new instance
+         */
+        public AudioSilenceStream(
+                AudioSilenceCaptureDevice dataSource,
+                FormatControl formatControl)
+        {
+            super(dataSource, formatControl);
+        }
+
+        /**
+         * Reads available media data from this instance into a specific
+         * <tt>Buffer</tt>.
+         *
+         * @param buffer the <tt>Buffer</tt> to write the available media data
+         * into
+         * @throws IOException if an I/O error has prevented the reading of
+         * available media data from this instance into the specified
+         * <tt>buffer</tt>
+         */
+        public void read(Buffer buffer)
+            throws IOException
+        {
+            if (CLOCK_ONLY)
+            {
+                buffer.setLength(0);
+            }
+            else
+            {
+                AudioFormat format = (AudioFormat) getFormat();
+                int frameSizeInBytes
+                    = format.getChannels()
+                        * (((int) format.getSampleRate()) / 50)
+                        * (format.getSampleSizeInBits() / 8);
+
+                byte[] data
+                    = AbstractCodec2.validateByteArraySize(
+                            buffer,
+                            frameSizeInBytes,
+                            false);
+
+                Arrays.fill(data, 0, frameSizeInBytes, (byte) 0);
+
+                buffer.setFormat(format);
+                buffer.setLength(frameSizeInBytes);
+                buffer.setOffset(0);
+            }
+        }
+
+        /**
+         * Runs in {@link #thread} and pushes available media data out of this
+         * instance to its consumer i.e. <tt>BufferTransferHandler</tt>.
+         */
+        public void run()
+        {
+            try
+            {
+                /*
+                 * Make sure that the current thread which implements the actual
+                 * ticking of the clock implemented by this instance uses a
+                 * thread priority considered appropriate for audio processing.
+                 */
+                AbstractAudioRenderer.useAudioThreadPriority();
+
+                /*
+                 * The method implements a clock which ticks at a certain and
+                 * regular interval of time which is not affected by the
+                 * duration of the execution of, for example, the invocation of
+                 * BufferTransferHandler.transferData(PushBufferStream).
+                 *
+                 * XXX The implementation utilizes System.currentTimeMillis()
+                 * and, consequently, may be broken by run-time adjustments to
+                 * the system time. 
+                 */
+                long tickTime = System.currentTimeMillis();
+
+                while (true)
+                {
+                    long sleepInterval = tickTime - System.currentTimeMillis();
+                    boolean tick = (sleepInterval <= 0);
+
+                    if (tick)
+                    {
+                        /*
+                         * The current thread has woken up just in time or too
+                         * late for the next scheduled clock tick and,
+                         * consequently, the clock should tick right now.
+                         */
+                        tickTime += CLOCK_TICK_INTERVAL;
+                    }
+                    else
+                    {
+                        /*
+                         * The current thread has woken up too early for the
+                         * next scheduled clock tick and, consequently, it
+                         * should sleep until the time of the next scheduled
+                         * clock tick comes.
+                         */
+                        try
+                        {
+                            Thread.sleep(sleepInterval);
+                        }
+                        catch (InterruptedException ie)
+                        {
+                        }
+                        /*
+                         * The clock will not tick and spurious wakeups will be
+                         * handled. However, the current thread will first check
+                         * whether it is still utilized by this
+                         * AudioSilenceStream in order to not delay stop
+                         * requests.
+                         */
+                    }
+
+                    synchronized (this)
+                    {
+                        /*
+                         * If the current Thread is no longer utilized by this
+                         * AudioSilenceStream, it no longer has the right to
+                         * touch it. If this AudioSilenceStream has been
+                         * stopped, the current Thread should stop as well. 
+                         */
+                        if ((thread != Thread.currentThread()) || !started)
+                            break;
+                    }
+
+                    if (tick)
+                    {
+                        BufferTransferHandler transferHandler
+                            = this.transferHandler;
+
+                        if (transferHandler != null)
+                        {
+                            try
+                            {
+                                transferHandler.transferData(this);
+                            }
+                            catch (Throwable t)
+                            {
+                                if (t instanceof ThreadDeath)
+                                    throw (ThreadDeath) t;
+                                else
+                                {
+                                    // TODO Auto-generated method stub
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            finally
+            {
+                synchronized (this)
+                {
+                    if (thread == Thread.currentThread())
+                    {
+                        thread = null;
+                        started = false;
+                        notifyAll();
+                    }
+                }
+            }
+        }
+
+        /**
+         * Starts the transfer of media data from this instance.
+         *
+         * @throws IOException if an error has prevented the start of the
+         * transfer of media from this instance
+         */
+        @Override
+        public synchronized void start()
+            throws IOException
+        {
+            if (thread == null)
+            {
+                String className = getClass().getName();
+
+                thread = new Thread(this, className);
+                thread.setDaemon(true);
+
+                boolean started = false;
+
+                try
+                {
+                    thread.start();
+                    started = true;
+                }
+                finally
+                {
+                    this.started = started;
+                    if (!started)
+                    {
+                        thread = null;
+                        notifyAll();
+
+                        throw new IOException("Failed to start " + className);
+                    }
+                }
+            }
+        }
+
+        /**
+         * Stops the transfer of media data from this instance.
+         *
+         * @throws IOException if an error has prevented the stopping of the
+         * transfer of media from this instance
+         */
+        @Override
+        public synchronized void stop()
+            throws IOException
+        {
+            this.started = false;
+            notifyAll();
+
+            boolean interrupted = false;
+
+            while (thread != null)
+            {
+                try
+                {
+                    wait();
+                }
+                catch (InterruptedException ie)
+                {
+                    interrupted = true;
+                }
+            }
+            if (interrupted)
+                Thread.currentThread().interrupt();
+        }
+    }
+}

src/plugins/jitsivideobridge/src/java/org/jitsi/videobridge/AudioSilenceMediaDevice.java

+/*
+ * Jitsi Videobridge, OpenSource video conferencing.
+ *
+ * Distributable under LGPL license.
+ * See terms of license at gnu.org.
+ */
+package org.jitsi.videobridge;
+
+import javax.media.*;
+import javax.media.protocol.*;
+
+import org.jitsi.impl.neomedia.device.*;
+import org.jitsi.service.neomedia.*;
+
+/**
+ * Implements a <tt>MediaDevice</tt> which provides silence in the form of audio
+ * media and does not play back any (audio) media (because Jitsi Videobridge is
+ * a server-side technology).
+ *
+ * @author Lyubomir Marinov
+ */
+public class AudioSilenceMediaDevice
+    extends AudioMediaDeviceImpl
+{
+    /**
+     * {@inheritDoc}
+     *
+     * Overrides the super implementation to initialize a <tt>CaptureDevice</tt>
+     * without asking FMJ to initialize one for a <tt>CaptureDeviceInfo</tt>.
+     */
+    @Override
+    protected CaptureDevice createCaptureDevice()
+    {
+        return new AudioSilenceCaptureDevice();
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * Overrides the super implementation to disable the very playback because
+     * Jitsi Videobridge is a server-side technology.
+     */
+    @Override
+    protected Processor createPlayer(DataSource dataSource)
+    {
+        return null;
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * Overrides the super implementation to initialize a
+     * <tt>MediaDeviceSession</tt> which disables the very playback because
+     * Jitsi Videobridge is a server-side technology.
+     */
+    @Override
+    public MediaDeviceSession createSession()
+    {
+        return
+            new AudioMediaDeviceSession(this)
+                    {
+                        /**
+                         * {@inheritDoc}
+                         *
+                         * Overrides the super implementation to disable the
+                         * very playback because Jitsi Videobridge is a
+                         * server-side technology.
+                         */
+                        @Override
+                        protected Player createPlayer(DataSource dataSource)
+                        {
+                            return null;
+                        }
+                    };
+    }
+
+    /**
+     * {@inheritDoc}
+     *
+     * Overrides the super implementation to always return
+     * {@link MediaDirection#SENDRECV} because this instance stands for a relay
+     * and because the super bases the <tt>MediaDirection</tt> on the
+     * <tt>CaptureDeviceInfo</tt> which this instance does not have.
+     */
+    @Override
+    public MediaDirection getDirection()
+    {
+        return MediaDirection.SENDRECV;
+    }
+}

src/plugins/jitsivideobridge/src/java/org/jitsi/videobridge/openfire/PluginImpl.java

@@ -14,6 +14,9 @@ import java.util.jar.*;
 import java.util.concurrent.ConcurrentHashMap;
 import java.security.cert.Certificate;
 
+import javax.media.*;
+import javax.media.protocol.*;
+
 import org.jitsi.service.neomedia.*;
 import org.jitsi.util.*;
 import org.jitsi.videobridge.*;
@@ -53,11 +56,14 @@ import org.dom4j.*;
 import org.jitsi.videobridge.*;
 import org.jitsi.impl.neomedia.*;
 import org.jitsi.impl.neomedia.format.*;
+import org.jitsi.impl.neomedia.device.*;
 import org.jitsi.service.neomedia.*;
 import org.jitsi.service.neomedia.device.*;
 import org.jitsi.service.neomedia.event.*;
 import org.jitsi.service.neomedia.format.*;
 import org.jitsi.service.libjitsi.*;
+import org.jitsi.util.*;
+
 
 /**
  * Implements <tt>org.jivesoftware.openfire.container.Plugin</tt> to integrate
@@ -274,7 +280,7 @@ public class PluginImpl  implements Plugin, PropertyEventListener
         try
         {
             String binaryPath =
-                (new URL(ComponentImpl.class.getProtectionDomain()
+                (new URL(Videobridge.class.getProtectionDomain()
                     .getCodeSource().getLocation(), ".")).openConnection()
                     .getPermission().getName();
 
@@ -559,12 +565,7 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 
 				Object object = colibriProvider.fromXML(element);
 
-				if (object instanceof ColibriCommand) {
-					ColibriCommand command = (ColibriCommand) object;
-					reply = handleColibriCommand(command, iq);
-
-				} else if (object instanceof RegisterCommand) {
-
+				if (object instanceof RegisterCommand) {
 					registry.put(fromId, from);
 					reply = IQ.createResultIQ(iq);
 
@@ -592,43 +593,6 @@ public class PluginImpl  implements Plugin, PropertyEventListener
         {
 			return new IQHandlerInfo("colibri", RAYO_COLIBRI);
 		}
-		/**
-		 *
-		 *
-		 */
-		private IQ handleColibriCommand(ColibriCommand command, IQ iq)
-		{
-			Log.info("ColibriIQHandler handleColibriCommand " + command);
-
-			IQ reply = IQ.createResultIQ(iq);
-			String vBridge = command.getVideobridge();
-
-			if (vBridge != null)
-			{
-				String focusAgentName = "jitsi.videobridge." + vBridge;
-				JID user = iq.getFrom();
-
-				Log.info("ColibriIQHandler handleColibriCommand bridge " + focusAgentName);
-
-				if (sessions.containsKey(focusAgentName))
-				{
-					FocusAgent focusAgent = sessions.get(focusAgentName);
-
-					if (focusAgent.isUser(user))
-					{
-						reply = focusAgent.handleColibriCommand(command, iq);
-
-					} else {
-						reply.setError(PacketError.Condition.item_not_found);
-					}
-
-				} else {
-					reply.setError(PacketError.Condition.not_allowed);
-				}
-			}
-
-			return reply;
-		}
 		/**
 		 *
 		 *
@@ -852,6 +816,7 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 		private LocalClientSession session;
 		private String domainName = XMPPServer.getInstance().getServerInfo().getXMPPDomain();
 		private MediaStream mediaStream;
+		private DataSink mediaSink;
 
 		public ConcurrentHashMap<String, Participant> users = new ConcurrentHashMap<String, Participant>();
 		public ConcurrentHashMap<String, Participant> ids = new ConcurrentHashMap<String, Participant>();
@@ -1096,9 +1061,11 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 			Log.info("removeColibriChannel " + count);
 		}
 		/**
+		 *
 		 *
 		 *
 		 */
+		 /*
 		public IQ handleColibriCommand(ColibriCommand command, IQ iq)
 		{
 			String focusJid = XMPPServer.getInstance().createJID(focusName, focusName).toString();
@@ -1157,6 +1124,7 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 			}
 			return reply;
 		}
+		*/
 		/**
 		 *
 		 *
@@ -1179,6 +1147,15 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 				mediaStream.stop();
 				mediaStream = null;
 			}
+
+			if (mediaSink != null)
+			{
+				try {
+					mediaSink.stop();
+				} catch (Exception e) {}
+
+				mediaSink = null;
+			}
 		}
 		/**
 		 *
@@ -1251,7 +1228,12 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 			if (iq.getType() == IQ.Type.result)
 			{
 				Element conference = iq.getChildElement().createCopy();
-				focusId = conference.attributeValue("id");
+
+				if (focusId == null)
+				{
+					focusId = conference.attributeValue("id");
+				}
+
 				String id = packet.getID();
 
 				if (ids.containsKey(id))
@@ -1283,7 +1265,7 @@ public class PluginImpl  implements Plugin, PropertyEventListener
 				Element root = iq.getChildElement();
 				Element conference = null;
 
-				if (user.toString().equals("jitsi-videobridge." + domainName)) // SSRC notification from videobridge, ignore
+				if (user.toString().equals("jitsi-videobridge." + domainName)) // SSRC notification from videobridge, brodcast, create recorder
 				{
 					conference = root.createCopy();	// rayo from participant
 
@@ -1292,6 +1274,30 @@ public class PluginImpl  implements Plugin, PropertyEventListener
  	 	 			if (channels.containsKey(channelId))
  	 	 				broadcastSSRC(channels.get(channelId));
 
+ 	 	 			if (mediaSink == null /*&& count > 1*/)		// recording not working, causing exception
+ 	 	 			{
+						try {
+/*
+							String focusJid = XMPPServer.getInstance().createJID(focusName, focusName).toString();
+							Content content = getVideoBridge().getConference(focusId, focusJid).getOrCreateContent("audio");
+							AudioMixerMediaDevice mediaDevice = (AudioMixerMediaDevice) content.getMixer();
+
+							MediaDeviceSession deviceSession = mediaDevice.createSession();
+							deviceSession.setContentDescriptor(new ContentDescriptor(FileTypeDescriptor.MPEG_AUDIO));
+							deviceSession.setMute(false);
+							deviceSession.start(MediaDirection.SENDRECV);
+							DataSource outputDataSource = deviceSession.getCaptureDevice();
+
+							mediaSink = Manager.createDataSink(outputDataSource, new MediaLocator("file:recording-" + focusName + ".mp3"));
+							mediaSink.open();
+							mediaSink.start();
+*/
+						} catch (Exception e) {
+
+							Log.error("Error creating recording file", e);
+						}
+					}
+
 				} else {
 
 					conference = root.element("conference").createCopy();	// rayo from participant