[banshee/gst#] [gst#] Visualisation support
- From: Olivier Dufour <dufoli src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [banshee/gst#] [gst#] Visualisation support
- Date: Fri, 17 Jun 2011 10:36:09 +0000 (UTC)
commit 3a991e6b5db28dad887ba1ceec3008ed7bce8dc9
Author: Olivier Dufour <olivier duff gmail com>
Date: Fri Jun 3 21:34:17 2011 +0200
[gst#] Visualisation support
.../Banshee.GStreamerSharp.csproj | 1 +
.../Banshee.GStreamerSharp.dll.config | 1 +
.../Banshee.GStreamerSharp/PlayerEngine.cs | 34 ++-
.../Banshee.GStreamerSharp/Visualization.cs | 331 ++++++++++++++++++++
src/Backends/Banshee.GStreamerSharp/Makefile.am | 3 +-
5 files changed, 368 insertions(+), 2 deletions(-)
---
diff --git a/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp.csproj b/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp.csproj
index 8996451..e930968 100644
--- a/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp.csproj
+++ b/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp.csproj
@@ -76,6 +76,7 @@
<Compile Include="Banshee.GStreamerSharp\Transcoder.cs" />
<Compile Include="Banshee.GStreamerSharp\VideoManager.cs" />
<Compile Include="Banshee.GStreamerSharp\CddaManager.cs" />
+ <Compile Include="Banshee.GStreamerSharp\Visualization.cs" />
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
</Project>
diff --git a/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp.dll.config b/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp.dll.config
index 8920c14..81165cf 100644
--- a/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp.dll.config
+++ b/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp.dll.config
@@ -1,4 +1,5 @@
<configuration>
<dllmap dll="libgdk.dll" target="libgdk-x11-2.0.so" os="!windows"/>
<dllmap dll="libgdk.dll" target="libgdk-win32-2.0-0.dll" os="windows"/>
+ <dllmap dll="libgstbase-0.10.dll" target="libgstbase-0.10.so.0"/>
</configuration>
diff --git a/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp/PlayerEngine.cs b/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp/PlayerEngine.cs
index 565493e..735c5ed 100644
--- a/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp/PlayerEngine.cs
+++ b/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp/PlayerEngine.cs
@@ -51,7 +51,7 @@ using Banshee.Preferences;
namespace Banshee.GStreamerSharp
{
- public class PlayerEngine : Banshee.MediaEngine.PlayerEngine, IEqualizer
+ public class PlayerEngine : Banshee.MediaEngine.PlayerEngine, IEqualizer, IVisualizationDataSource
{
private class AudioSinkBin : Bin
{
@@ -62,6 +62,7 @@ namespace Banshee.GStreamerSharp
Element preamp;
Element first;
GhostPad visible_sink;
+ Element audiotee;
object pipeline_lock = new object ();
public AudioSinkBin (string elementName) : base(elementName)
@@ -70,6 +71,12 @@ namespace Banshee.GStreamerSharp
Add (hw_audio_sink);
first = hw_audio_sink;
+ // Our audio sink is a tee, so plugins can attach their own pipelines
+ audiotee = ElementFactory.Make ("tee", "audiotee");
+ if (audiotee == null) {
+ Log.Error ("Can not create audio tee!");
+ }
+
volume = FindVolumeProvider (hw_audio_sink);
if (volume != null) {
// If the sink provides its own volume property we assume that it will
@@ -96,6 +103,13 @@ namespace Banshee.GStreamerSharp
Log.Debug ("Built and linked Equalizer");
}
+ // Link the first tee pad to the primary audio sink queue
+ Pad sinkpad = first.GetStaticPad ("sink");
+ Pad pad = audiotee.GetRequestPad ("src%d");
+ audiotee ["alloc-pad"] = pad;
+ pad.Link (sinkpad);
+ first = audiotee;
+
visible_sink = new GhostPad ("sink", first.GetStaticPad ("sink"));
AddPad (visible_sink);
}
@@ -273,6 +287,10 @@ namespace Banshee.GStreamerSharp
}
}
+ public Pad RequestTeePad ()
+ {
+ return audiotee.GetRequestPad ("src%d");
+ }
}
@@ -283,6 +301,7 @@ namespace Banshee.GStreamerSharp
ManualResetEvent next_track_set;
CddaManager cddaManager;
VideoManager videoManager;
+ Visualization visualization;
public PlayerEngine ()
{
@@ -321,6 +340,9 @@ namespace Banshee.GStreamerSharp
Volume = (ushort)PlayerEngineService.VolumeSchema.Get ();
}
+ Pad teepad = audio_sink.RequestTeePad ();
+ visualization = new Visualization (audio_sink, teepad);
+
playbin.AddNotification ("volume", OnVolumeChanged);
playbin.Bus.AddWatch (OnBusMessage);
playbin.AboutToFinish += OnAboutToFinish;
@@ -352,6 +374,16 @@ namespace Banshee.GStreamerSharp
base.Dispose ();
}
+ public event VisualizationDataHandler DataAvailable {
+ add {
+ visualization.DataAvailable += value;
+ }
+
+ remove {
+ visualization.DataAvailable -= value;
+ }
+ }
+
public override void VideoExpose (IntPtr window, bool direct)
{
videoManager.WindowExpose (window, direct);
diff --git a/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp/Visualization.cs b/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp/Visualization.cs
new file mode 100644
index 0000000..60f6a3c
--- /dev/null
+++ b/src/Backends/Banshee.GStreamerSharp/Banshee.GStreamerSharp/Visualization.cs
@@ -0,0 +1,331 @@
+ï//
+// Visualization.cs
+//
+// Author:
+// olivier dufour <olivier duff gmail com>
+//
+// Copyright (C) 2011 olivier dufour.
+//
+// Permission is hereby granted, free of charge, to any person obtaining
+// a copy of this software and associated documentation files (the
+// "Software"), to deal in the Software without restriction, including
+// without limitation the rights to use, copy, modify, merge, publish,
+// distribute, sublicense, and/or sell copies of the Software, and to
+// permit persons to whom the Software is furnished to do so, subject to
+// the following conditions:
+//
+// The above copyright notice and this permission notice shall be
+// included in all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+//
+
+using Gst;
+using Gst.Base;
+
+using Hyena;
+
+using Banshee.MediaEngine;
+using System.Runtime.InteropServices;
+using Gst.CorePlugins;
+
+namespace Banshee.GStreamerSharp
+{
+ public class Visualization
+ {
+ private readonly int SLICE_SIZE = 735;
+ Element vis_resampler;
+ Adapter vis_buffer;
+ bool active;
+ bool vis_thawing;
+ GstFFTF32 vis_fft;
+ GstFFTF32Complex[] vis_fft_buffer;
+ float[] vis_fft_sample_buffer;
+ uint wanted_size;
+
+ [StructLayout(LayoutKind.Sequential)]
+ struct GstFFTF32 {
+ };
+
+ [StructLayout(LayoutKind.Sequential)]
+ struct GstFFTF32Complex {
+ public float r;
+ public float i;
+ };
+
+ enum FFTWindow
+ {
+ Rectangular,
+ Hamming,
+ Hann,
+ Bartlett,
+ Blackman
+ }
+
+ [DllImport ("libgstbase-0.10.dll")]
+ private static extern GstFFTF32 gst_fft_f32_new (int len, bool inverse);
+
+ [DllImport ("libgstbase-0.10.dll")]
+ private static extern void gst_fft_f32_window (GstFFTF32 self, [MarshalAs (UnmanagedType.LPArray)] float [] timedata, FFTWindow window);
+
+ [DllImport ("libgstbase-0.10.dll")]
+ private static extern void gst_fft_f32_fft (GstFFTF32 self, [MarshalAs (UnmanagedType.LPArray)] float [] timedata, [MarshalAs (UnmanagedType.LPArray, ArraySubType=UnmanagedType.Struct)] GstFFTF32Complex [] freqdata);
+
+ [DllImport ("libgstbase-0.10.dll")]
+ private static extern void gst_fft_f32_free (GstFFTF32 self);
+
+ public Visualization (Bin audiobin, Pad teepad)
+ {
+ // The basic pipeline we're constructing is:
+ // .audiotee ! queue ! audioresample ! audioconvert ! fakesink
+
+ Element converter, resampler, audiosinkqueue;
+ Pad pad;
+
+ vis_buffer = null;
+ vis_fft = gst_fft_f32_new (SLICE_SIZE * 2, false);
+ vis_fft_buffer = new GstFFTF32Complex [SLICE_SIZE + 1];
+ vis_fft_sample_buffer = new float [SLICE_SIZE];
+
+ // Core elements, if something fails here, it's the end of the world
+ audiosinkqueue = ElementFactory.Make ("queue", "vis-queue");
+
+ pad = audiosinkqueue.GetStaticPad ("sink");
+ pad.AddEventProbe (new PadEventProbeCallback (EventProbe));
+
+ resampler = ElementFactory.Make ("audioresample", "vis-resample");
+ converter = ElementFactory.Make ("audioconvert", "vis-convert");
+ FakeSink fakesink = ElementFactory.Make ("fakesink", "vis-sink") as FakeSink;
+
+ // channels * slice size * float size = size of chunks we want
+ wanted_size = (uint)(2 * SLICE_SIZE * sizeof(float));
+
+ if (audiosinkqueue == null || resampler == null || converter == null || fakesink == null) {
+ Log.Debug ("Could not construct visualization pipeline, a fundamental element could not be created");
+ return;
+ }
+
+ // Keep around the 5 most recent seconds of audio so that when resuming
+ // visualization we have something to show right away.
+ audiosinkqueue ["leaky"] = 2;
+ audiosinkqueue ["max-size-buffers"] = 0;
+ audiosinkqueue ["max-size-bytes"] = 0;
+ audiosinkqueue ["max-size-time"] = Clock.Second * 5;
+
+ fakesink.Handoff += PCMHandoff;
+
+
+ // This enables the handoff signal.
+ fakesink ["signal-handoffs"] = true;
+ // Synchronize so we see vis at the same time as we hear it.
+ fakesink ["sync"] = true;
+ // Drop buffers if they come in too late. This is mainly used when
+ // thawing the vis pipeline.
+ fakesink ["max-lateness"] = Clock.Second / 120;
+ // Deliver buffers one frame early. This allows for rendering
+ // time. (TODO: It would be great to calculate this on-the-fly so
+ // we match the rendering time.
+ fakesink ["ts-offset"] = -(long)(Clock.Second / 60);
+ // Don't go to PAUSED when we freeze the pipeline.
+ fakesink ["async"] = false;
+
+ audiobin.Add (audiosinkqueue, resampler, converter, fakesink);
+
+ pad = audiosinkqueue.GetStaticPad ("sink");
+ teepad.Link (pad);
+
+ Element.Link (audiosinkqueue, resampler, converter);
+
+ converter.LinkFiltered (fakesink, caps);
+
+ vis_buffer = new Adapter ();
+ vis_resampler = resampler;
+ vis_thawing = false;
+ active = false;
+
+ // Disable the pipeline till we hear otherwise from managed land.
+ Blocked = true;
+ }
+
+ public bool Active
+ {
+ set {
+ Blocked = !value;
+ active = value;
+ }
+ }
+
+ private Caps caps = Caps.FromString (
+ "audio/x-raw-float, " +
+ "rate = (int) 44100, " +
+ "channels = (int) 2, " +
+ "endianness = (int) BYTE_ORDER, " +
+ "width = (int) 32");
+
+ private void BlockCallback (Pad pad, bool blocked)
+ {
+ if (!blocked) {
+ // Set thawing mode (discards buffers that are too old from the queue).
+ vis_thawing = true;
+ }
+ }
+
+ private bool Blocked
+ {
+ set {
+ if (vis_resampler == null)
+ return;
+ Pad queue_sink = vis_resampler.GetStaticPad ("src");
+ queue_sink.SetBlocked (value, new PadBlockCallback (BlockCallback));
+ }
+ }
+ private event VisualizationDataHandler OnDataAvailable = null;
+ public event VisualizationDataHandler DataAvailable {
+ add {
+ if (value == null) {
+ return;
+ } else if (OnDataAvailable == null) {
+ Active = true;
+ }
+
+ OnDataAvailable += value;
+ }
+
+ remove {
+ if (value == null) {
+ return;
+ }
+
+ OnDataAvailable -= value;
+
+ if (OnDataAvailable == null) {
+ Active = false;
+ }
+ }
+ }
+
+ private void PCMHandoff (object o, FakeSink.HandoffArgs args)
+ {
+ Buffer data;
+
+ if (OnDataAvailable == null) {
+ return;
+ }
+
+ if (vis_thawing) {
+ // Flush our buffers out.
+ vis_buffer.Clear ();
+ System.Array.Clear (vis_fft_sample_buffer, 0, vis_fft_sample_buffer.Length);
+
+ vis_thawing = false;
+ }
+
+ Structure structure = args.Buffer.Caps [0];
+ int channels = (int)structure.GetValue ("channels");
+
+ wanted_size = (uint)(channels * SLICE_SIZE * sizeof (float));
+
+ //TODO see if buffer need a copy or not
+ //but copy is no available in gst# ;(
+ vis_buffer.Push (args.Buffer);
+ int i, j;
+ while ((data = vis_buffer.Peek (wanted_size)) != null) {
+ float[] buff = new float[data.Size];
+ Marshal.Copy (data.Data, buff, 0, (int) data.Size);
+ float[] deinterlaced = new float [channels * SLICE_SIZE];
+ float[] specbuf = new float [SLICE_SIZE * 2];
+
+ System.Array.Copy (specbuf, vis_fft_sample_buffer, SLICE_SIZE);
+
+ for (i = 0; i < SLICE_SIZE; i++) {
+ float avg = 0.0f;
+
+ for (j = 0; j < channels; j++) {
+ float sample = buff[i * channels + j];
+
+ deinterlaced[j * SLICE_SIZE + i] = sample;
+ avg += sample;
+ }
+
+ avg /= channels;
+ specbuf[i + SLICE_SIZE] = avg;
+ }
+
+ System.Array.Copy (vis_fft_sample_buffer, 0, specbuf, SLICE_SIZE, SLICE_SIZE);
+
+ gst_fft_f32_window (vis_fft, specbuf, FFTWindow.Hamming);
+ gst_fft_f32_fft (vis_fft, specbuf, vis_fft_buffer);
+
+ for (i = 0; i < SLICE_SIZE; i++) {
+ float val;
+
+ GstFFTF32Complex cplx = vis_fft_buffer[i];
+
+ val = cplx.r * cplx.r + cplx.i * cplx.i;
+ val /= SLICE_SIZE * SLICE_SIZE;
+ val = (float)(10.0f * System.Math.Log10 ((double)val));
+
+ val = (val + 60.0f) / 60.0f;
+ if (val < 0.0f)
+ val = 0.0f;
+
+ specbuf[i] = val;
+ }
+
+ float [] flat = new float[channels * SLICE_SIZE];
+ System.Array.Copy (deinterlaced, flat, flat.Length);
+
+ float [][] cbd = new float[channels][];
+ for (int k = 0; k < channels; k++) {
+ float [] channel = new float[SLICE_SIZE];
+ System.Array.Copy (flat, k * SLICE_SIZE, channel, 0, SLICE_SIZE);
+ cbd [k] = channel;
+ }
+
+ float [] spec = new float [SLICE_SIZE];
+ System.Array.Copy (specbuf, spec, SLICE_SIZE);
+
+ try {
+ OnDataAvailable (cbd, new float[][] { spec });
+ } catch (System.Exception e) {
+ Log.Exception ("Uncaught exception during visualization data post.", e);
+ }
+
+ vis_buffer.Flush ((uint)wanted_size);
+ }
+ }
+
+ bool EventProbe (Pad pad, Event padEvent)
+ {
+ switch (padEvent.Type) {
+ case EventType.FlushStart:
+ case EventType.FlushStop:
+ case EventType.Seek:
+ case EventType.NewSegment:
+ vis_thawing = true;
+ break;
+ }
+
+ if (active)
+ return true;
+
+ switch (padEvent.Type) {
+ case EventType.Eos:
+ Blocked = false;
+ break;
+
+ case EventType.NewSegment:
+ Blocked = true;
+ break;
+ }
+
+ return true;
+ }
+ }
+}
diff --git a/src/Backends/Banshee.GStreamerSharp/Makefile.am b/src/Backends/Banshee.GStreamerSharp/Makefile.am
index 1d968e2..6547079 100644
--- a/src/Backends/Banshee.GStreamerSharp/Makefile.am
+++ b/src/Backends/Banshee.GStreamerSharp/Makefile.am
@@ -9,7 +9,8 @@ SOURCES = \
Banshee.GStreamerSharp/CddaManager.cs \
Banshee.GStreamerSharp/PlayerEngine.cs \
Banshee.GStreamerSharp/Transcoder.cs \
- Banshee.GStreamerSharp/VideoManager.cs
+ Banshee.GStreamerSharp/VideoManager.cs \
+ Banshee.GStreamerSharp/Visualization.cs
RESOURCES = Banshee.GStreamerSharp.addin.xml
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
