How to develop a Zoom analogue for TV boxes on RDK and Linux. Understanding the GStreamer framework

The COVID-19 pandemic has become a catalyst for useful new services. For example, Zoom has become so successful that it overtook IBM in value this month. We were inspired by this example, and we decided to go even further: what if we implement online conferences on set-top boxes and Smart TVs in order to communicate not only at work, but to arrange remote gatherings on the couch with friends? But then you can shout together at football, watch a movie or play sports under the supervision of a coach. 

- , - Linux/Android RDK. « Zoom» Smart TV. GStreamer. , .

- . , desktop-, , , embedded- .

, -:

1. . STB-   ARM-, , / . , — .

2. . Android, — RDK, — Linux . . desktop-. .

3. . Ethernet wifi. / — .

4. . .

5. .

. Zoom - :

• /

• /

:

GStreamer, .. .

/  

1) GStreamer

, . , 30 640x480. , RGB24 :

640 480 3 30 = 27 648 000 , .. 26 , .

— - . , , GStreamer. ? :

1. Linux Android.

2. RDK Gstreamer / -.

3. , . FFmpeg, , - GStreamer’.

4. (pipeline). / , , .

5. API /C++ .

6. /, OpenMAX API — -.

2) GStreamer  

, , .  GStreamer , :

gst-inspect-1.0 , , , .

gst-launch-1.0 (pipeline).

GStreamer , , source, sink-. source — , ,  (sink) — , , ( RTP).

. mp4-:

gst-launch-1.0 filesrc location=file.mp4 ! qtdemux ! h264parse ! avdec_h264 ! videoconvert ! autovideosink

mp4-, mp4 — qtdemux, h264, , , , .

autovideosink filesink .

3) GStreamer C/C++ API.

, gst-launch-1.0, , . : (pipeline), GStreamer glib-.

filesrc filesink — «GStreamer: ». H264-.

GStreamer-

gstinit (NULL, NULL);

,

gst_debug_set_active(TRUE);
gst_debug_set_default_threshold(GST_LEVEL_LOG);

: , gstinit .

event-loop, :

GMainLoop *loop;
loop = g_main_loop_new (NULL, FALSE);

:

, GstElement:

GstElement *pipeline, *source, *demuxer, *parser, *decoder, *conv, *sink;

pipeline = gst_pipeline_new ("video-decoder");
source   = gst_element_factory_make ("filesrc",       "file-source");
demuxer  = gst_element_factory_make ("qtdemux",      "h264-demuxer");
parser   = gst_element_factory_make ("h264parse",      "h264-parser");
decoder  = gst_element_factory_make ("avdec_h264",     "h264-decoder");
conv     = gst_element_factory_make ("videoconvert",  "converter");
sink     = gst_element_factory_make ("appsink", "video-output");

gst_element_factory_make, , — GStreamer, , , .

, , gst_element_factory_make NULL.

if (!pipeline || !source || !demuxer || !parser || !decoder || !conv || !sink) {
		//     - 
		return;
}

location gob_ject_set:

gob_ject_set (G_OBJECT (source), "location", argv[1], NULL);

.

GStreamer, bus_call:

GstBus *bus;
	guint bus_watch_id;
	bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline));
	bus_watch_id = gst_bus_add_watch (bus, bus_call, loop);
	gst_object_unref (bus);

gst_object_unref .

:

static gboolean
bus_call (GstBus     *bus,
          GstMessage *msg,
          gpointer    data)
{
  GMainLoop *loop = (GMainLoop *) data;

  switch (GST_MESSAGE_TYPE (msg)) {

    case GST_MESSAGE_EOS:
      LOGI ("End of stream\n");
      g_main_loop_quit (loop);
      break;

    case GST_MESSAGE_ERROR: {
      gchar  *debug;
      GError *error;

      gst_message_parse_error (msg, &error, &debug);
      g_free (debug);

      LOGE ("Error: %s\n", error->message);
      g_error_free (error);

      g_main_loop_quit (loop);
      break;
    }
    default:
      break;
  }

  return TRUE;
}

: , gst-launch. , , :

gst_bin_add_many (GST_BIN (pipeline), source, demuxer, parser, decoder, conv, sink, NULL);
gst_element_link_many (source, demuxer, parser, decoder, conv, sink, NULL);

, , (autovideosink) :

  gst_element_link (source, demuxer);
  gst_element_link_many (parser, decoder, conv, sink, NULL);
  g_signal_connect (demuxer, "pad-added", G_CALLBACK (on_pad_added), parser);

static void
on_pad_added (GstElement *element,
              GstPad     *pad,
              gpointer    data)
{
  GstPad *sinkpad;
  GstElement *decoder = (GstElement *) data;

  /* We can now link this pad with the sink pad */
  g_print ("Dynamic pad created, linking demuxer/decoder\n");

  sinkpad = gst_element_get_static_pad (decoder, "sink");

  gst_pad_link (pad, sinkpad);

  gst_object_unref (sinkpad);
}

, .

, , :

gst_element_set_state (pipeline, GST_STATE_PLAYING);

event-loop:

g_main_loop_run (loop);

:

gst_element_set_state (pipeline, GST_STATE_NULL);
  gst_object_unref (GST_OBJECT (pipeline));
  g_source_remove (bus_watch_id);
  g_main_loop_unref (loop);

4) .

, — , .

gst_element_factory_find, , factory :

if(gst_element_factory_find("omxh264dec"))
		decoder  = gst_element_factory_make ("omxh264dec",     "h264-decoder");
	else
		decoder  = gst_element_factory_make ("avdec_h264",     "h264-decoder");

OMX RDK, .

, , GstElement ( ):

gst_plugin_feature_get_name(gst_element_get_factory(encoder))

.

5)

, . YUV, RGB.

YUYV. , GStreamer, I420. , gl-, I420-. . , .

GStreamer’ , , - . 

1)

  . , , filesrc filesink .

appsrc / appsink. - . 

, ? , . , I420. , ? ?

need-data, :

g_signal_connect (source, "need-data", G_CALLBACK (encoder_cb_need_data), NULL);

:

encoder_cb_need_data (GstElement *appsrc,
          guint       unused_size,
          gpointer    user_data)
{
  GstBuffer *buffer;
  GstFlowReturn ret;
  GstMapInfo map;

   int size;
   uint8_t* image;
  // get image

  buffer = gst_buffer_new_allocate (NULL, size, NULL);
  gst_buffer_map (buffer, &map, GST_MAP_WRITE);
  
  memcpy((guchar *)map.data, image,  gst_buffer_get_size( buffer ) );
  gst_buffer_unmap(buffer, &map);

  g_signal_emit_by_name (appsrc, "push-buffer", buffer, &ret);
  gst_buffer_unref(buffer);
}

image — , , I420.

gst_buffer_new_allocate , .

gst_buffer_map , memcpy, . 

, , GStream’ , .

: gst_buffer_unmap, gst_buffer_unref. . , , , .

, , : caps .

need-data:

	g_object_set (G_OBJECT (source),
        "stream-type", 0,
        "format", GST_FORMAT_TIME, NULL);

	g_object_set (G_OBJECT (source), "caps",
		gst_caps_new_simple ("video/x-raw",
					"format", G_TYPE_STRING, "I420",
					"width", G_TYPE_INT, 640,
					"height", G_TYPE_INT, 480,
					"framerate", GST_TYPE_FRACTION, 30, 1,
		NULL), 
	NULL);

GstElement, g_object_set.

, caps — . , appsrc I420 c 640x480 30 .

, , . , GStreamer - need-data .

, . 

2)

, .

sink pad:

	GstPad *pad = gst_element_get_static_pad (sink, "sink");
  	gst_pad_add_probe  (pad, GST_PAD_PROBE_TYPE_BUFFER, encoder_cb_have_data, NULL, NULL);
  	gst_object_unref (pad);

sink pad — GST_PAD_PROBE_TYPE_BUFFER, — sink pad.

static GstPadProbeReturn
encoder_cb_have_data (GstPad * pad,
                        GstPadProbeInfo * info,
                        gpointer user_data) {

  GstBuffer *buf = gst_pad_probe_info_get_buffer (info);
  GstMemory *bufMem = gst_buffer_get_memory(buf, 0);
  GstMapInfo bufInfo;
  gst_memory_map(bufMem, &bufInfo, GST_MAP_READ);

  // bufInfo.data, bufInfo.size

  gst_memory_unmap(bufMem, &bufInfo);

  return GST_PAD_PROBE_OK;
}

. . GstBuffer, , gst_buffer_get_memory 0 ( ). , , gst_memory_map, bufInfo.data bufInfo.size.

— .

, Smart TV — Zoom -: , / GStreamer, / .

. — — embedded- RDK, Linux Android. — , .

This idea with a video conferencing service via Smart TV can be developed further, both in terms of engineering solutions and scenarios of their use. So share your thoughts in the comments.