From cebe72da5c7633fcfd3854d122e9c65d7e418bb3 Mon Sep 17 00:00:00 2001 From: Song Bing Date: Wed, 22 Apr 2015 18:06:35 +0800 Subject: [PATCH 01/65] mpegtsmux: Need get pid when create streams. when camerabin use mpegtsmux as muxer, start video recording and then stop video recording and then start video recording, mpegtsmux get wrong pid. https://bugzilla.gnome.org/show_bug.cgi?id=748288 UpStream Status: Pending (cherry picked from commit aa65cbc085acba15b93e52a2ff0432522042889a) --- gst/mpegtsmux/gstbasetsmux.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gst/mpegtsmux/gstbasetsmux.c b/gst/mpegtsmux/gstbasetsmux.c index 7e0b981cee..db5d516c99 100644 --- a/gst/mpegtsmux/gstbasetsmux.c +++ b/gst/mpegtsmux/gstbasetsmux.c @@ -934,6 +934,21 @@ gst_base_ts_mux_create_pad_stream (GstBaseTsMux * mux, GstPad * pad, ts_pad->prog_id = idx; } else { ts_pad->prog_id = DEFAULT_PROG_ID; + + if (!ts_pad->pid) { + gint pid = -1; + + name = GST_PAD_NAME (pad); + if (name != NULL && sscanf (name, "sink_%d", &pid) == 1) { + if (tsmux_find_stream (mux->tsmux, pid)) { + GST_WARNING_OBJECT (mux, "Duplicate PID"); + } + } else { + pid = tsmux_get_new_pid (mux->tsmux); + } + + ts_pad->pid = pid; + } } } -- 2.25.1 From 06d15f6f58d4657a77b7a5930bb43b3aa3d8f532 Mon Sep 17 00:00:00 2001 From: Lyon Wang Date: Fri, 29 May 2015 09:54:56 +0800 Subject: [PATCH 02/65] modifiy the videoparse rank - Modify the videparsers rank down to avoid link them in . h263parse rank down to 63 . mpegvideoparse rank down to 63 . mpeg4videpparse rank down to 63 . pngparse, rank down to 63 . h265parse rank down to 63 Upstream status: [i.MX specific] internal use only Signed-off-by: Lyon Wang [vidoeparse] Roll-back h264parse rank Roll back the h264parse rank. MMFMWK-7012 For SW video decoder, it only support aligment au, But in GST1.6, rtph264depay output aligment nal format video. Need link h264parse to convert it to au alginment, make the sw decodear work Set h265parse to GST_RANK_PRIMARY + 1,(similar reason as 264parse) http://sw-jira.freescale.net/browse/MMFMWK-7012 Upstream Status: [i.MX specific] Signed-off-by: Lyon Wang (cherry picked from commit 809447e926dc1a5a1e7c7489e40df4d49e8c4b36) --- gst/videoparsers/gsth263parse.c | 2 +- gst/videoparsers/gsth265parse.c | 2 +- gst/videoparsers/gstmpeg4videoparse.c | 2 +- gst/videoparsers/gstmpegvideoparse.c | 2 +- gst/videoparsers/gstpngparse.c | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gst/videoparsers/gsth263parse.c b/gst/videoparsers/gsth263parse.c index d2c1e49549..02e9071498 100644 --- a/gst/videoparsers/gsth263parse.c +++ b/gst/videoparsers/gsth263parse.c @@ -55,7 +55,7 @@ GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, #define parent_class gst_h263_parse_parent_class G_DEFINE_TYPE (GstH263Parse, gst_h263_parse, GST_TYPE_BASE_PARSE); GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (h263parse, "h263parse", - GST_RANK_PRIMARY + 1, GST_TYPE_H263_PARSE, + GST_RANK_MARGINAL - 1, GST_TYPE_H263_PARSE, videoparsers_element_init (plugin)); static gboolean gst_h263_parse_start (GstBaseParse * parse); diff --git a/gst/videoparsers/gsth265parse.c b/gst/videoparsers/gsth265parse.c index 5994abafc5..7c48bb89c3 100644 --- a/gst/videoparsers/gsth265parse.c +++ b/gst/videoparsers/gsth265parse.c @@ -93,7 +93,7 @@ static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src", #define parent_class gst_h265_parse_parent_class G_DEFINE_TYPE (GstH265Parse, gst_h265_parse, GST_TYPE_BASE_PARSE); GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (h265parse, "h265parse", - GST_RANK_SECONDARY, GST_TYPE_H265_PARSE, + GST_RANK_PRIMARY + 1, GST_TYPE_H265_PARSE, videoparsers_element_init (plugin)); static void gst_h265_parse_finalize (GObject * object); diff --git a/gst/videoparsers/gstmpeg4videoparse.c b/gst/videoparsers/gstmpeg4videoparse.c index 1214a26559..cd6d6e4052 100644 --- a/gst/videoparsers/gstmpeg4videoparse.c +++ b/gst/videoparsers/gstmpeg4videoparse.c @@ -73,7 +73,7 @@ enum #define gst_mpeg4vparse_parent_class parent_class G_DEFINE_TYPE (GstMpeg4VParse, gst_mpeg4vparse, GST_TYPE_BASE_PARSE); GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (mpeg4videoparse, "mpeg4videoparse", - GST_RANK_PRIMARY + 1, GST_TYPE_MPEG4VIDEO_PARSE, + GST_RANK_MARGINAL - 1, GST_TYPE_MPEG4VIDEO_PARSE, videoparsers_element_init (plugin)); static gboolean gst_mpeg4vparse_start (GstBaseParse * parse); diff --git a/gst/videoparsers/gstmpegvideoparse.c b/gst/videoparsers/gstmpegvideoparse.c index 5132bf7569..8daa73a316 100644 --- a/gst/videoparsers/gstmpegvideoparse.c +++ b/gst/videoparsers/gstmpegvideoparse.c @@ -66,7 +66,7 @@ enum #define parent_class gst_mpegv_parse_parent_class G_DEFINE_TYPE (GstMpegvParse, gst_mpegv_parse, GST_TYPE_BASE_PARSE); GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (mpegvideoparse, "mpegvideoparse", - GST_RANK_PRIMARY + 1, GST_TYPE_MPEGVIDEO_PARSE, + GST_RANK_MARGINAL - 1, GST_TYPE_MPEGVIDEO_PARSE, videoparsers_element_init (plugin)); static gboolean gst_mpegv_parse_start (GstBaseParse * parse); diff --git a/gst/videoparsers/gstpngparse.c b/gst/videoparsers/gstpngparse.c index 81621d993b..d6a1c80625 100644 --- a/gst/videoparsers/gstpngparse.c +++ b/gst/videoparsers/gstpngparse.c @@ -48,7 +48,7 @@ GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, #define parent_class gst_png_parse_parent_class G_DEFINE_TYPE (GstPngParse, gst_png_parse, GST_TYPE_BASE_PARSE); -GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (pngparse, "pngparse", GST_RANK_PRIMARY, +GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (pngparse, "pngparse", GST_RANK_MARGINAL - 1, GST_TYPE_PNG_PARSE, videoparsers_element_init (plugin)); static gboolean gst_png_parse_start (GstBaseParse * parse); -- 2.25.1 From 1a69c9f2e1c62773076b92d7b95638612c0292cc Mon Sep 17 00:00:00 2001 From: Song Bing Date: Fri, 13 Mar 2015 17:31:29 +0800 Subject: [PATCH 03/65] camerabin: Add one property to set sink element for video recording pipeline Add one property to set sink element for video recording. Default is filesink. https://bugzilla.gnome.org/show_bug.cgi?id=744508 Upstream Status: Inappropriate [i.MX specific] (cherry picked from commit 8451e6611c8d04580376123e0d4b8025bdec0d84) --- gst/camerabin2/gstcamerabin2.c | 73 +++++++++++++++++++++++++++++----- gst/camerabin2/gstcamerabin2.h | 1 + 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/gst/camerabin2/gstcamerabin2.c b/gst/camerabin2/gstcamerabin2.c index f05192be5e..6fade72cad 100644 --- a/gst/camerabin2/gstcamerabin2.c +++ b/gst/camerabin2/gstcamerabin2.c @@ -203,6 +203,7 @@ enum PROP_MUTE_AUDIO, PROP_AUDIO_CAPTURE_SUPPORTED_CAPS, PROP_AUDIO_CAPTURE_CAPS, + PROP_VIDEO_SINK, PROP_ZOOM, PROP_MAX_ZOOM, PROP_IMAGE_ENCODING_PROFILE, @@ -347,7 +348,7 @@ gst_camera_bin_start_capture (GstCameraBin2 * camerabin) /* check that we have a valid location */ if (camerabin->mode == MODE_VIDEO) { - if (camerabin->location == NULL) { + if (camerabin->location == NULL && !camerabin->user_video_sink) { GST_ELEMENT_ERROR (camerabin, RESOURCE, OPEN_WRITE, (_("File location is set to NULL, please set it to a valid filename")), (NULL)); return; @@ -482,10 +483,13 @@ gst_camera_bin_src_notify_readyforcapture (GObject * obj, GParamSpec * pspec, if (camera->mode == MODE_VIDEO) { /* a video recording is about to start, change the filesink location */ gst_element_set_state (camera->videosink, GST_STATE_NULL); - location = g_strdup_printf (camera->location, camera->capture_index); - GST_DEBUG_OBJECT (camera, "Switching videobin location to %s", location); - g_object_set (camera->videosink, "location", location, NULL); - g_free (location); + /* shouldn't set location for user_video_sink */ + if (!camera->user_video_sink) { + location = g_strdup_printf (camera->location, camera->capture_index); + GST_DEBUG_OBJECT (camera, "Switching videobin location to %s", location); + g_object_set (camera->videosink, "location", location, NULL); + g_free (location); + } if (gst_element_set_state (camera->videosink, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { /* Resets the latest state change return, that would be a failure @@ -540,6 +544,8 @@ gst_camera_bin_dispose (GObject * object) if (camerabin->videosink) gst_object_unref (camerabin->videosink); + if (camerabin->user_video_sink) + gst_object_unref (camerabin->user_video_sink); if (camerabin->video_encodebin) gst_object_unref (camerabin->video_encodebin); if (camerabin->videobin_capsfilter) @@ -660,6 +666,12 @@ gst_camera_bin_class_init (GstCameraBin2Class * klass) " taken into use on the next null to ready transition", GST_TYPE_ELEMENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (object_class, PROP_VIDEO_SINK, + g_param_spec_object ("video-sink", "Video sink", + "The video sink element to be used on video recordings. It is only" + " taken into use on the next null to ready transition", + GST_TYPE_ELEMENT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (object_class, PROP_MUTE_AUDIO, g_param_spec_boolean ("mute", "Mute", "If the audio recording should be muted. Note that this still " @@ -1526,13 +1538,30 @@ gst_camera_bin_create_elements (GstCameraBin2 * camera) g_signal_connect (camera->video_encodebin, "element-added", (GCallback) encodebin_element_added, camera); - camera->videosink = - gst_element_factory_make ("filesink", "videobin-filesink"); + /* check if we need to replace the videosink */ + if (camera->videosink) { + if (camera->user_video_sink && camera->user_video_sink != camera->videosink) { + gst_bin_remove (GST_BIN_CAST (camera), camera->videosink); + gst_object_unref (camera->videosink); + camera->videosink = NULL; + } + } + if (!camera->videosink) { - missing_element_name = "filesink"; - goto missing_element; + if (camera->user_video_sink) { + camera->videosink = gst_object_ref (camera->user_video_sink); + } else { + camera->videosink = + gst_element_factory_make ("filesink", "videobin-filesink"); + if (!camera->videosink) { + missing_element_name = "filesink"; + goto missing_element; + } + g_object_set (camera->videosink, "async", FALSE, NULL); + } } - g_object_set (camera->videosink, "async", FALSE, NULL); + + g_assert (camera->videosink != NULL); /* audio elements */ if (!camera->audio_volume) { @@ -1655,7 +1684,9 @@ gst_camera_bin_create_elements (GstCameraBin2 * camera) gst_element_set_locked_state (camera->videosink, TRUE); gst_element_set_locked_state (camera->imagesink, TRUE); - g_object_set (camera->videosink, "location", camera->location, NULL); + if (!camera->user_video_sink) { + g_object_set (camera->videosink, "location", camera->location, NULL); + } g_object_set (camera->imagesink, "location", camera->location, NULL); } @@ -2039,6 +2070,20 @@ gst_camera_bin_set_audio_src (GstCameraBin2 * camera, GstElement * src) camera->user_audio_src = src; } +static void +gst_camera_bin_set_video_sink (GstCameraBin2 * camera, GstElement * sink) +{ + GST_DEBUG_OBJECT (GST_OBJECT (camera), + "Setting video sink %" GST_PTR_FORMAT, sink); + + if (camera->user_video_sink) + g_object_unref (camera->user_video_sink); + + if (sink) + g_object_ref (sink); + camera->user_video_sink = sink; +} + static void gst_camera_bin_set_camera_src (GstCameraBin2 * camera, GstElement * src) { @@ -2072,6 +2117,9 @@ gst_camera_bin_set_property (GObject * object, guint prop_id, case PROP_AUDIO_SRC: gst_camera_bin_set_audio_src (camera, g_value_get_object (value)); break; + case PROP_VIDEO_SINK: + gst_camera_bin_set_video_sink (camera, g_value_get_object (value)); + break; case PROP_MUTE_AUDIO: g_object_set (camera->audio_volume, "mute", g_value_get_boolean (value), NULL); @@ -2255,6 +2303,9 @@ gst_camera_bin_get_property (GObject * object, guint prop_id, case PROP_AUDIO_SRC: g_value_set_object (value, camera->user_audio_src); break; + case PROP_VIDEO_SINK: + g_value_set_object (value, camera->user_video_sink); + break; case PROP_MUTE_AUDIO:{ gboolean mute; diff --git a/gst/camerabin2/gstcamerabin2.h b/gst/camerabin2/gstcamerabin2.h index f478daa11e..2ae3aba2f1 100644 --- a/gst/camerabin2/gstcamerabin2.h +++ b/gst/camerabin2/gstcamerabin2.h @@ -71,6 +71,7 @@ struct _GstCameraBin2 GstElement *video_encodebin; gulong video_encodebin_signal_id; GstElement *videosink; + GstElement *user_video_sink; GstElement *videobin_capsfilter; GstElement *viewfinderbin; -- 2.25.1 From cb3157646945ca9fddbcb3b73badefcbe24d4527 Mon Sep 17 00:00:00 2001 From: Song Bing Date: Wed, 20 May 2015 15:15:08 +0800 Subject: [PATCH 04/65] mpegvideoparse: Need detect picture coding type when drain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Need detect picture coding type when drain Upstream Status: Waiting for review. https://bugzilla.gnome.org/show_bug.cgi?id=749617 (cherry picked from commit be626f76286c38435f5cadb76113f1aa0bbd2a4a) --- gst/videoparsers/gstmpegvideoparse.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/gst/videoparsers/gstmpegvideoparse.c b/gst/videoparsers/gstmpegvideoparse.c index 8daa73a316..c9299e2d70 100644 --- a/gst/videoparsers/gstmpegvideoparse.c +++ b/gst/videoparsers/gstmpegvideoparse.c @@ -749,6 +749,22 @@ need_more: if (GST_BASE_PARSE_DRAINING (parse)) { GST_LOG_OBJECT (mpvparse, "draining, accepting all data"); off = size; + /* decide picture codding type */ + if (mpvparse->pic_offset >= 0 && mpvparse->pic_offset < off) { + GstMpegVideoPacket header; + + header.data = map.data; + header.type = GST_MPEG_VIDEO_PACKET_PICTURE; + header.offset = mpvparse->pic_offset; + header.size = map.size - mpvparse->pic_offset; + if (gst_mpeg_video_packet_parse_picture_header (&header, &mpvparse->pichdr)) + GST_LOG_OBJECT (mpvparse, "picture_coding_type %d (%s), ending" + "frame of size %d", mpvparse->pichdr.pic_type, + picture_type_name (mpvparse->pichdr.pic_type), off - 4); + else + GST_LOG_OBJECT (mpvparse, "Couldn't parse picture at offset %d", + mpvparse->pic_offset); + } ret = TRUE; } else { GST_LOG_OBJECT (mpvparse, "need more data"); -- 2.25.1 From 2374d3b04678c59cfde296a1d681da2bbeda64e2 Mon Sep 17 00:00:00 2001 From: Lyon Wang Date: Fri, 30 Dec 2016 15:53:21 +0800 Subject: [PATCH 05/65] Specific patches for gstplay API play: Add get_rotate, set_rotate API - Add gstplay get_rotate() and set_rotate() API play: Add force-aspect-ratio config - Add get/set force-aspect-ratio config API play: Add set audio / text sink API - Add get/set audio / text sink API Upstream-Status: Inappropriate [i.MX specific] Signed-off-by: Lyon Wang (cherry picked from commit c513c0b28326d968dd416b81b47f9e2acaa4e9a3) --- gst-libs/gst/play/gstplay.c | 284 ++++++++++++++++++++++++++++++++++++ gst-libs/gst/play/gstplay.h | 12 ++ 2 files changed, 296 insertions(+) diff --git a/gst-libs/gst/play/gstplay.c b/gst-libs/gst/play/gstplay.c index 64adcd12d9..61bd9bf8cb 100644 --- a/gst-libs/gst/play/gstplay.c +++ b/gst-libs/gst/play/gstplay.c @@ -127,6 +127,7 @@ typedef enum CONFIG_QUARK_POSITION_INTERVAL_UPDATE, CONFIG_QUARK_ACCURATE_SEEK, CONFIG_QUARK_PIPELINE_DUMP_IN_ERROR_DETAILS, + CONFIG_QUARK_FORCE_ASPECT_RATIO, CONFIG_QUARK_MAX } ConfigQuarkId; @@ -136,6 +137,7 @@ static const gchar *_config_quark_strings[] = { "position-interval-update", "accurate-seek", "pipeline-dump-in-error-details", + "force-aspect-ratio", }; static GQuark _config_quark_table[CONFIG_QUARK_MAX]; @@ -320,6 +322,7 @@ gst_play_init (GstPlay * self) CONFIG_QUARK (POSITION_INTERVAL_UPDATE), G_TYPE_UINT, DEFAULT_POSITION_UPDATE_INTERVAL_MS, CONFIG_QUARK (ACCURATE_SEEK), G_TYPE_BOOLEAN, FALSE, CONFIG_QUARK (PIPELINE_DUMP_IN_ERROR_DETAILS), G_TYPE_BOOLEAN, FALSE, + CONFIG_QUARK (FORCE_ASPECT_RATIO), G_TYPE_BOOLEAN, TRUE, NULL); /* *INDENT-ON* */ @@ -4695,6 +4698,287 @@ gst_play_get_video_snapshot (GstPlay * self, return sample; } +/** + * gst_get_video_sink: + * @play: #GstPlay instance + * + * Returns: actual video sink element + */ +static GstElement * +gst_play_get_video_sink (GstPlay * self) +{ + GstElement *sink = NULL; + GstElement *actual_sink = NULL; + GstIteratorResult rc; + GstIterator *it; + GValue item = { 0, }; + g_return_val_if_fail (GST_IS_PLAY (self), NULL); + + g_object_get (G_OBJECT (self->playbin), "video-sink", &sink, NULL); + if (NULL == sink) { + GST_WARNING_OBJECT (self, "No video-sink found"); + return NULL; + } + it = gst_bin_iterate_sinks ((GstBin *) sink); + do { + rc = gst_iterator_next (it, &item); + if (rc == GST_ITERATOR_OK) { + break; + } + } while (rc != GST_ITERATOR_DONE); + + g_object_unref (sink); + actual_sink = g_value_get_object (&item); + g_value_unset (&item); + gst_iterator_free (it); + + if (NULL == actual_sink) { + GST_WARNING_OBJECT (self, "No video-sink found"); + return NULL; + } + + return actual_sink; +} + +/** + * gst_palyer_set_rotate: + * @play: #GstPlay instance + * @rotation: rotation degree value + * + * Returns: %TRUE or %FALSE + * + * Set the rotation vaule + */ +gboolean +gst_play_set_rotate (GstPlay * self, gint rotation) +{ + GstElement *video_sink = NULL; + GObjectClass *gobjclass = NULL; + g_return_val_if_fail (GST_IS_PLAY (self), FALSE); + + video_sink = gst_play_get_video_sink (self); + if (NULL == video_sink) { + GST_WARNING_OBJECT (self, " cannot get video sink "); + return FALSE; + } + GST_DEBUG_OBJECT (self, "set rotation degree '%d'", rotation); + + gobjclass = G_OBJECT_GET_CLASS (G_OBJECT (video_sink)); + if (g_object_class_find_property (gobjclass, "rotate") + && g_object_class_find_property (gobjclass, "reconfig")) { + g_object_set (G_OBJECT (video_sink), "rotate", rotation / 90, NULL); + g_object_set (G_OBJECT (video_sink), "reconfig", 1, NULL); + } else if (g_object_class_find_property (gobjclass, "rotate-method")) { + g_object_set (G_OBJECT (video_sink), "rotate-method", rotation / 90, NULL); + } else { + GST_INFO_OBJECT (self, "can't set rotation for current video sink %s'", + gst_element_get_name (video_sink)); + return FALSE; + } + + return TRUE; +} + +/** + * gst_play_get_rotate: + * @play: #GstPlay instance + * + * Returns: the rotation degree value + */ +gint +gst_play_get_rotate (GstPlay * self) +{ + GstElement *video_sink = NULL; + GObjectClass *gobjclass = NULL; + gint rotation = 0; + g_return_val_if_fail (GST_IS_PLAY (self), 0); + + video_sink = gst_play_get_video_sink (self); + if (NULL == video_sink) { + GST_WARNING_OBJECT (self, " cannot get video sink "); + return 0; + } + + /* check if the element has "rotate" property */ + gobjclass = G_OBJECT_GET_CLASS (video_sink); + if (g_object_class_find_property (gobjclass, "rotate")) { + g_object_get (G_OBJECT (video_sink), "rotate", &rotation, NULL); + rotation = rotation * 90; + } else if (g_object_class_find_property (gobjclass, "rotate-method")) { + g_object_get (G_OBJECT (video_sink), "rotate-method", &rotation, NULL); + rotation = rotation * 90; + } + + GST_DEBUG_OBJECT (self, "get rotation degree '%d'", rotation); + + return rotation; +} + +/** + * gst_play_config_set_force_aspect_ratio: + * @play: #GstPlay instance + * @force_aspect_ratio: keey original aspect ratio or not + * + * Enable or disable force aspect ratio + * force_aspect_ratio seeking is TRUE by default. + * + * Since: 1.12 + */ +void +gst_play_config_set_force_aspect_ratio (GstPlay * self, gboolean force_aspect_ratio) +{ + GstStructure *config = self->config; + g_return_if_fail (config != NULL); + + gst_structure_id_set (config, + CONFIG_QUARK (FORCE_ASPECT_RATIO), G_TYPE_BOOLEAN, force_aspect_ratio, NULL); + + g_object_set(self->playbin, "force-aspect-ratio", force_aspect_ratio, NULL); +} + +/** + * gst_play_config_get_force_aspect_ratio: + * @config: a #GstPlay configuration + * + * Returns: %TRUE if force-aspect-ratio is enabled + * + * Since 1.12 + */ +gboolean +gst_play_config_get_force_aspect_ratio (const GstStructure * config) +{ + gboolean force_aspect_ratio = TRUE; + + g_return_val_if_fail (config != NULL, FALSE); + + gst_structure_id_get (config, + CONFIG_QUARK (FORCE_ASPECT_RATIO), G_TYPE_BOOLEAN, &force_aspect_ratio, NULL); + + return force_aspect_ratio; +} + +/** + * gst_play_set_audio_sink: + * @play: #GstPlay instance + * @audio_sink: the custom audio sink to set + * + * Returns: %TRUE or %FALSE + * + * Set the customize audio sink + */ +gboolean +gst_play_set_audio_sink (GstPlay * self, GstElement * audio_sink) +{ + g_return_val_if_fail (GST_IS_PLAY (self), FALSE); + g_return_val_if_fail (audio_sink != NULL, FALSE); + + g_object_set (G_OBJECT (self->playbin), "audio-sink", audio_sink, NULL); + return TRUE; +} + +/** + * gst_play_set_text_sink: + * @play: #GstPlay instance + * @text_sink: the custom text sink to set + * + * Returns: %TRUE or %FALSE + * + * Set the customize text sink + */ +gboolean +gst_play_set_text_sink (GstPlay * self, GstElement * text_sink) +{ + g_return_val_if_fail (GST_IS_PLAY (self), FALSE); + g_return_val_if_fail (text_sink != NULL, FALSE); + + g_object_set (G_OBJECT (self->playbin), "text-sink", text_sink, NULL); + return TRUE; +} + +/** + * gst_play_get_audio_sink: + * @play: #GstPlay instance + * + * Returns: actual audio sink element + */ +GstElement * +gst_play_get_audio_sink (GstPlay * self) +{ + GstElement *sink = NULL; + GstElement *actual_sink = NULL; + GstIteratorResult rc; + GstIterator *it; + GValue item = { 0, }; + g_return_val_if_fail (GST_IS_PLAY (self), NULL); + + g_object_get (G_OBJECT (self->playbin), "audio-sink", &sink, NULL); + if (NULL == sink) { + GST_WARNING_OBJECT (self, "No audio-sink found"); + return NULL; + } + it = gst_bin_iterate_sinks ((GstBin *) sink); + do { + rc = gst_iterator_next (it, &item); + if (rc == GST_ITERATOR_OK) { + break; + } + } while (rc != GST_ITERATOR_DONE); + + g_object_unref (sink); + actual_sink = g_value_get_object (&item); + g_value_unset (&item); + gst_iterator_free (it); + + if (NULL == actual_sink) { + GST_WARNING_OBJECT (self, "No auido-sink found"); + return NULL; + } + + return actual_sink; +} + +/** + * gst_play_get_text_sink: + * @play: #GstPlay instance + * + * Returns: actual text sink element + */ +GstElement * +gst_play_get_text_sink (GstPlay * self) +{ + GstElement *sink = NULL; + GstElement *actual_sink = NULL; + GstIteratorResult rc; + GstIterator *it; + GValue item = { 0, }; + g_return_val_if_fail (GST_IS_PLAY (self), NULL); + + g_object_get (G_OBJECT (self->playbin), "text-sink", &sink, NULL); + if (NULL == sink) { + GST_WARNING_OBJECT (self, "No text-sink found"); + return NULL; + } + it = gst_bin_iterate_sinks ((GstBin *) sink); + do { + rc = gst_iterator_next (it, &item); + if (rc == GST_ITERATOR_OK) { + break; + } + } while (rc != GST_ITERATOR_DONE); + + g_object_unref (sink); + actual_sink = g_value_get_object (&item); + g_value_unset (&item); + gst_iterator_free (it); + + if (NULL == actual_sink) { + GST_WARNING_OBJECT (self, "No text-sink found"); + return NULL; + } + + return actual_sink; +} + /** * gst_play_is_play_message: * @msg: A #GstMessage diff --git a/gst-libs/gst/play/gstplay.h b/gst-libs/gst/play/gstplay.h index f733fd2991..17d3044a01 100644 --- a/gst-libs/gst/play/gstplay.h +++ b/gst-libs/gst/play/gstplay.h @@ -444,6 +444,18 @@ void gst_play_message_parse_volume_changed (GstMessage *ms GST_PLAY_API void gst_play_message_parse_muted_changed (GstMessage *msg, gboolean *muted); +/* Custom gstplay API */ +gboolean gst_play_set_rotate (GstPlay * play, gint rotation); +gint gst_play_get_rotate (GstPlay * play); + +void gst_play_config_set_force_aspect_ratio (GstPlay * self, gboolean force_aspect_ratio); +gboolean gst_play_config_get_force_aspect_ratio (const GstStructure * config); + +gboolean gst_play_set_audio_sink (GstPlay * play, GstElement * audio_sink); +gboolean gst_play_set_text_sink (GstPlay * play, GstElement * text_sink); +GstElement * gst_play_get_audio_sink (GstPlay * play); +GstElement * gst_play_get_text_sink (GstPlay * play); + G_END_DECLS #endif /* __GST_PLAY_H__ */ -- 2.25.1 From 7748c5d06fe76323fb8a5859f2edf58463a54a2d Mon Sep 17 00:00:00 2001 From: Lyon Wang Date: Fri, 17 Feb 2017 17:22:31 +0800 Subject: [PATCH 06/65] gstplay: Add gst_play_get_state() API - Add gst_play_get_state() API https://bugzilla.gnome.org/show_bug.cgi?id=778379 Upstream-Status: Submitted Signed-off-by: Lyon Wang (cherry picked from commit 3b5ab8d6803371c8b3ea8a8804a6dbe58b477204) --- gst-libs/gst/play/gstplay.c | 20 ++++++++++++++++++++ gst-libs/gst/play/gstplay.h | 2 ++ 2 files changed, 22 insertions(+) diff --git a/gst-libs/gst/play/gstplay.c b/gst-libs/gst/play/gstplay.c index 61bd9bf8cb..05bff30d08 100644 --- a/gst-libs/gst/play/gstplay.c +++ b/gst-libs/gst/play/gstplay.c @@ -4979,6 +4979,26 @@ gst_play_get_text_sink (GstPlay * self) return actual_sink; } +/** + * gst_get_state: + * @play: #GstPlay instance + * + * Gets internal GstPlay state. + * It's not guaranteed that the state returned is the current state, + * it might've changed in the meantime. + * + * Returns: (transfer none): internal GstPlayState + * + * Since 1.12 + */ +GstPlayState +gst_play_get_state (GstPlay * self) +{ + g_return_val_if_fail (GST_IS_PLAY (self), GST_PLAY_STATE_STOPPED); + + return self->app_state; +} + /** * gst_play_is_play_message: * @msg: A #GstMessage diff --git a/gst-libs/gst/play/gstplay.h b/gst-libs/gst/play/gstplay.h index 17d3044a01..cba758286d 100644 --- a/gst-libs/gst/play/gstplay.h +++ b/gst-libs/gst/play/gstplay.h @@ -456,6 +456,8 @@ gboolean gst_play_set_text_sink (GstPlay * play, GstElement * text_sink); GstElement * gst_play_get_audio_sink (GstPlay * play); GstElement * gst_play_get_text_sink (GstPlay * play); +GstPlayState gst_play_get_state (GstPlay * play); + G_END_DECLS #endif /* __GST_PLAY_H__ */ -- 2.25.1 From b888ce686a6df976c5689349582fd1681eba3dca Mon Sep 17 00:00:00 2001 From: Lyon Wang Date: Thu, 16 Feb 2017 18:50:28 +0800 Subject: [PATCH 07/65] gstplay: Add play/stop sync API - Add play/stop/pause sync API gst_play_play_sync() gst_play_stop_sync() gst_play_pause_sync() https://bugzilla.gnome.org/show_bug.cgi?id=778390 Upstream Status: [i.MX specific] Signed-off-by: Lyon Wang (cherry picked from commit e5493891f87de82214d6626e3753d15ae34af624) --- gst-libs/gst/play/gstplay.c | 113 ++++++++++++++++++++++++++++++++++++ gst-libs/gst/play/gstplay.h | 4 ++ 2 files changed, 117 insertions(+) diff --git a/gst-libs/gst/play/gstplay.c b/gst-libs/gst/play/gstplay.c index 05bff30d08..7c526add1d 100644 --- a/gst-libs/gst/play/gstplay.c +++ b/gst-libs/gst/play/gstplay.c @@ -228,6 +228,10 @@ struct _GstPlay gchar *audio_sid; gchar *subtitle_sid; gulong stream_notify_id; + + /* When error occur, will set this flag to TRUE, + * so that it could quit for sync play/stop loop */ + gboolean got_error; }; struct _GstPlayClass @@ -329,6 +333,7 @@ gst_play_init (GstPlay * self) self->seek_pending = FALSE; self->seek_position = GST_CLOCK_TIME_NONE; self->last_seek_time = GST_CLOCK_TIME_NONE; + self->got_error = FALSE; self->cached_position = 0; self->cached_duration = GST_CLOCK_TIME_NONE; @@ -999,6 +1004,9 @@ on_error (GstPlay * self, GError * err, const GstStructure * details) NULL); } #endif + + self->got_error = TRUE; + api_bus_post_message (self, GST_PLAY_MESSAGE_ERROR, GST_PLAY_MESSAGE_DATA_ERROR, G_TYPE_ERROR, err, GST_PLAY_MESSAGE_DATA_ERROR_DETAILS, GST_TYPE_STRUCTURE, @@ -4999,6 +5007,111 @@ gst_play_get_state (GstPlay * self) return self->app_state; } +/** + * gst_play_wait_state + * @play: #GstPlay instance + * @target_state: target state + * @time_out: time out value + * negtive (< 0): infinitely waiting for state change. + * positive (>0): wait until time out. + * zero (0), do not wait for the state change. + * + * Wait for target state, quit loop when time out + */ +static void +gst_play_wait_state (GstPlay * self, GstPlayState target_state, + gint time_out) +{ + gint wait_cnt = 0; + + while (time_out < 0 || wait_cnt < time_out * 20) { + if (self->app_state == target_state) { + break; + } else if (self->got_error == TRUE) { + self->got_error = FALSE; + return; + } else if (self->is_eos == TRUE) { + return; + } else { + wait_cnt++; + g_usleep (50000); + } + } + if (time_out > 0 && wait_cnt >= time_out * 20) { + on_error (self, g_error_new (GST_PLAY_ERROR, + GST_PLAY_ERROR_FAILED, + "try to play /stop /pause failed, time out"), NULL); + } + + return; +} + +/** + * gst_play_play_sync: + * @play: #GstPlay instance + * @time_out: time out value + * negtive (< 0): infinitely waiting for state change. + * positive (>0): wait until time out. + * zero (0), do not wait for the state change. + * + * Request to play the loaded stream in sync mode. + */ +void +gst_play_play_sync (GstPlay * self, gint time_out) +{ + g_return_if_fail (GST_IS_PLAY (self)); + + gst_play_play (self); + + gst_play_wait_state (self, GST_PLAY_STATE_PLAYING, time_out); + + return; +} + +/** + * gst_play_stop_sync: + * @play: #GstPlay instance + * @time_out: time out value + * negtive (< 0): infinitely waiting for state change. + * positive (>0): wait until time out. + * zero (0), do not wait for the state change. + * + * Stops playing the current stream in sync mode. + */ +void +gst_play_stop_sync (GstPlay * self, gint time_out) +{ + g_return_if_fail (GST_IS_PLAY (self)); + + gst_play_stop (self); + + gst_play_wait_state (self, GST_PLAY_STATE_STOPPED, time_out); + + return; +} + +/** + * gst_play_pause_sync: + * @play: #GstPlay instance + * @time_out: time out value + * negtive (< 0): infinitely waiting for state change. + * positive (>0): wait until time out. + * zero (0), do not wait for the state change. + * + * Pause current stream in sync mode. + */ +void +gst_play_pause_sync (GstPlay * self, gint time_out) +{ + g_return_if_fail (GST_IS_PLAY (self)); + + gst_play_pause (self); + + gst_play_wait_state (self, GST_PLAY_STATE_PAUSED, time_out); + + return; +} + /** * gst_play_is_play_message: * @msg: A #GstMessage diff --git a/gst-libs/gst/play/gstplay.h b/gst-libs/gst/play/gstplay.h index cba758286d..7a3fe5cc63 100644 --- a/gst-libs/gst/play/gstplay.h +++ b/gst-libs/gst/play/gstplay.h @@ -458,6 +458,10 @@ GstElement * gst_play_get_text_sink (GstPlay * play); GstPlayState gst_play_get_state (GstPlay * play); +void gst_play_play_sync (GstPlay * play, gint time_out); +void gst_play_stop_sync (GstPlay * play, gint time_out); +void gst_play_pause_sync (GstPlay * play, gint time_out); + G_END_DECLS #endif /* __GST_PLAY_H__ */ -- 2.25.1 From 2d36b386a03e918072f0a9250a17a3d01f872f7f Mon Sep 17 00:00:00 2001 From: Lyon Wang Date: Thu, 1 Feb 2018 11:37:34 +0800 Subject: [PATCH 08/65] Update for gplay not set video-sink by default MMFMWK-7883 - Add video-direction property in set-rotate - Update get video-sink for autoplug video-sink case - Add kmssink priority logic - kmssink include gstimxcommon.h Signed-off-by: Lyon Wang (cherry picked from commit 39c98b4abcacae9757e19252baa26f5a619b6b22) This reverts commit 89b9bfa84c419acadbfa2ca06cc05af0d12b1dae. kmssink: don't raise rank of kmssink --- gst-libs/gst/play/gstplay.c | 8 ++++++++ sys/kms/gstkmssink.c | 1 + 2 files changed, 9 insertions(+) diff --git a/gst-libs/gst/play/gstplay.c b/gst-libs/gst/play/gstplay.c index 7c526add1d..1e8434c82a 100644 --- a/gst-libs/gst/play/gstplay.c +++ b/gst-libs/gst/play/gstplay.c @@ -4727,6 +4727,8 @@ gst_play_get_video_sink (GstPlay * self) GST_WARNING_OBJECT (self, "No video-sink found"); return NULL; } + if (TRUE != GST_IS_BIN((GstBin*) sink)) + return sink; it = gst_bin_iterate_sinks ((GstBin *) sink); do { rc = gst_iterator_next (it, &item); @@ -4778,6 +4780,9 @@ gst_play_set_rotate (GstPlay * self, gint rotation) g_object_set (G_OBJECT (video_sink), "reconfig", 1, NULL); } else if (g_object_class_find_property (gobjclass, "rotate-method")) { g_object_set (G_OBJECT (video_sink), "rotate-method", rotation / 90, NULL); + } else if (g_object_class_find_property (gobjclass, "video-direction")) { + g_object_set (G_OBJECT (video_sink), "video-direction", rotation / 90, NULL); + g_object_set (G_OBJECT (video_sink), "reconfig", 1, NULL); } else { GST_INFO_OBJECT (self, "can't set rotation for current video sink %s'", gst_element_get_name (video_sink)); @@ -4815,6 +4820,9 @@ gst_play_get_rotate (GstPlay * self) } else if (g_object_class_find_property (gobjclass, "rotate-method")) { g_object_get (G_OBJECT (video_sink), "rotate-method", &rotation, NULL); rotation = rotation * 90; + } else if (g_object_class_find_property (gobjclass, "video-direction")) { + g_object_get (G_OBJECT (video_sink), "video-direction", &rotation, NULL); + rotation = rotation * 90; } GST_DEBUG_OBJECT (self, "get rotation degree '%d'", rotation); diff --git a/sys/kms/gstkmssink.c b/sys/kms/gstkmssink.c index c1e9f64ae3..20716b6d9c 100644 --- a/sys/kms/gstkmssink.c +++ b/sys/kms/gstkmssink.c @@ -62,6 +62,7 @@ #include "gstkmsutils.h" #include "gstkmsbufferpool.h" #include "gstkmsallocator.h" +#include "gstimxcommon.h" #ifdef HAVE_DRM_HDR #include -- 2.25.1 From 7758b837f835b3ca6e65f343a1d5261b6439ba7d Mon Sep 17 00:00:00 2001 From: Lyon Wang Date: Wed, 18 Apr 2018 15:04:32 +0800 Subject: [PATCH 09/65] Revert "player: Fix-up set_seek_accurate() configuration to take a player config instead of a player instance" This reverts commit cc58bd6ae071dec4ea7b4be626034accd0372755. Revert this commits, as in set_config() Config flag can only be set in STOPPED state (cherry picked from commit b6922ed098879deeda0e87c9a4b35e37449ef7a8) --- gst-libs/gst/play/gstplay.c | 5 +++-- gst-libs/gst/play/gstplay.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/play/gstplay.c b/gst-libs/gst/play/gstplay.c index 1e8434c82a..f0691cbd60 100644 --- a/gst-libs/gst/play/gstplay.c +++ b/gst-libs/gst/play/gstplay.c @@ -4508,7 +4508,7 @@ gst_play_config_get_position_update_interval (const GstStructure * config) /** * gst_play_config_set_seek_accurate: - * @config: a #GstPlay configuration + * @play: #GstPlay instance * @accurate: accurate seek or not * * Enable or disable accurate seeking. When enabled, elements will try harder @@ -4524,8 +4524,9 @@ gst_play_config_get_position_update_interval (const GstStructure * config) * Since: 1.20 */ void -gst_play_config_set_seek_accurate (GstStructure * config, gboolean accurate) +gst_play_config_set_seek_accurate (GstPlay * self, gboolean accurate) { + GstStructure *config = self->config; g_return_if_fail (config != NULL); gst_structure_id_set (config, diff --git a/gst-libs/gst/play/gstplay.h b/gst-libs/gst/play/gstplay.h index 7a3fe5cc63..df6c47ec0d 100644 --- a/gst-libs/gst/play/gstplay.h +++ b/gst-libs/gst/play/gstplay.h @@ -373,7 +373,7 @@ GST_PLAY_API guint gst_play_config_get_position_update_interval (const GstStructure * config); GST_PLAY_API -void gst_play_config_set_seek_accurate (GstStructure * config, gboolean accurate); +void gst_play_config_set_seek_accurate (GstPlay * play, gboolean accurate); GST_PLAY_API gboolean gst_play_config_get_seek_accurate (const GstStructure * config); -- 2.25.1 From 768c7226b6812e919ecaf384c92ec2ecf48edf00 Mon Sep 17 00:00:00 2001 From: Song Bing Date: Mon, 21 May 2018 16:12:37 -0700 Subject: [PATCH 10/65] Revert "h26xparse: Resend PPS/SPS after seek" This reverts commit fe116092cdcda0d1d100e1dbbc4577cd9e4e074a. Our video decoder needn't SPS/PPS after seek. The commit will cause below stream seek hang. SHAVectors/H265Dec/Conformance/TS/HEVC_640x360_AACLC_44.1Khz_2ch.ts SHAVectors/H265Dec/Conformance/MKV/HEVC_848x480_24fps_MP2_48Khz_2ch_320Kbps_Sintel-Trailer.mkv upstream status: imx specific (cherry picked from commit 58f7b9919697aada110eb34f0e961c21035acbd7) --- gst/videoparsers/gsth264parse.c | 1 - gst/videoparsers/gsth265parse.c | 1 - 2 files changed, 2 deletions(-) diff --git a/gst/videoparsers/gsth264parse.c b/gst/videoparsers/gsth264parse.c index d7fea7c026..fb0b77c9e2 100644 --- a/gst/videoparsers/gsth264parse.c +++ b/gst/videoparsers/gsth264parse.c @@ -3822,7 +3822,6 @@ gst_h264_parse_event (GstBaseParse * parse, GstEvent * event) case GST_EVENT_SEGMENT_DONE: h264parse->dts = GST_CLOCK_TIME_NONE; h264parse->ts_trn_nb = GST_CLOCK_TIME_NONE; - h264parse->push_codec = TRUE; res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event); break; diff --git a/gst/videoparsers/gsth265parse.c b/gst/videoparsers/gsth265parse.c index 7c48bb89c3..a6bae754cc 100644 --- a/gst/videoparsers/gsth265parse.c +++ b/gst/videoparsers/gsth265parse.c @@ -3444,7 +3444,6 @@ gst_h265_parse_event (GstBaseParse * parse, GstEvent * event) } case GST_EVENT_FLUSH_STOP: case GST_EVENT_SEGMENT_DONE: - h265parse->push_codec = TRUE; res = GST_BASE_PARSE_CLASS (parent_class)->sink_event (parse, event); break; case GST_EVENT_SEGMENT: -- 2.25.1 From 2d74fca77653ba49873ad73fa52d23853407c110 Mon Sep 17 00:00:00 2001 From: Bing Song Date: Fri, 13 Mar 2020 22:15:45 +0800 Subject: [PATCH 11/65] MMFMWK-8740 dvbsrc: don't fail if no frontend tuner. There no frontend tuner when use local stream. Signed-off-by: Bing Song (cherry picked from commit 215971c7ef25dbe57af038952c89b14e9cb3693f) --- sys/dvb/gstdvbsrc.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/sys/dvb/gstdvbsrc.c b/sys/dvb/gstdvbsrc.c index 33ee3ffd15..aed93cace7 100644 --- a/sys/dvb/gstdvbsrc.c +++ b/sys/dvb/gstdvbsrc.c @@ -1593,18 +1593,15 @@ gst_dvbsrc_open_frontend (GstDvbSrc * object, gboolean writable) if (object->fd_frontend < 0) { switch (errno) { case ENOENT: - GST_ELEMENT_ERROR (object, RESOURCE, NOT_FOUND, - (_("Device \"%s\" does not exist."), frontend_dev), (NULL)); + GST_WARNING_OBJECT (object, "Device \"%s\" does not exist.", frontend_dev); break; default: - GST_ELEMENT_ERROR (object, RESOURCE, OPEN_READ_WRITE, - (_("Could not open frontend device \"%s\"."), frontend_dev), - GST_ERROR_SYSTEM); + GST_WARNING_OBJECT (object, "Could not open frontend device \"%s\".", frontend_dev); break; } g_free (frontend_dev); - return FALSE; + return TRUE; } if (object->supported_delsys) @@ -2198,6 +2195,11 @@ gst_dvbsrc_output_frontend_stats (GstDvbSrc * src, fe_status_t * status) GstStructure *structure; gint err; + if (src->fd_frontend < 0) { + GST_WARNING_OBJECT (src, "Frontend not open"); + return TRUE; + } + errno = 0; LOOP_WHILE_EINTR (err, ioctl (src->fd_frontend, FE_READ_STATUS, status)); @@ -2337,8 +2339,8 @@ gst_dvbsrc_tune_fe (GstDvbSrc * object) GST_DEBUG_OBJECT (object, "Starting the frontend tuning process"); if (object->fd_frontend < 0) { - GST_INFO_OBJECT (object, "Frontend not open: tuning later"); - return FALSE; + GST_WARNING_OBJECT (object, "Frontend not open: tuning later"); + return TRUE; } /* If set, confirm the chosen delivery system is actually -- 2.25.1 From 3ad8a83bfcd602354e10b271fcb048966af0985f Mon Sep 17 00:00:00 2001 From: Bing Song Date: Mon, 27 Apr 2020 17:04:54 +0800 Subject: [PATCH 12/65] tinycompresssink: add audio sink based on libtinycompress API. Audio sink based on libtinycompress API. (cherry picked from commit 8e18d3d62819709218a7447807489682ce4ac4dd) --- meson_options.txt | 1 + sys/meson.build | 1 + sys/tinycompress/meson.build | 29 + sys/tinycompress/tinycompress.c | 43 ++ sys/tinycompress/tinycompresssink.c | 818 ++++++++++++++++++++++++++++ sys/tinycompress/tinycompresssink.h | 89 +++ 6 files changed, 981 insertions(+) create mode 100644 sys/tinycompress/meson.build create mode 100644 sys/tinycompress/tinycompress.c create mode 100644 sys/tinycompress/tinycompresssink.c create mode 100644 sys/tinycompress/tinycompresssink.h diff --git a/meson_options.txt b/meson_options.txt index 5f91f42a3e..8ab06d23b3 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -206,6 +206,7 @@ option('magicleap', type : 'feature', value : 'auto', description : 'Magic Leap option('v4l2codecs', type : 'feature', value : 'auto', description : 'Video4Linux Stateless CODECs support') option('uvcgadget', type : 'feature', value : 'auto', description : 'uvc video gadget plugin') option('isac', type : 'feature', value : 'auto', description : 'iSAC plugin') +option('tinycompress', type : 'feature', value : 'auto', description : 'tinycompresssink based on libtinycompress API') # AJA plugin options option('aja-sdk-dir', type : 'string', value : '', diff --git a/sys/meson.build b/sys/meson.build index 64c5eccf68..c8fe5fcccb 100644 --- a/sys/meson.build +++ b/sys/meson.build @@ -33,3 +33,4 @@ subdir('wic') subdir('win32ipc') subdir('winks') subdir('winscreencap') +subdir('tinycompress') diff --git a/sys/tinycompress/meson.build b/sys/tinycompress/meson.build new file mode 100644 index 0000000000..5cc93e321f --- /dev/null +++ b/sys/tinycompress/meson.build @@ -0,0 +1,29 @@ +if get_option('tinycompress').disabled() + subdir_done() +endif + +have_tinycompress = cc.has_header ('tinycompress/tinycompress.h') +if have_tinycompress + tinycompress_dep = cc.find_library('tinycompress', required: false) + have_tinycompress = tinycompress_dep.found() +endif + +if not have_tinycompress + tinycompress_dep = dependency('tinycompress', required: get_option('tinycompress'), + fallback: ['tinycompress', 'tinycompress_dep']) + have_tinycompress = tinycompress_dep.found() +endif + +if have_tinycompress + gsttinycompress = library('gsttinycompress', + 'tinycompress.c', 'tinycompresssink.c', + c_args: gst_plugins_bad_args, + include_directories: [configinc], + dependencies : [gstbase_dep, gstaudio_dep, tinycompress_dep], + install: true, + install_dir: plugins_install_dir + ) + pkgconfig.generate(gsttinycompress, install_dir: plugins_pkgconfig_install_dir) +elif get_option('tinycompress').enabled() + error('tinycompress plugin enabled but TinyALSA library or headers not found') +endif diff --git a/sys/tinycompress/tinycompress.c b/sys/tinycompress/tinycompress.c new file mode 100644 index 0000000000..bd9daf3a51 --- /dev/null +++ b/sys/tinycompress/tinycompress.c @@ -0,0 +1,43 @@ +/* GStreamer + * Copyright (C) 2020 Linux Foundation. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "tinycompresssink.h" + +static gboolean +plugin_init (GstPlugin * plugin) +{ + if (!gst_element_register (plugin, "tinycompresssink", GST_RANK_NONE, + GST_TYPE_TINYCOMPRESSSINK)) { + return FALSE; + } + + return TRUE; +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + tinycompress, + "tinycompress plugin library", + plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/sys/tinycompress/tinycompresssink.c b/sys/tinycompress/tinycompresssink.c new file mode 100644 index 0000000000..788da5b71b --- /dev/null +++ b/sys/tinycompress/tinycompresssink.c @@ -0,0 +1,818 @@ +/* + * GStreamer + * Copyright (C) 2020 Linux Foundation. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +/** + * SECTION:element-tinycompresssink + * + * This element outputs audio to tinycompress. + * + * + * Example pipelines + * |[ + * gst-launch-1.0 -v filesrc location=example.mp3 ! mpegaudioparse ! tinycompresssink + * ]| Play an mp3 file. + * |[ + * + */ + +#ifdef HAVE_CONFIG_H +#include +#endif +#include +#include "sound/compress_params.h" +#include "tinycompresssink.h" + +GST_DEBUG_CATEGORY_STATIC (tinycompresssink_debug); +#define GST_CAT_DEFAULT tinycompresssink_debug + +#define DEFAULT_DEVICE "hw:0,0" +#define DEFAULT_TIMESTAMP FALSE +#define DEFAULT_ENABLE_LPA FALSE +#define DEFAULT_TLENGTH -1 +#define DEFAULT_MINREQ -1 +#define DEFAULT_MAXLENGTH -1 +#define DEFAULT_PREBUF -1 +#define DEFAULT_PROVIDE_CLOCK FALSE + +enum +{ + PROP_0, + PROP_DEVICE, + PROP_TIMESTAMP, + PROP_ENABLE_LPA, + PROP_TLENGTH, + PROP_MINREQ, + PROP_MAXLENGTH, + PROP_PREBUF, + PROP_PROVIDE_CLOCK, + PROP_LAST +}; + +/* the capabilities of the sink pad */ +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (TINY_COMPRESS_SINK_TEMPLATE_CAPS)); + +static gboolean gst_tinycompresssink_start (GstBaseSink * basesink); +static gboolean gst_tinycompresssink_stop (GstBaseSink * basesink); +static gboolean gst_tinycompresssink_set_caps (GstBaseSink * sink, + GstCaps * caps); +static void gst_tinycompresssink_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_tinycompresssink_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); +static void gst_tinycompresssink_finalize (GObject * obj); +static GstStateChangeReturn gst_tinycompresssink_change_state (GstElement * + element, GstStateChange transition); +static GstClock *gst_tinycompresssink_provide_clock (GstElement * element); +static GstFlowReturn gst_tinycompresssink_render (GstBaseSink * bsink, + GstBuffer * buffer); +static GstFlowReturn gst_tinycompresssink_unlock (GstBaseSink * bsink); +static GstFlowReturn gst_tinycompresssink_unlock_stop (GstBaseSink * bsink); +static gboolean gst_tinycompresssink_query (GstBaseSink * bsink, + GstQuery * query); +static gboolean gst_tinycompresssink_event (GstBaseSink * bsink, + GstEvent * event); +static GstFlowReturn gst_tinycompresssink_wait_event (GstBaseSink * bsink, + GstEvent * event); +static void gst_tinycompresssink_get_times (GstBaseSink * bsink, + GstBuffer * buffer, GstClockTime * start, GstClockTime * end); +static GstClockTime gst_tinycompresssink_get_time (GstClock * clock, + void *userdata); + +#define parent_class gst_tinycompresssink_parent_class +G_DEFINE_TYPE (GstTinyCompressSink, gst_tinycompresssink, GST_TYPE_BASE_SINK); + +/* initialize the tinycompresssink's class */ +static void +gst_tinycompresssink_class_init (GstTinyCompressSinkClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *gstelement_class = GST_ELEMENT_CLASS (klass); + GstBaseSinkClass *gstbasesink_class = GST_BASE_SINK_CLASS (klass); + + gobject_class->set_property = gst_tinycompresssink_set_property; + gobject_class->get_property = gst_tinycompresssink_get_property; + gobject_class->finalize = gst_tinycompresssink_finalize; + + gstelement_class->change_state = + GST_DEBUG_FUNCPTR (gst_tinycompresssink_change_state); + gstelement_class->provide_clock = + GST_DEBUG_FUNCPTR (gst_tinycompresssink_provide_clock); + + gstbasesink_class->set_caps = + GST_DEBUG_FUNCPTR (gst_tinycompresssink_set_caps); + gstbasesink_class->start = GST_DEBUG_FUNCPTR (gst_tinycompresssink_start); + gstbasesink_class->stop = GST_DEBUG_FUNCPTR (gst_tinycompresssink_stop); + gstbasesink_class->render = GST_DEBUG_FUNCPTR (gst_tinycompresssink_render); + gstbasesink_class->unlock = GST_DEBUG_FUNCPTR (gst_tinycompresssink_unlock); + gstbasesink_class->unlock_stop = + GST_DEBUG_FUNCPTR (gst_tinycompresssink_unlock_stop); + gstbasesink_class->query = GST_DEBUG_FUNCPTR (gst_tinycompresssink_query); + gstbasesink_class->event = GST_DEBUG_FUNCPTR (gst_tinycompresssink_event); + gstbasesink_class->wait_event = + GST_DEBUG_FUNCPTR (gst_tinycompresssink_wait_event); + gstbasesink_class->get_times = + GST_DEBUG_FUNCPTR (gst_tinycompresssink_get_times); + + /* Properties of tinycompresssink */ + g_object_class_install_property (gobject_class, PROP_DEVICE, + g_param_spec_string ("device", "Device", + "The Tinycompress sink device to connect to", DEFAULT_DEVICE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_TIMESTAMP, + g_param_spec_boolean ("timestamp", "Timestamp", + "Provide buffers with timestamp", DEFAULT_TIMESTAMP, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_ENABLE_LPA, + g_param_spec_boolean ("enable-lpa", "Enable lpa", + "Enable LPA", DEFAULT_ENABLE_LPA, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_TLENGTH, + g_param_spec_uint ("tlength", "Target length", + "The target buffer level (total latency) to request (in bytes)", + 0, G_MAXUINT32, DEFAULT_TLENGTH, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_MINREQ, + g_param_spec_uint ("minreq", "Minmum request size", + "The minmum amount of data that server will request (in bytes)", + 0, G_MAXUINT32, DEFAULT_MINREQ, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_MAXLENGTH, + g_param_spec_uint ("maxlength", "Maximum buffer length", + "Maximum stream buffer size that the server should hold (in bytes)", + 0, G_MAXUINT32, DEFAULT_MAXLENGTH, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PREBUF, + g_param_spec_uint ("prebuf", "Prebuffering length", + "Minimum amount of data required for playback to start (in bytes)", + 0, G_MAXUINT32, DEFAULT_PREBUF, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PROVIDE_CLOCK, + g_param_spec_boolean ("provide-clock", "Provide clock", + "Provide a clock that can be used as the pipeline clock", + DEFAULT_PROVIDE_CLOCK, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + gst_element_class_set_static_metadata (gstelement_class, + "Tinycompress Audio Direct Sink", + "Sink/Audio", "Plays audio to tinycompress", + "Bing Song "); + + gst_element_class_add_static_pad_template (gstelement_class, &sink_template); + + GST_DEBUG_CATEGORY_INIT (tinycompresssink_debug, "tinycompresssink", 0, + "tinycompress Sink"); +} + +static void +gst_tinycompresssink_set_provide_clock (GstTinyCompressSink * csink, + gboolean provide_clock) +{ + GST_OBJECT_LOCK (csink); + + csink->provide_clock = provide_clock; + + if (csink->provide_clock) + GST_OBJECT_FLAG_SET (csink, GST_ELEMENT_FLAG_PROVIDE_CLOCK); + else + GST_OBJECT_FLAG_UNSET (csink, GST_ELEMENT_FLAG_PROVIDE_CLOCK); + + GST_OBJECT_UNLOCK (csink); +} + +static void +gst_tinycompresssink_init (GstTinyCompressSink * csink) +{ + csink->device = g_strdup (DEFAULT_DEVICE); + + csink->provide_clock = DEFAULT_PROVIDE_CLOCK; + csink->timestamp = DEFAULT_TIMESTAMP; + csink->enable_lpa = DEFAULT_ENABLE_LPA; + csink->tlength = DEFAULT_TLENGTH; + csink->minreq = DEFAULT_MINREQ; + csink->maxlength = DEFAULT_MAXLENGTH; + csink->prebuf = DEFAULT_PREBUF; + + csink->clock = gst_audio_clock_new ("TinyCompressSinkClock", + gst_tinycompresssink_get_time, csink, NULL); + + gst_tinycompresssink_set_provide_clock (csink, DEFAULT_PROVIDE_CLOCK); + + g_atomic_int_set (&csink->unlocked, 0); +} + +static void +gst_tinycompresssink_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK_CAST (object); + + switch (prop_id) { + case PROP_DEVICE: + g_free (csink->device); + csink->device = g_value_dup_string (value); + break; + + case PROP_TIMESTAMP: + csink->timestamp = g_value_get_boolean (value); + break; + + case PROP_ENABLE_LPA: + csink->enable_lpa = g_value_get_boolean (value); + break; + + case PROP_TLENGTH: + csink->tlength = g_value_get_uint (value); + break; + + case PROP_MINREQ: + csink->minreq = g_value_get_uint (value); + break; + + case PROP_MAXLENGTH: + csink->maxlength = g_value_get_uint (value); + break; + + case PROP_PREBUF: + csink->prebuf = g_value_get_uint (value); + break; + + case PROP_PROVIDE_CLOCK: + gst_tinycompresssink_set_provide_clock (csink, + g_value_get_boolean (value)); + break; + + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_tinycompresssink_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK_CAST (object); + + switch (prop_id) { + case PROP_DEVICE: + g_value_set_string (value, csink->device); + break; + + case PROP_TIMESTAMP: + g_value_set_boolean (value, csink->timestamp); + break; + + case PROP_ENABLE_LPA: + g_value_set_boolean (value, csink->enable_lpa); + break; + + case PROP_TLENGTH: + g_value_set_uint (value, csink->tlength); + break; + + case PROP_MINREQ: + g_value_set_uint (value, csink->minreq); + break; + + case PROP_MAXLENGTH: + g_value_set_uint (value, csink->maxlength); + break; + + case PROP_PREBUF: + g_value_set_uint (value, csink->prebuf); + break; + + case PROP_PROVIDE_CLOCK: + g_value_set_boolean (value, csink->provide_clock); + break; + + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_tinycompresssink_finalize (GObject * obj) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK_CAST (obj); + + g_free (csink->device); + + gst_object_unref (csink->clock); + + /* TODO: free any properties or proplist */ + G_OBJECT_CLASS (parent_class)->finalize (obj); +} + +static GstStateChangeReturn +gst_tinycompresssink_change_state (GstElement * element, + GstStateChange transition) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK (element); + GstStateChangeReturn ret; + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: + break; + + case GST_STATE_CHANGE_READY_TO_PAUSED: + gst_element_post_message (element, + gst_message_new_clock_provide (GST_OBJECT_CAST (element), + csink->clock, TRUE)); + break; + + case GST_STATE_CHANGE_PAUSED_TO_PLAYING: + compress_resume(csink->compress); + break; + + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + goto state_failure; + + switch (transition) { + case GST_STATE_CHANGE_PLAYING_TO_PAUSED: + compress_pause(csink->compress); + break; + + case GST_STATE_CHANGE_PAUSED_TO_READY: + gst_caps_unref (csink->caps); + csink->caps = NULL; + + gst_element_post_message (element, + gst_message_new_clock_lost (GST_OBJECT_CAST (element), csink->clock)); + break; + + case GST_STATE_CHANGE_READY_TO_NULL: + break; + + default: + break; + } + + return ret; + + /* ERRORS */ +state_failure: + { + return ret; + } +} + +static GstClock * +gst_tinycompresssink_provide_clock (GstElement * element) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK (element); + GstClock *ret = NULL; + + GST_OBJECT_LOCK (csink); + + if (csink->compress && csink->provide_clock) + ret = gst_object_ref (csink->clock); + else + GST_DEBUG_OBJECT (csink, + "No stream or clock disabled, cannot provide clock"); + + GST_OBJECT_UNLOCK (csink); + + return ret; +} + +static GstFlowReturn +gst_tinycompresssink_render (GstBaseSink * bsink, GstBuffer * buf) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK_CAST (bsink); + GstMapInfo info; + GstFlowReturn ret; + unsigned int available; + struct timespec tstamp; + int wrote; + + gst_buffer_map (buf, &info, GST_MAP_READ); + + GST_LOG_OBJECT (csink, "Writing %" G_GSIZE_FORMAT " bytes", info.size); + + for (;;) { + if (bsink->flushing) { + GST_LOG_OBJECT (csink, "In flushing"); + ret = GST_FLOW_FLUSHING; + goto done; + } + + if (compress_get_hpointer(csink->compress, &available, &tstamp) != 0) + goto writable_size_failed; + + /* We have space to write now, let's do it */ + if (available >= info.size) + break; + + GST_LOG_OBJECT (csink, "Waiting for space, available = %" G_GSIZE_FORMAT, + available); + + /* The buffer is full, let's wait till we're asked for more data */ + if (csink->enable_lpa) + system ("echo mem > /sys/power/state"); + else + compress_wait(csink->compress, 10); + + if (g_atomic_int_get (&csink->unlocked)) { + /* We've been asked to unlock, wait until we can proceed */ + ret = gst_base_sink_wait_preroll (bsink); + + if (ret != GST_FLOW_OK) + goto unlock_and_fail; + } + } + + /* FIXME: we skip basesink sync, is there anything we need to do here? */ + + /* FIXME: perform segment clipping */ + + if (info.size > 0) { + wrote = compress_write(csink->compress, info.data, info.size); + if (wrote < 0) { + GST_ERROR_OBJECT (csink, "Error playing sample\n"); + GST_ERROR_OBJECT (csink, "ERR: %s\n", compress_get_error(csink->compress)); + goto writable_size_failed; + } + if (wrote != info.size) { + GST_ERROR_OBJECT (csink, "We wrote %d, DSP accepted %d\n", info.size, wrote); + } + GST_DEBUG_OBJECT (csink, "%s: wrote %d\n", __func__, wrote); + + if (csink->pauseed) { + /* We were pauseed, but the buffer is now full, so let's unpause */ + compress_start(csink->compress); + csink->pauseed = FALSE; + } + } + + + ret = GST_FLOW_OK; + +done: + gst_buffer_unmap (buf, &info); + return ret; + + /* ERRORS */ +unlock_and_fail: + { + goto done; + } +writable_size_failed: + { + GST_ELEMENT_ERROR (csink, RESOURCE, FAILED, + ("compress_get_hpointer() failed: %s", + compress_get_error(csink->compress)), (NULL)); + ret = GST_FLOW_ERROR; + goto unlock_and_fail; + } +} + +static GstFlowReturn +gst_tinycompresssink_unlock (GstBaseSink * bsink) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK (bsink); + + GST_LOG_OBJECT (csink, "triggering unlock"); + + g_atomic_int_set (&csink->unlocked, 1); + + return GST_FLOW_OK; +} + +static GstFlowReturn +gst_tinycompresssink_unlock_stop (GstBaseSink * bsink) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK (bsink); + + GST_LOG_OBJECT (csink, "stopping unlock"); + + g_atomic_int_set (&csink->unlocked, 0); + + return GST_FLOW_OK; +} + +static gboolean +gst_tinycompresssink_open_device (GstTinyCompressSink * csink) +{ + struct compr_config config = {0}; + struct snd_codec codec = {0}; + unsigned int card = 0, device = 0; + + config.codec = &codec; + codec.id = csink->codec_id; + codec.ch_in = csink->channels; + codec.ch_out = csink->channels; + codec.sample_rate = csink->rate; + + sscanf (csink->device, "hw:%d,%d", &card, &device); + GST_INFO_OBJECT (csink, "device: %s card: %d device: %d", csink->device, card, device); + + csink->compress = compress_open(card, device, COMPRESS_IN, &config); + if (!csink->compress || !is_compress_ready(csink->compress)) { + GST_ERROR_OBJECT (csink, "Unable to open Compress device %d:%d\n", + card, device); + GST_ERROR_OBJECT (csink, "ERR: %s\n", compress_get_error(csink->compress)); + return FALSE; + }; + + compress_nonblock(csink->compress, 1); + + csink->pauseed = TRUE; + + return TRUE; +} + +static gboolean +gst_tinycompresssink_start (GstBaseSink * basesink) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK (basesink); + +} + +static gboolean +gst_tinycompresssink_close_device (GstTinyCompressSink * csink) +{ + GST_LOG_OBJECT (csink, "closing device"); + + if (csink->compress) + compress_stop(csink->compress); + + if (csink->compress) + compress_close(csink->compress); + + return TRUE; +} + +static gboolean +gst_tinycompresssink_stop (GstBaseSink * basesink) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK (basesink); + + return gst_tinycompresssink_close_device (csink); +} + +static GstClockTime +gst_tinycompresssink_get_time (GstClock * clock, void *userdata) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK (userdata); + unsigned int available; + struct timespec tstamp; + + if (!csink->compress) + return GST_CLOCK_TIME_NONE; + + if (compress_get_hpointer(csink->compress, &available, &tstamp) != 0) { + GST_ERROR_OBJECT (csink, "Error querying timestamp\n"); + GST_ERROR_OBJECT (csink, "ERR: %s\n", compress_get_error(csink->compress)); + GST_DEBUG_OBJECT (csink, "could not get time"); + return GST_CLOCK_TIME_NONE; + } else { + GST_ERROR_OBJECT (csink, "DSP played %jd.%jd\n", tstamp.tv_sec, tstamp.tv_nsec*1000); + GST_LOG_OBJECT (csink, "got time: %" GST_TIME_FORMAT, GST_TIMESPEC_TO_TIME (tstamp)); + return GST_TIMESPEC_TO_TIME (tstamp); + } +} + +static GstCaps * +gst_tinycompresssink_query_getcaps (GstTinyCompressSink * csink, GstCaps * filter) +{ + GstCaps *ret = NULL; + + if (!csink->compress) { + ret = gst_pad_get_pad_template_caps (GST_BASE_SINK_PAD (csink)); + goto out; + } + + ret = gst_caps_ref (csink->caps); + +out: + if (filter) { + GstCaps *tmp = gst_caps_intersect_full (filter, ret, + GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (ret); + ret = tmp; + } + + GST_LOG_OBJECT (csink, "returning caps caps %" GST_PTR_FORMAT, ret); + + return ret; +} + +static gboolean +gst_tinycompresssink_query (GstBaseSink * bsink, GstQuery * query) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK_CAST (bsink); + gboolean ret = FALSE; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CAPS: + { + GstCaps *caps, *filter; + + gst_query_parse_caps (query, &filter); + caps = gst_tinycompresssink_query_getcaps (csink, filter); + + if (caps) { + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + ret = TRUE; + } + + break; + } + + default: + ret = + GST_BASE_SINK_CLASS (parent_class)->query (GST_BASE_SINK (csink), + query); + break; + } + + return ret; +} + +static gboolean +gst_tinycompresssink_event (GstBaseSink * bsink, GstEvent * event) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK (bsink); + gboolean ret = TRUE; + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_FLUSH_STOP: + GST_DEBUG_OBJECT (csink, "Flushing stream"); + if (csink->compress) + compress_stop(csink->compress); + csink->pauseed = TRUE; + break; + + default: + break; + } + + if (ret) + ret = GST_BASE_SINK_CLASS (parent_class)->event (bsink, event); + + return ret; +} + +static GstFlowReturn +gst_tinycompresssink_wait_event (GstBaseSink * bsink, GstEvent * event) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK (bsink); + GstFlowReturn ret; + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_EOS: + /* Force the stream to start */ + GST_DEBUG_OBJECT (csink, "Forcing start if needed"); + + break; + + default: + break; + } + + ret = GST_BASE_SINK_CLASS (parent_class)->wait_event (bsink, event); + if (ret != GST_FLOW_OK) + return ret; + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_EOS: + GST_DEBUG_OBJECT (csink, "Draining on EOS"); + compress_write(csink->compress, NULL, 1); + /* remove suspend as bitstream maybe returned before systen suspended*/ + /*if (csink->enable_lpa) + system ("echo mem > /sys/power/state");*/ + compress_drain(csink->compress); + break; + + default: + break; + } + + return ret; +} + +static void +gst_tinycompresssink_get_times (GstBaseSink * bsink, GstBuffer * buffer, + GstClockTime * start, GstClockTime * end) +{ + /* We need to buffer up some data etc. so we can't let basesink do + * synchronisation for us. */ + *start = GST_CLOCK_TIME_NONE; + *end = GST_CLOCK_TIME_NONE; +} + +static gboolean +gst_tinycompresssink_set_caps (GstBaseSink * bsink, GstCaps * caps) +{ + GstTinyCompressSink *csink = GST_TINYCOMPRESSSINK (bsink); + const gchar *format; + GstStructure *structure; + GstAudioInfo info; + + GST_INFO_OBJECT (bsink, "setting caps %" GST_PTR_FORMAT, caps); + + csink->caps = gst_caps_ref (caps); + structure = gst_caps_get_structure (caps, 0); + gst_audio_info_init (&info); + + format = gst_structure_get_name (structure); + + if (g_str_equal (format, "audio/x-raw")) { + if (!gst_audio_info_from_caps (&info, caps)) + goto parse_error; + } else if (g_str_equal (format, "audio/mpeg")) { + gint mpegversion, rate, channels; + const gchar *stream_format = NULL; + + if (!gst_structure_get_int (structure, "mpegversion", &mpegversion)) + goto parse_error; + + stream_format = gst_structure_get_string (structure, "stream-format"); + if (mpegversion > 1 && !stream_format) + goto parse_error; + + switch (mpegversion) { + case 1: + csink->codec_id = SND_AUDIOCODEC_MP3; + break; + + case 2: + if (g_str_equal (stream_format, "raw")) + csink->codec_id = SND_AUDIOCODEC_AAC; + else + csink->codec_id = SND_AUDIOCODEC_AAC; + break; + + case 4: + if (g_str_equal (stream_format, "raw")) + csink->codec_id = SND_AUDIOCODEC_AAC; + else + csink->codec_id = SND_AUDIOCODEC_AAC; + break; + + default: + g_assert_not_reached (); + } + + if (!gst_structure_get_int (structure, "rate", &rate)) + goto parse_error; + if (!gst_structure_get_int (structure, "channels", &channels)) + goto parse_error; + + csink->rate = rate; + csink->channels = channels; + } else { + /* There should be no other format we support as of now */ + g_assert_not_reached (); + } + + gst_tinycompresssink_close_device (csink); + if (!gst_tinycompresssink_open_device (csink)) + return FALSE; + + /* Reset clock as we have a new stream */ + gst_audio_clock_reset (GST_AUDIO_CLOCK (csink->clock), 0); + + return TRUE; + + /* ERRORS */ +parse_error: + { + GST_DEBUG ("could not parse caps"); + return FALSE; + } +} diff --git a/sys/tinycompress/tinycompresssink.h b/sys/tinycompress/tinycompresssink.h new file mode 100644 index 0000000000..a8c5286126 --- /dev/null +++ b/sys/tinycompress/tinycompresssink.h @@ -0,0 +1,89 @@ +/* + * GStreamer + * Copyright (C) 2020 Linux Foundation. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#ifndef __GST_TINYCOMPRESSSINK_H__ +#define __GST_TINYCOMPRESSSINK_H__ + +#include +#include + +G_BEGIN_DECLS +#define GST_TYPE_TINYCOMPRESSSINK \ + (gst_tinycompresssink_get_type()) +#define GST_TINYCOMPRESSSINK(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_TINYCOMPRESSSINK,GstTinyCompressSink)) +#define GST_TINYCOMPRESSSINK_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_TINYCOMPRESSSINK,GstTinyCompressSinkClass)) +#define GST_TINYCOMPRESSSINK_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_TINYCOMPRESSSINK,GstTinyCompressSinkClass)) +#define GST_IS_TINYCOMPRESSSINK(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_TINYCOMPRESSSINK)) +#define GST_IS_TINYCOMPRESSSINK_CLASS(obj) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_TINYCOMPRESSSINK)) +#define GST_TINYCOMPRESSSINK_CAST(obj) \ + ((GstTinyCompressSink *)(obj)) + +#define _TINYCOMPRESS_CAPS_MP3 "audio/mpeg, mpegversion = (int) 1, " \ + "mpegaudioversion = (int) [ 1, 3 ], parsed = (boolean) true; " +#define _TINYCOMPRESS_CAPS_AAC "audio/mpeg, mpegversion = (int) { 2, 4 }, " \ + "framed = (boolean) true, stream-format = (string) { adts, raw };" + +typedef struct _GstTinyCompressSink GstTinyCompressSink; +typedef struct _GstTinyCompressSinkClass GstTinyCompressSinkClass; + +struct _GstTinyCompressSink +{ + GstBaseSink base_sink; + struct compress *compress; + gchar *device; + gboolean provide_clock; + gboolean timestamp; + gboolean enable_lpa; + gboolean pauseed; + + /* buffer attributes */ + guint32 tlength; + guint32 minreq; + guint32 maxlength; + guint32 prebuf; + + guint channels; + guint rate; + guint codec_id; + + GstCaps *caps; + GstClock *clock; + + volatile gint unlocked; +}; + +struct _GstTinyCompressSinkClass +{ + GstBaseSinkClass parent_class; +}; + +GType gst_tinycompresssink_get_type (void); + +#define TINY_COMPRESS_SINK_TEMPLATE_CAPS \ + _TINYCOMPRESS_CAPS_MP3 \ + _TINYCOMPRESS_CAPS_AAC \ + +G_END_DECLS +#endif /* __GST_TINYCOMPRESSSINK_H__ */ -- 2.25.1 From 4bf6cf71c8d25cc5dd865150f0d4b742fd3c94a7 Mon Sep 17 00:00:00 2001 From: Bing Song Date: Mon, 27 Jul 2020 18:34:31 +0800 Subject: [PATCH 13/65] tinycompresssink: reset tiny compress handle when stop. reset tiny compress handle when stop, so we can playback repeatly. (cherry picked from commit 3b8aaa129a0a4d1bfa026f62200e726183a5d6fd) --- sys/tinycompress/tinycompresssink.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sys/tinycompress/tinycompresssink.c b/sys/tinycompress/tinycompresssink.c index 788da5b71b..de51a58a6e 100644 --- a/sys/tinycompress/tinycompresssink.c +++ b/sys/tinycompress/tinycompresssink.c @@ -572,6 +572,8 @@ gst_tinycompresssink_close_device (GstTinyCompressSink * csink) if (csink->compress) compress_close(csink->compress); + csink->compress = NULL; + return TRUE; } -- 2.25.1 From 86ddc33a9a1ef31bb5bde366e5d101db4a28c5e6 Mon Sep 17 00:00:00 2001 From: Bing Song Date: Fri, 14 Aug 2020 22:38:45 +0800 Subject: [PATCH 14/65] MMFMWK-8845 [8mp] support PCM format for LPA. Add PCM support in tinycompress. Signed-off-by: Bing Song (cherry picked from commit 823e88132727758db05acfdfe59f569195ac1137) --- sys/tinycompress/tinycompresssink.c | 50 +++++++++++++++++++++++++---- sys/tinycompress/tinycompresssink.h | 17 +++++++--- 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/sys/tinycompress/tinycompresssink.c b/sys/tinycompress/tinycompresssink.c index de51a58a6e..e09ad0a2c4 100644 --- a/sys/tinycompress/tinycompresssink.c +++ b/sys/tinycompress/tinycompresssink.c @@ -35,6 +35,7 @@ #ifdef HAVE_CONFIG_H #include #endif +#include #include #include "sound/compress_params.h" #include "tinycompresssink.h" @@ -49,7 +50,7 @@ GST_DEBUG_CATEGORY_STATIC (tinycompresssink_debug); #define DEFAULT_MINREQ -1 #define DEFAULT_MAXLENGTH -1 #define DEFAULT_PREBUF -1 -#define DEFAULT_PROVIDE_CLOCK FALSE +#define DEFAULT_PROVIDE_CLOCK TRUE enum { @@ -416,12 +417,16 @@ gst_tinycompresssink_render (GstBaseSink * bsink, GstBuffer * buf) GstFlowReturn ret; unsigned int available; struct timespec tstamp; - int wrote; + guint8 *buf_ptr; + int wrote, size; gst_buffer_map (buf, &info, GST_MAP_READ); + buf_ptr = info.data; + size = info.size; GST_LOG_OBJECT (csink, "Writing %" G_GSIZE_FORMAT " bytes", info.size); +again: for (;;) { if (bsink->flushing) { GST_LOG_OBJECT (csink, "In flushing"); @@ -433,7 +438,7 @@ gst_tinycompresssink_render (GstBaseSink * bsink, GstBuffer * buf) goto writable_size_failed; /* We have space to write now, let's do it */ - if (available >= info.size) + if (available > 0) break; GST_LOG_OBJECT (csink, "Waiting for space, available = %" G_GSIZE_FORMAT, @@ -458,15 +463,18 @@ gst_tinycompresssink_render (GstBaseSink * bsink, GstBuffer * buf) /* FIXME: perform segment clipping */ - if (info.size > 0) { - wrote = compress_write(csink->compress, info.data, info.size); + if (size > 0) { + wrote = compress_write(csink->compress, buf_ptr, size); if (wrote < 0) { GST_ERROR_OBJECT (csink, "Error playing sample\n"); GST_ERROR_OBJECT (csink, "ERR: %s\n", compress_get_error(csink->compress)); goto writable_size_failed; } - if (wrote != info.size) { - GST_ERROR_OBJECT (csink, "We wrote %d, DSP accepted %d\n", info.size, wrote); + if (wrote != size) { + buf_ptr += wrote; + size -= wrote; + GST_ERROR_OBJECT (csink, "We wrote %d, DSP accepted %d\n", size, wrote); + goto again; } GST_DEBUG_OBJECT (csink, "%s: wrote %d\n", __func__, wrote); @@ -532,12 +540,14 @@ gst_tinycompresssink_open_device (GstTinyCompressSink * csink) config.codec = &codec; codec.id = csink->codec_id; + codec.format = csink->format; codec.ch_in = csink->channels; codec.ch_out = csink->channels; codec.sample_rate = csink->rate; sscanf (csink->device, "hw:%d,%d", &card, &device); GST_INFO_OBJECT (csink, "device: %s card: %d device: %d", csink->device, card, device); + GST_INFO_OBJECT (csink, "codec id: %d format: %d channels: %d rate: %d", csink->codec_id, csink->format, csink->channels, csink->rate); csink->compress = compress_open(card, device, COMPRESS_IN, &config); if (!csink->compress || !is_compress_ready(csink->compress)) { @@ -737,6 +747,27 @@ gst_tinycompresssink_get_times (GstBaseSink * bsink, GstBuffer * buffer, *end = GST_CLOCK_TIME_NONE; } +static guint +pcm_format_from_gst (GstAudioFormat format) +{ + switch (format) { + case GST_AUDIO_FORMAT_S8: + return SND_PCM_FORMAT_S8; + + case GST_AUDIO_FORMAT_S16LE: + return SND_PCM_FORMAT_S16_LE; + + case GST_AUDIO_FORMAT_S24_32LE: + return SND_PCM_FORMAT_S24_LE; + + case GST_AUDIO_FORMAT_S32LE: + return SND_PCM_FORMAT_S32_LE; + + default: + g_assert_not_reached (); + } +} + static gboolean gst_tinycompresssink_set_caps (GstBaseSink * bsink, GstCaps * caps) { @@ -756,6 +787,11 @@ gst_tinycompresssink_set_caps (GstBaseSink * bsink, GstCaps * caps) if (g_str_equal (format, "audio/x-raw")) { if (!gst_audio_info_from_caps (&info, caps)) goto parse_error; + + csink->codec_id = SND_AUDIOCODEC_PCM; + csink->format = pcm_format_from_gst (GST_AUDIO_INFO_FORMAT (&info)); + csink->channels = GST_AUDIO_INFO_CHANNELS (&info); + csink->rate = GST_AUDIO_INFO_RATE (&info); } else if (g_str_equal (format, "audio/mpeg")) { gint mpegversion, rate, channels; const gchar *stream_format = NULL; diff --git a/sys/tinycompress/tinycompresssink.h b/sys/tinycompress/tinycompresssink.h index a8c5286126..83c86a8683 100644 --- a/sys/tinycompress/tinycompresssink.h +++ b/sys/tinycompress/tinycompresssink.h @@ -40,10 +40,16 @@ G_BEGIN_DECLS #define GST_TINYCOMPRESSSINK_CAST(obj) \ ((GstTinyCompressSink *)(obj)) + +#define _TINYCOMPRESS_CAPS_PCM "audio/x-raw, " \ + "format = (string) { S16LE, S32LE }, " \ + "layout = (string) interleaved, " \ + "rate = (int) [ 8000, 192000 ], " \ + "channels = (int) [ 1, 2 ]; " #define _TINYCOMPRESS_CAPS_MP3 "audio/mpeg, mpegversion = (int) 1, " \ - "mpegaudioversion = (int) [ 1, 3 ], parsed = (boolean) true; " + "mpegaudioversion = (int) [ 1, 3 ]; " #define _TINYCOMPRESS_CAPS_AAC "audio/mpeg, mpegversion = (int) { 2, 4 }, " \ - "framed = (boolean) true, stream-format = (string) { adts, raw };" + "stream-format = (string) { adts, raw };" typedef struct _GstTinyCompressSink GstTinyCompressSink; typedef struct _GstTinyCompressSinkClass GstTinyCompressSinkClass; @@ -64,9 +70,10 @@ struct _GstTinyCompressSink guint32 maxlength; guint32 prebuf; + guint codec_id; + guint format; guint channels; guint rate; - guint codec_id; GstCaps *caps; GstClock *clock; @@ -82,8 +89,8 @@ struct _GstTinyCompressSinkClass GType gst_tinycompresssink_get_type (void); #define TINY_COMPRESS_SINK_TEMPLATE_CAPS \ - _TINYCOMPRESS_CAPS_MP3 \ - _TINYCOMPRESS_CAPS_AAC \ + _TINYCOMPRESS_CAPS_PCM \ + _TINYCOMPRESS_CAPS_MP3 G_END_DECLS #endif /* __GST_TINYCOMPRESSSINK_H__ */ -- 2.25.1 From 08763924758af34957f88375999ee481d07a844c Mon Sep 17 00:00:00 2001 From: Bing Song Date: Tue, 18 Aug 2020 23:28:22 +0800 Subject: [PATCH 15/65] tinycompresssink: send one NULL buffer to drain buffer. Send one NULL buffer with 0 size to force driver send data to FW. (cherry picked from commit 0455b850939b2b9d29c5bb20703d76150fbba0e7) --- sys/tinycompress/tinycompresssink.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sys/tinycompress/tinycompresssink.c b/sys/tinycompress/tinycompresssink.c index e09ad0a2c4..6f3e744914 100644 --- a/sys/tinycompress/tinycompresssink.c +++ b/sys/tinycompress/tinycompresssink.c @@ -50,7 +50,7 @@ GST_DEBUG_CATEGORY_STATIC (tinycompresssink_debug); #define DEFAULT_MINREQ -1 #define DEFAULT_MAXLENGTH -1 #define DEFAULT_PREBUF -1 -#define DEFAULT_PROVIDE_CLOCK TRUE +#define DEFAULT_PROVIDE_CLOCK FALSE enum { @@ -485,6 +485,9 @@ again: } } + /* Send evevy frame to FW when non-LPA */ + if (!csink->enable_lpa) + compress_write(csink->compress, NULL, 0); ret = GST_FLOW_OK; @@ -723,11 +726,13 @@ gst_tinycompresssink_wait_event (GstBaseSink * bsink, GstEvent * event) switch (GST_EVENT_TYPE (event)) { case GST_EVENT_EOS: GST_DEBUG_OBJECT (csink, "Draining on EOS"); - compress_write(csink->compress, NULL, 1); + if (csink->enable_lpa) + compress_write(csink->compress, NULL, 0); /* remove suspend as bitstream maybe returned before systen suspended*/ /*if (csink->enable_lpa) system ("echo mem > /sys/power/state");*/ compress_drain(csink->compress); + GST_DEBUG_OBJECT (csink, "Draining done on EOS"); break; default: -- 2.25.1 From b1cb53f4de5485efeb854cc970b536369513958a Mon Sep 17 00:00:00 2001 From: Bing Song Date: Thu, 27 Aug 2020 22:56:03 +0800 Subject: [PATCH 16/65] MMFMWK-8852 [8mp] cannot playback stream which sample rate bigger then 48K with tinycompress LPA FW ASRC can't support sample rate bigger then 48K. Signed-off-by: Bing Song (cherry picked from commit 71d0351f9078fde6e2179fe179b30cba15550cec) --- sys/tinycompress/tinycompresssink.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/tinycompress/tinycompresssink.h b/sys/tinycompress/tinycompresssink.h index 83c86a8683..4d6ff85bb9 100644 --- a/sys/tinycompress/tinycompresssink.h +++ b/sys/tinycompress/tinycompresssink.h @@ -44,7 +44,7 @@ G_BEGIN_DECLS #define _TINYCOMPRESS_CAPS_PCM "audio/x-raw, " \ "format = (string) { S16LE, S32LE }, " \ "layout = (string) interleaved, " \ - "rate = (int) [ 8000, 192000 ], " \ + "rate = (int) [ 8000, 48000 ], " \ "channels = (int) [ 1, 2 ]; " #define _TINYCOMPRESS_CAPS_MP3 "audio/mpeg, mpegversion = (int) 1, " \ "mpegaudioversion = (int) [ 1, 3 ]; " -- 2.25.1 From b9c5df7668404972a78b1e0803fd96de8d0a9dd9 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 30 Sep 2020 16:44:39 +0800 Subject: [PATCH 17/65] gst-play: export custom gstplay api add GST_PLAY_API declare to expose custom api Signed-off-by: Haihua Hu (cherry picked from commit 6cc15dd52acbfa02af00656249879086a27dee11) --- gst-libs/gst/play/gstplay.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gst-libs/gst/play/gstplay.h b/gst-libs/gst/play/gstplay.h index df6c47ec0d..0296438760 100644 --- a/gst-libs/gst/play/gstplay.h +++ b/gst-libs/gst/play/gstplay.h @@ -445,21 +445,33 @@ GST_PLAY_API void gst_play_message_parse_muted_changed (GstMessage *msg, gboolean *muted); /* Custom gstplay API */ +GST_PLAY_API gboolean gst_play_set_rotate (GstPlay * play, gint rotation); +GST_PLAY_API gint gst_play_get_rotate (GstPlay * play); +GST_PLAY_API void gst_play_config_set_force_aspect_ratio (GstPlay * self, gboolean force_aspect_ratio); +GST_PLAY_API gboolean gst_play_config_get_force_aspect_ratio (const GstStructure * config); +GST_PLAY_API gboolean gst_play_set_audio_sink (GstPlay * play, GstElement * audio_sink); +GST_PLAY_API gboolean gst_play_set_text_sink (GstPlay * play, GstElement * text_sink); +GST_PLAY_API GstElement * gst_play_get_audio_sink (GstPlay * play); +GST_PLAY_API GstElement * gst_play_get_text_sink (GstPlay * play); +GST_PLAY_API GstPlayState gst_play_get_state (GstPlay * play); +GST_PLAY_API void gst_play_play_sync (GstPlay * play, gint time_out); +GST_PLAY_API void gst_play_stop_sync (GstPlay * play, gint time_out); +GST_PLAY_API void gst_play_pause_sync (GstPlay * play, gint time_out); G_END_DECLS -- 2.25.1 From c86810dbcbcf01d9ccc985643bda39503a49d55a Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Tue, 20 Oct 2020 17:05:45 +0800 Subject: [PATCH 18/65] tinycompresssink: not break build when dependency is not found upstream status: imx specific (cherry picked from commit 166e646289c3af9d8004c6103bc67625756a4488) --- sys/tinycompress/meson.build | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sys/tinycompress/meson.build b/sys/tinycompress/meson.build index 5cc93e321f..f9578dfed4 100644 --- a/sys/tinycompress/meson.build +++ b/sys/tinycompress/meson.build @@ -9,7 +9,7 @@ if have_tinycompress endif if not have_tinycompress - tinycompress_dep = dependency('tinycompress', required: get_option('tinycompress'), + tinycompress_dep = dependency('tinycompress', required: false, fallback: ['tinycompress', 'tinycompress_dep']) have_tinycompress = tinycompress_dep.found() endif @@ -25,5 +25,6 @@ if have_tinycompress ) pkgconfig.generate(gsttinycompress, install_dir: plugins_pkgconfig_install_dir) elif get_option('tinycompress').enabled() - error('tinycompress plugin enabled but TinyALSA library or headers not found') + message('tinycompress plugin enabled but TinyALSA library or headers not found') + subdir_done() endif -- 2.25.1 From ba0b2ac5623e3a8af0a25d7824ccc599004eb197 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Thu, 17 Dec 2020 10:03:06 +0800 Subject: [PATCH 19/65] play: support set connection-speed property of playbin Add an API to support set connection-speed property of playbin upstream status: pending https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1889 (cherry picked from commit 7c71a57d356750451b28047ddc71e23f8adfd37c) --- gst-libs/gst/play/gstplay.c | 17 +++++++++++++++++ gst-libs/gst/play/gstplay.h | 4 ++++ 2 files changed, 21 insertions(+) diff --git a/gst-libs/gst/play/gstplay.c b/gst-libs/gst/play/gstplay.c index f0691cbd60..124f043a4d 100644 --- a/gst-libs/gst/play/gstplay.c +++ b/gst-libs/gst/play/gstplay.c @@ -3869,6 +3869,23 @@ gst_play_set_visualization_enabled (GstPlay * self, gboolean enabled) enabled ? "Enabled" : "Disabled"); } +/** + * gst_play_set_connection_speed: + * @play: #GstPlay instance + * @connnection_speed: adaptive playback connection speed in kbps + * + * set adaptive playback connection speed. + */ +gboolean +gst_play_set_connection_speed (GstPlay * self, guint connnection_speed) +{ + g_return_val_if_fail (GST_IS_PLAY (self), 0); + + g_object_set (G_OBJECT (self->playbin), "connection-speed", connnection_speed, + NULL); + return TRUE; +} + struct CBChannelMap { const gchar *label; /* channel label name */ diff --git a/gst-libs/gst/play/gstplay.h b/gst-libs/gst/play/gstplay.h index 0296438760..01d51311ac 100644 --- a/gst-libs/gst/play/gstplay.h +++ b/gst-libs/gst/play/gstplay.h @@ -285,6 +285,10 @@ GST_PLAY_API gboolean gst_play_set_subtitle_track (GstPlay *play, gint stream_index); +GST_PLAY_API +gboolean gst_play_set_connection_speed (GstPlay *play, + guint connnection_speed); + GST_PLAY_API GstPlayMediaInfo * gst_play_get_media_info (GstPlay * play); -- 2.25.1 From f25ed9bdbbdca97adfa4b45969ff72d274969aad Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 23 Dec 2020 19:50:24 +0800 Subject: [PATCH 20/65] dashdemux: setup streams with connection speed chose the default representation by connection speed insead of get lowest bandwidth representation upstream status: pending https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1904 (cherry picked from commit 08279014b965a8bf267a6159d34bc1170add777d) --- ext/dash/gstdashdemux.c | 5 +++++ ext/dash/gstmpdclient.c | 24 +++++++++++++++++------- ext/dash/gstmpdclient.h | 1 + 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/ext/dash/gstdashdemux.c b/ext/dash/gstdashdemux.c index 6e478f1b35..fbd709d32f 100644 --- a/ext/dash/gstdashdemux.c +++ b/ext/dash/gstdashdemux.c @@ -754,9 +754,14 @@ static gboolean gst_dash_demux_setup_mpdparser_streams (GstDashDemux * demux, GstMPDClient * client) { + GstAdaptiveDemux *adaptivedemux = GST_ADAPTIVE_DEMUX_CAST (demux); gboolean has_streams = FALSE; GList *adapt_sets, *iter; + if (adaptivedemux->connection_speed) { + client->connection_speed = adaptivedemux->connection_speed; + } + adapt_sets = gst_mpd_client_get_adaptation_sets (client); for (iter = adapt_sets; iter; iter = g_list_next (iter)) { GstMPDAdaptationSetNode *adapt_set_node = iter->data; diff --git a/ext/dash/gstmpdclient.c b/ext/dash/gstmpdclient.c index 9be0bd02b9..c4a12c0b8b 100644 --- a/ext/dash/gstmpdclient.c +++ b/ext/dash/gstmpdclient.c @@ -157,7 +157,6 @@ gst_mpd_client_get_lowest_representation (GList * Representations) return lowest; } -#if 0 static GstMPDRepresentationNode * gst_mpdparser_get_highest_representation (GList * Representations) { @@ -177,6 +176,7 @@ gst_mpdparser_get_representation_with_max_bandwidth (GList * Representations, { GList *list = NULL; GstMPDRepresentationNode *representation, *best_rep = NULL; + gint best_bandwidth = 0; if (Representations == NULL) return NULL; @@ -186,14 +186,15 @@ gst_mpdparser_get_representation_with_max_bandwidth (GList * Representations, for (list = g_list_first (Representations); list; list = g_list_next (list)) { representation = (GstMPDRepresentationNode *) list->data; - if (representation && representation->bandwidth <= max_bandwidth) { + if (representation && representation->bandwidth <= max_bandwidth + && representation->bandwidth > best_bandwidth) { best_rep = representation; + best_bandwidth = representation->bandwidth; } } return best_rep; } -#endif static GstMPDSegmentListNode * gst_mpd_client_fetch_external_segment_list (GstMPDClient * client, @@ -454,6 +455,7 @@ gst_mpd_client_class_init (GstMPDClientClass * klass) static void gst_mpd_client_init (GstMPDClient * client) { + client->connection_speed = 0; } GstMPDClient * @@ -1616,7 +1618,7 @@ gboolean gst_mpd_client_setup_streaming (GstMPDClient * client, GstMPDAdaptationSetNode * adapt_set) { - GstMPDRepresentationNode *representation; + GstMPDRepresentationNode *representation = NULL; GList *rep_list = NULL; GstActiveStream *stream; @@ -1632,7 +1634,8 @@ gst_mpd_client_setup_streaming (GstMPDClient * client, stream->baseURL_idx = 0; stream->cur_adapt_set = adapt_set; - GST_DEBUG ("0. Current stream %p", stream); + GST_DEBUG ("0. Current stream %p connection speed %u kbps", stream, + client->connection_speed / 1000); #if 0 /* fast start */ @@ -1646,10 +1649,17 @@ gst_mpd_client_setup_streaming (GstMPDClient * client, representation = gst_mpd_client_get_lowest_representation (rep_list); } #else - /* slow start */ - representation = gst_mpd_client_get_lowest_representation (rep_list); + /* select representation based on connection-speed */ + if (client->connection_speed) + representation = + gst_mpdparser_get_representation_with_max_bandwidth (rep_list, + client->connection_speed); #endif + /* if no matched representation, then slow start */ + if (!representation) + representation = gst_mpd_client_get_lowest_representation (rep_list); + if (!representation) { GST_WARNING ("No valid representation in the MPD file, aborting..."); gst_mpdparser_free_active_stream (stream); diff --git a/ext/dash/gstmpdclient.h b/ext/dash/gstmpdclient.h index b15b5cad3b..0809cbfe9c 100644 --- a/ext/dash/gstmpdclient.h +++ b/ext/dash/gstmpdclient.h @@ -42,6 +42,7 @@ struct _GstMPDClient gchar *mpd_uri; /* manifest file URI */ gchar *mpd_base_uri; /* base URI for resolving relative URIs. * this will be different for redirects */ + guint connection_speed; /* connection speed set by player */ /* profiles */ gboolean profile_isoff_ondemand; -- 2.25.1 From 5c78a04ba94f40faa1bca90dd90130340e9cff36 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 23 Dec 2020 19:59:39 +0800 Subject: [PATCH 21/65] dashdemux: create stream collection and push downstream create stream collection info and push it downstream via gstevent upstream status: pending https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1904 (cherry picked from commit 094ab8fd063c8998fe0dd8d609797db99744b674) --- ext/dash/gstdashdemux.c | 13 ++++++++ ext/dash/gstmpdclient.c | 67 +++++++++++++++++++++++++++++++++++++++++ ext/dash/gstmpdclient.h | 1 + ext/dash/gstmpdhelper.c | 17 +++++++++++ ext/dash/gstmpdhelper.h | 1 + 5 files changed, 99 insertions(+) diff --git a/ext/dash/gstdashdemux.c b/ext/dash/gstdashdemux.c index fbd709d32f..5471b83640 100644 --- a/ext/dash/gstdashdemux.c +++ b/ext/dash/gstdashdemux.c @@ -933,6 +933,19 @@ gst_dash_demux_setup_all_streams (GstDashDemux * demux) gst_dash_demux_send_content_protection_event, stream); } + if (active_stream->cur_adapt_set) { + GstStreamCollection *collection = NULL; + collection = + gst_mpd_client_get_stream_collection (demux->client, + active_stream->cur_adapt_set); + if (collection) { + GstEvent *event = NULL; + event = gst_event_new_stream_collection (collection); + gst_adaptive_demux_stream_queue_event ((GstAdaptiveDemuxStream *) + stream, event); + } + } + gst_isoff_sidx_parser_init (&stream->sidx_parser); } diff --git a/ext/dash/gstmpdclient.c b/ext/dash/gstmpdclient.c index c4a12c0b8b..b5179da327 100644 --- a/ext/dash/gstmpdclient.c +++ b/ext/dash/gstmpdclient.c @@ -2726,6 +2726,73 @@ gst_mpd_client_active_stream_contains_subtitles (GstActiveStream * stream) || (rep_codecs && g_str_has_prefix (rep_codecs, "stpp")); } +GstStreamCollection * +gst_mpd_client_get_stream_collection (GstMPDClient * client, + GstMPDAdaptationSetNode * adapt_set) +{ + GList *list = NULL; + GstMPDRepresentationBaseNode *base_rep = NULL; + GstMPDRepresentationNode *rep = NULL; + GstStreamCollection *collection = NULL; + const gchar *caps_mime; + + if (adapt_set == NULL) + return NULL; + + collection = gst_stream_collection_new (NULL); + + for (list = g_list_first (adapt_set->Representations); list; + list = g_list_next (list)) { + GstStream *stream = NULL; + GstCaps *caps = NULL; + GstStreamType type = GST_STREAM_TYPE_UNKNOWN; + gchar *caps_string = NULL; + + base_rep = GST_MPD_REPRESENTATION_BASE_NODE (list->data); + rep = (GstMPDRepresentationNode *) list->data; + caps_mime = gst_mpd_helper_mimetype_to_caps (base_rep->mimeType); + type = gst_mpd_helper_mimetype_to_stream_type (base_rep->mimeType); + + switch (type) { + case GST_STREAM_TYPE_VIDEO: + caps_string = + g_strdup_printf ("%s,width=%u,height=%u,bitrate=%u", caps_mime, + base_rep->width, base_rep->height, rep->bandwidth); + break; + case GST_STREAM_TYPE_AUDIO: + /* fixme: add channel configure */ + caps_string = + g_strdup_printf ("%s,rate=%s", caps_mime, + base_rep->audioSamplingRate); + break; + case GST_STREAM_TYPE_TEXT: + caps_string = g_strdup_printf ("%s", caps_mime); + break; + case GST_STREAM_TYPE_UNKNOWN: + case GST_STREAM_TYPE_CONTAINER: + GST_DEBUG_OBJECT (client, "meet not support stream type"); + break; + } + + if (caps_string) { + caps = gst_caps_from_string (caps_string); + g_free (caps_string); + } + + if (!caps) { + gst_object_unref (collection); + return NULL; + } + + stream = gst_stream_new (rep->id, caps, type, GST_STREAM_FLAG_NONE); + gst_stream_collection_add_stream (collection, stream); + + gst_caps_unref (caps); + } + + return collection; +} + GstCaps * gst_mpd_client_get_stream_caps (GstActiveStream * stream) { diff --git a/ext/dash/gstmpdclient.h b/ext/dash/gstmpdclient.h index 0809cbfe9c..f35ddac8c3 100644 --- a/ext/dash/gstmpdclient.h +++ b/ext/dash/gstmpdclient.h @@ -126,6 +126,7 @@ void gst_mpd_client_seek_to_first_segment (GstMPDClient * client); GstDateTime *gst_mpd_client_get_next_segment_availability_start_time (GstMPDClient * client, GstActiveStream * stream); /* Get audio/video stream parameters (caps, width, height, rate, number of channels) */ +GstStreamCollection * gst_mpd_client_get_stream_collection (GstMPDClient * client, GstMPDAdaptationSetNode * adapt_set); GstCaps * gst_mpd_client_get_stream_caps (GstActiveStream * stream); gboolean gst_mpd_client_get_bitstream_switching_flag (GstActiveStream * stream); guint gst_mpd_client_get_video_stream_width (GstActiveStream * stream); diff --git a/ext/dash/gstmpdhelper.c b/ext/dash/gstmpdhelper.c index a66de89b92..26c78bab8d 100644 --- a/ext/dash/gstmpdhelper.c +++ b/ext/dash/gstmpdhelper.c @@ -78,6 +78,23 @@ gst_mpd_helper_get_SAP_type (xmlNode * a_node, return exists; } +GstStreamType +gst_mpd_helper_mimetype_to_stream_type (const gchar * mimeType) +{ + if (mimeType == NULL) + return GST_STREAM_TYPE_UNKNOWN; + if (strcmp (mimeType, "video/mp2t") == 0) { + return GST_STREAM_TYPE_VIDEO; + } else if (strcmp (mimeType, "video/mp4") == 0) { + return GST_STREAM_TYPE_VIDEO; + } else if (strcmp (mimeType, "audio/mp4") == 0) { + return GST_STREAM_TYPE_AUDIO; + } else if (strcmp (mimeType, "text/vtt") == 0) { + return GST_STREAM_TYPE_TEXT; + } else + return GST_STREAM_TYPE_UNKNOWN; +} + const gchar * gst_mpd_helper_mimetype_to_caps (const gchar * mimeType) { diff --git a/ext/dash/gstmpdhelper.h b/ext/dash/gstmpdhelper.h index 95eaec9c87..2c09ab1479 100644 --- a/ext/dash/gstmpdhelper.h +++ b/ext/dash/gstmpdhelper.h @@ -60,6 +60,7 @@ typedef enum gboolean gst_mpd_helper_get_mpd_type (xmlNode * a_node, const gchar * property_name, GstMPDFileType * property_value); gboolean gst_mpd_helper_get_SAP_type (xmlNode * a_node, const gchar * property_name, GstMPDSAPType * property_value); +GstStreamType gst_mpd_helper_mimetype_to_stream_type (const gchar * mimeType); const gchar * gst_mpd_helper_mimetype_to_caps (const gchar * mimeType); GstUri *gst_mpd_helper_combine_urls (GstUri * base, GList * list, gchar ** query, guint idx); int gst_mpd_helper_strncmp_ext (const char *s1, const char *s2); -- 2.25.1 From 1ec3017582e9ad216cef5b55172d2233adfd3fb1 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Thu, 2 Nov 2017 14:56:44 +0800 Subject: [PATCH 22/65] kmssink: support imx8 platform support HDR10 video playback for 8mq support hantro tile format for 8mq support video playback for 8qxp/8qm --- sys/kms/gstkmsallocator.c | 32 +- sys/kms/gstkmsallocator.h | 1 + sys/kms/gstkmssink.c | 686 ++++++++++++++++++++++++++++++++++++-- sys/kms/gstkmssink.h | 32 ++ sys/kms/gstkmsutils.c | 38 +++ sys/kms/gstkmsutils.h | 1 + 6 files changed, 754 insertions(+), 36 deletions(-) diff --git a/sys/kms/gstkmsallocator.c b/sys/kms/gstkmsallocator.c index 3ec2fdb32d..c7ecd22b42 100644 --- a/sys/kms/gstkmsallocator.c +++ b/sys/kms/gstkmsallocator.c @@ -33,6 +33,7 @@ #include #include #include +#include /* it needs to be below because is internal to libdrm */ #include @@ -347,14 +348,15 @@ gst_kms_allocator_new (int fd) * which are relative to the GstBuffer start. */ static gboolean gst_kms_allocator_add_fb (GstKMSAllocator * alloc, GstKMSMemory * kmsmem, - gsize in_offsets[GST_VIDEO_MAX_PLANES], GstVideoInfo * vinfo, + gint64 drm_modifier, gsize in_offsets[GST_VIDEO_MAX_PLANES], GstVideoInfo * vinfo, guint32 bo_handles[4]) { gint i, ret; gint num_planes = GST_VIDEO_INFO_N_PLANES (vinfo); - guint32 w, h, fmt; + guint32 w, h, fmt, alignment; guint32 pitches[4] = { 0, }; guint32 offsets[4] = { 0, }; + guint64 modifier[4] = { 0, }; if (kmsmem->fb_id) return TRUE; @@ -362,17 +364,33 @@ gst_kms_allocator_add_fb (GstKMSAllocator * alloc, GstKMSMemory * kmsmem, w = GST_VIDEO_INFO_WIDTH (vinfo); h = GST_VIDEO_INFO_HEIGHT (vinfo); fmt = gst_drm_format_from_video (GST_VIDEO_INFO_FORMAT (vinfo)); + alignment = gst_drm_alignment_from_drm_format (fmt); + w = GST_ROUND_UP_N (w, alignment); + h = GST_ROUND_UP_N (h, alignment); for (i = 0; i < num_planes; i++) { pitches[i] = GST_VIDEO_INFO_PLANE_STRIDE (vinfo, i); offsets[i] = in_offsets[i]; + modifier[i] = drm_modifier; } GST_DEBUG_OBJECT (alloc, "bo handles: %d, %d, %d, %d", bo_handles[0], bo_handles[1], bo_handles[2], bo_handles[3]); + if (drm_modifier == DRM_FORMAT_MOD_AMPHION_TILED) { + /* for qxp, dpu need fb width align to 8 + * and height align to 256 */ + w = GST_ROUND_UP_8 (w); + h = GST_ROUND_UP_N (h, 256); + } + + if (drm_modifier) { + ret = drmModeAddFB2WithModifiers (alloc->priv->fd, w, h, fmt, bo_handles, pitches, + offsets, modifier, &kmsmem->fb_id, DRM_MODE_FB_MODIFIERS); + } else { + ret = drmModeAddFB2 (alloc->priv->fd, w, h, fmt, bo_handles, pitches, + offsets, &kmsmem->fb_id, 0); + } - ret = drmModeAddFB2 (alloc->priv->fd, w, h, fmt, bo_handles, pitches, - offsets, &kmsmem->fb_id, 0); if (ret) { GST_ERROR_OBJECT (alloc, "Failed to bind to framebuffer: %s (%d)", g_strerror (errno), errno); @@ -408,7 +426,7 @@ gst_kms_allocator_bo_alloc (GstAllocator * allocator, GstVideoInfo * vinfo) for (i = 0; i < GST_VIDEO_INFO_N_PLANES (vinfo); i++) bo_handle[i] = gst_drm_dumb_memory_get_handle (kmsmem->bo); - if (!gst_kms_allocator_add_fb (alloc, kmsmem, vinfo->offset, vinfo, + if (!gst_kms_allocator_add_fb (alloc, kmsmem, 0, vinfo->offset, vinfo, bo_handle)) goto fail; @@ -422,7 +440,7 @@ fail: GstKMSMemory * gst_kms_allocator_dmabuf_import (GstAllocator * allocator, gint * prime_fds, - gint n_planes, gsize offsets[GST_VIDEO_MAX_PLANES], GstVideoInfo * vinfo) + gint n_planes, gint64 drm_modifier, gsize offsets[GST_VIDEO_MAX_PLANES], GstVideoInfo * vinfo) { GstKMSAllocator *alloc; GstKMSMemory *kmsmem; @@ -445,7 +463,7 @@ gst_kms_allocator_dmabuf_import (GstAllocator * allocator, gint * prime_fds, goto import_fd_failed; } - if (!gst_kms_allocator_add_fb (alloc, kmsmem, offsets, vinfo, gem_handle)) + if (!gst_kms_allocator_add_fb (alloc, kmsmem, drm_modifier, offsets, vinfo, gem_handle)) goto failed; done: diff --git a/sys/kms/gstkmsallocator.h b/sys/kms/gstkmsallocator.h index 258efbaba9..f518282592 100644 --- a/sys/kms/gstkmsallocator.h +++ b/sys/kms/gstkmsallocator.h @@ -84,6 +84,7 @@ GstMemory* gst_kms_allocator_bo_alloc (GstAllocator *allocator, GstKMSMemory* gst_kms_allocator_dmabuf_import (GstAllocator *allocator, gint *prime_fds, gint n_planes, + gint64 drm_modifier, gsize offsets[GST_VIDEO_MAX_PLANES], GstVideoInfo *vinfo); diff --git a/sys/kms/gstkmssink.c b/sys/kms/gstkmssink.c index 20716b6d9c..52d10b2461 100644 --- a/sys/kms/gstkmssink.c +++ b/sys/kms/gstkmssink.c @@ -51,12 +51,17 @@ #include #include #include +#include +#include + +#include -#include #include #include #include #include +#include +#include #include "gstkmssink.h" #include "gstkmsutils.h" @@ -80,6 +85,8 @@ static GstFlowReturn gst_kms_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf); static void gst_kms_sink_video_overlay_init (GstVideoOverlayInterface * iface); static void gst_kms_sink_drain (GstKMSSink * self); +static void ensure_kms_allocator (GstKMSSink * self); +static void gst_kms_sink_config_hdr10 (GstKMSSink *self, const GstCaps * caps); #define parent_class gst_kms_sink_parent_class G_DEFINE_TYPE_WITH_CODE (GstKMSSink, gst_kms_sink, GST_TYPE_VIDEO_SINK, @@ -106,11 +113,15 @@ enum PROP_PLANE_PROPS, PROP_FD, PROP_SKIP_VSYNC, - PROP_N, + PROP_GLOBAL_ALPHA, + PROP_FORCE_HANTROTILE, + PROP_N }; static GParamSpec *g_properties[PROP_N] = { NULL, }; +#define SCALE_RATIO_NO_LIMITATION 100000 + #ifdef HAVE_DRM_HDR enum hdmi_metadata_type { @@ -485,6 +496,7 @@ kms_open (gchar ** driver) "exynos", "amdgpu", "imx-drm", "imx-lcdif", "rockchip", "atmel-hlcdc", "msm", "xlnx", "vc4", "meson", "stm", "sun4i-drm", "mxsfb-drm", "tegra", "tidss", "xilinx_drm", /* DEPRECATED. Replaced by xlnx */ + "imx-dcss", /* 8mq dcss device name on L5.4 */ }; int i, fd = -1; @@ -723,6 +735,50 @@ get_drm_caps (GstKMSSink * self) return TRUE; } +static gint +get_commit_fd (GstKMSSink * self) +{ +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) + return self->ctrl_fd; +#else + return self->fd; +#endif +} + +static guint +check_upscale (GstKMSSink * self, guint32 fb_id) { + guint32 src_w = self->hdisplay / 10; + guint32 src_h = self->vdisplay / 10; + guint ratio; + gint fd = get_commit_fd(self); + + for (ratio = 10; ratio > 0; ratio--) { + if (!drmModeSetPlane (fd, self->plane_id, self->crtc_id, fb_id, 0, + 0, 0, src_w * ratio, src_h * ratio, + 0, 0, src_w << 16, src_h << 16)) + break; + } + + return ratio; +} + +static guint +check_downscale (GstKMSSink * self, guint32 fb_id) { + guint32 src_w = self->hdisplay / 10; + guint32 src_h = self->vdisplay / 10; + guint ratio; + gint fd = get_commit_fd(self); + + for (ratio = 10; ratio > 0; ratio--) { + if (!drmModeSetPlane (fd, self->plane_id, self->crtc_id, fb_id, 0, + 0, 0, src_w, src_h, + 0, 0, (src_w * ratio) << 16, (src_h * ratio) << 16)) + break; + } + + return ratio; +} + static void ensure_kms_allocator (GstKMSSink * self) { @@ -731,6 +787,81 @@ ensure_kms_allocator (GstKMSSink * self) self->allocator = gst_kms_allocator_new (self->fd); } +static void +check_scaleable (GstKMSSink * self) +{ + guint32 fb_id; + GstKMSMemory *kmsmem = NULL; + GstMemory *mem = NULL; + GstVideoInfo vinfo; + + /* we assume driver can scale at initialize, + * if scale is checked or can not scale, we + * don't need check again */ + if (self->scale_checked || !self->can_scale) + return; + + if (self->conn_id < 0 || !self->display_connected) + return; + + /* FIXME: for dpu, we can only hard code the scale ratio, + * dpu has no limitation when do upscale but can not support + * downscale */ + if (HAS_DPU()) { + self->downscale_ratio = 1; + self->upscale_ratio = SCALE_RATIO_NO_LIMITATION; + return; + } + + gst_video_info_init (&vinfo); + gst_video_info_set_format (&vinfo, GST_VIDEO_FORMAT_NV12, self->hdisplay, self->vdisplay); + + ensure_kms_allocator (self); + + kmsmem = (GstKMSMemory *) gst_kms_allocator_bo_alloc (self->allocator, &vinfo); + if (!kmsmem) + return; + + fb_id = kmsmem->fb_id; + + GST_INFO_OBJECT (self, "checking scaleable"); + self->downscale_ratio = check_downscale (self, fb_id); + self->upscale_ratio = check_upscale (self, fb_id); + + GST_INFO_OBJECT (self, "got scale ratio: up (%d) down (%d)", + self->upscale_ratio, self->downscale_ratio); + + self->scale_checked = TRUE; + mem = (GstMemory *)kmsmem; + g_clear_pointer (&mem, gst_memory_unref); +} + +static gboolean +check_vsi_tile_enable (GstKMSSink * self, GstBuffer * buffer) +{ + GstDmabufMeta *dmabuf_meta; + gint64 drm_modifier = 0; + + if (!buffer) + buffer = self->hold_buf[0]; + + if (!buffer) + return FALSE; + + if (!gst_is_dmabuf_memory (gst_buffer_peek_memory (buffer, 0))) + return FALSE; + + dmabuf_meta = gst_buffer_get_dmabuf_meta (buffer); + if (dmabuf_meta) + drm_modifier = dmabuf_meta->drm_modifier; + + GST_INFO_OBJECT (self, "buffer modifier type %"G_GINT64_FORMAT, drm_modifier); + + return drm_modifier == DRM_FORMAT_MOD_VSI_G1_TILED + || drm_modifier == DRM_FORMAT_MOD_VSI_G2_TILED + || drm_modifier == DRM_FORMAT_MOD_VSI_G2_TILED_COMPRESSED; +} + static gboolean configure_mode_setting (GstKMSSink * self, GstVideoInfo * vinfo) { @@ -830,7 +961,7 @@ ensure_allowed_caps (GstKMSSink * self, drmModeConnector * conn, if (!out_caps) return FALSE; - if (conn && self->modesetting_enabled) + if (conn && self->modesetting_enabled && self->display_connected) count_modes = conn->count_modes; else count_modes = 1; @@ -872,6 +1003,15 @@ ensure_allowed_caps (GstKMSSink * self, drmModeConnector * conn, tmp_caps = gst_caps_merge (tmp_caps, caps); } + /* FIXME: Add NV12_10LE40 caps here, no need this code + * when new drm fourcc added*/ + caps = gst_caps_new_simple ("video/x-raw", + "format", G_TYPE_STRING, "NV12_10LE40", + "width", GST_TYPE_INT_RANGE, res->min_width, res->max_width, + "height", GST_TYPE_INT_RANGE, res->min_height, res->max_height, + "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); + + tmp_caps = gst_caps_merge (tmp_caps, caps); out_caps = gst_caps_merge (out_caps, gst_caps_simplify (tmp_caps)); } @@ -890,6 +1030,23 @@ ensure_allowed_caps (GstKMSSink * self, drmModeConnector * conn, return TRUE; } +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) +static gint +get_drm_minor_base (gint type) +{ + switch (type) { + case DRM_NODE_PRIMARY: + return 0; + case DRM_NODE_CONTROL: + return 64; + case DRM_NODE_RENDER: + return 128; + default: + return -1; + } +} +#endif + static gboolean set_drm_property (gint fd, guint32 object, guint32 object_type, drmModeObjectPropertiesPtr properties, const gchar * prop_name, @@ -1043,6 +1200,21 @@ gst_kms_sink_start (GstBaseSink * bsink) self->fd = kms_open (&self->devname); } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) + gint minor; + minor = get_drm_minor_base (DRM_NODE_CONTROL); + self->ctrl_fd = drmOpenControl(minor); + if (self->ctrl_fd < 0) { + if (self->fd >= 0) + drmClose (self->fd); + GST_ELEMENT_ERROR (self, RESOURCE, OPEN_READ_WRITE, + ("Could not open DRM ctrl node %s", GST_STR_NULL (self->devname)), + ("reason: %s (%d)", strerror (errno), errno)); + return FALSE; + } +#endif + if (self->fd < 0) goto open_failed; @@ -1061,11 +1233,19 @@ gst_kms_sink_start (GstBaseSink * bsink) if (!conn) goto connector_failed; + if (conn->connection == DRM_MODE_CONNECTED) + self->display_connected = TRUE; + else + self->display_connected = FALSE; + + GST_DEBUG_OBJECT (self, "display connection status: %s", + self->display_connected ? "Connected" : "disconnected"); + crtc = find_crtc_for_connector (self->fd, res, conn, &self->pipe); if (!crtc) goto crtc_failed; - if (!crtc->mode_valid || self->modesetting_enabled) { + if ((!crtc->mode_valid || self->modesetting_enabled) && self->display_connected) { GST_DEBUG_OBJECT (self, "enabling modesetting"); self->modesetting_enabled = TRUE; universal_planes = TRUE; @@ -1123,6 +1303,8 @@ retry_find_plane: GST_INFO_OBJECT (self, "display size: pixels = %dx%d / millimeters = %dx%d", self->hdisplay, self->vdisplay, self->mm_width, self->mm_height); + check_scaleable (self); + self->pollfd.fd = self->fd; gst_poll_add_fd (self->poll, &self->pollfd); gst_poll_fd_ctl_read (self->poll, &self->pollfd, TRUE); @@ -1261,6 +1443,13 @@ gst_kms_sink_stop (GstBaseSink * bsink) self->fd = -1; } +#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0) + if (self->ctrl_fd >= 0) { + drmClose (self->ctrl_fd); + self->ctrl_fd = -1; + } +#endif + GST_OBJECT_LOCK (bsink); self->hdisplay = 0; self->vdisplay = 0; @@ -1466,6 +1655,8 @@ gst_kms_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) if (self->modesetting_enabled && !configure_mode_setting (self, &vinfo)) goto modesetting_failed; + check_scaleable (self); + GST_OBJECT_LOCK (self); if (self->reconfigure) { self->reconfigure = FALSE; @@ -1473,6 +1664,8 @@ gst_kms_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) } GST_OBJECT_UNLOCK (self); + gst_kms_sink_config_hdr10 (self, caps); + GST_DEBUG_OBJECT (self, "negotiated caps = %" GST_PTR_FORMAT, caps); return TRUE; @@ -1515,6 +1708,10 @@ gst_kms_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) gboolean need_pool; GstVideoInfo vinfo; GstBufferPool *pool; + guint64 drm_modifier; + drmModeObjectPropertiesPtr props = NULL; + drmModePropertyPtr prop = NULL; + guint i; gsize size; self = GST_KMS_SINK (bsink); @@ -1552,6 +1749,27 @@ gst_kms_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); gst_query_add_allocation_meta (query, GST_VIDEO_CROP_META_API_TYPE, NULL); + drm_modifier = DRM_FORMAT_MOD_AMPHION_TILED; + gst_query_add_allocation_dmabuf_meta (query, drm_modifier); + + if (self->hantro_tile_enabled) { + props = drmModeObjectGetProperties (self->fd, self->plane_id, DRM_MODE_OBJECT_PLANE); + for (i = 0; i < props->count_props; ++i) { + prop = drmModeGetProperty(self->fd, props->props[i]); + if (!strcmp(prop->name, "dtrc_table_ofs")) { + GST_DEBUG ("has dtrc_table_ofs property, can support VSI tile format"); + drm_modifier = DRM_FORMAT_MOD_VSI_G1_TILED; + gst_query_add_allocation_dmabuf_meta (query, drm_modifier); + drm_modifier = DRM_FORMAT_MOD_VSI_G2_TILED; + gst_query_add_allocation_dmabuf_meta (query, drm_modifier); + drm_modifier = DRM_FORMAT_MOD_VSI_G2_TILED_COMPRESSED; + gst_query_add_allocation_dmabuf_meta (query, drm_modifier); + } + drmModeFreeProperty (prop); + prop = NULL; + } + } + return TRUE; /* ERRORS */ @@ -1655,6 +1873,8 @@ gst_kms_sink_import_dmabuf (GstKMSSink * self, GstBuffer * inbuf, { gint prime_fds[GST_VIDEO_MAX_PLANES] = { 0, }; GstVideoMeta *meta; + GstDmabufMeta *dmabuf_meta; + gint64 drm_modifier = 0; guint i, n_mem, n_planes; GstKMSMemory *kmsmem; guint mems_idx[GST_VIDEO_MAX_PLANES]; @@ -1671,6 +1891,9 @@ gst_kms_sink_import_dmabuf (GstKMSSink * self, GstBuffer * inbuf, n_planes = GST_VIDEO_INFO_N_PLANES (&self->vinfo); n_mem = gst_buffer_n_memory (inbuf); meta = gst_buffer_get_video_meta (inbuf); + dmabuf_meta = gst_buffer_get_dmabuf_meta (inbuf); + if (dmabuf_meta) + drm_modifier = dmabuf_meta->drm_modifier; GST_TRACE_OBJECT (self, "Found a dmabuf with %u planes and %u memories", n_planes, n_mem); @@ -1727,7 +1950,7 @@ gst_kms_sink_import_dmabuf (GstKMSSink * self, GstBuffer * inbuf, prime_fds[1], prime_fds[2], prime_fds[3]); kmsmem = gst_kms_allocator_dmabuf_import (self->allocator, - prime_fds, n_planes, mems_skip, &self->vinfo); + prime_fds, n_planes, drm_modifier, mems_skip, &self->vinfo); if (!kmsmem) return FALSE; @@ -1785,7 +2008,156 @@ activate_pool_failed: gst_object_unref (pool); return FALSE; } +} + +static void +gst_kms_sink_set_kmsproperty (GstKMSSink * self, guint alpha, guint64 dtrc_table_ofs) +{ + drmModeRes *res = NULL; + drmModePlaneRes *pres = NULL; + drmModePlane *plane = NULL; + drmModeObjectPropertiesPtr props = NULL; + drmModePropertyPtr prop = NULL; + guint i; + gint fd = get_commit_fd (self); + + props = drmModeObjectGetProperties (self->fd, self->plane_id, DRM_MODE_OBJECT_PLANE); + for (i = 0; i < props->count_props; ++i) { + prop = drmModeGetProperty(self->fd, props->props[i]); + if (!strcmp(prop->name, "dtrc_table_ofs") && dtrc_table_ofs) { + GST_DEBUG ("set DTRC table offset %"G_GUINT64_FORMAT" to primary plane %d property %d", + dtrc_table_ofs, self->plane_id, prop->prop_id); + drmModeObjectSetProperty (fd, self->plane_id, DRM_MODE_OBJECT_PLANE, prop->prop_id, dtrc_table_ofs); + } + drmModeFreeProperty (prop); + prop = NULL; + } + + res = drmModeGetResources (self->fd); + if (!res) + goto out; + + drmSetClientCap (self->fd, DRM_CLIENT_CAP_UNIVERSAL_PLANES, 1); + + pres = drmModeGetPlaneResources (self->fd); + if (!pres) + goto out; + + plane = find_plane_for_crtc (self->fd, res, pres, self->crtc_id); + if (!plane) + goto out; + + props = drmModeObjectGetProperties (self->fd, plane->plane_id, DRM_MODE_OBJECT_PLANE); + for (i = 0; i < props->count_props; ++i) { + prop = drmModeGetProperty(self->fd, props->props[i]); + if (!strcmp(prop->name, "alpha")) { + GST_DEBUG ("set global alpha %d to primary plane %d property %d", + alpha, plane->plane_id, prop->prop_id); + drmModeObjectSetProperty (fd, plane->plane_id, DRM_MODE_OBJECT_PLANE, prop->prop_id, alpha); + self->primary_plane_id = plane->plane_id; + } + drmModeFreeProperty (prop); + prop = NULL; + } + +out: + if (res) + drmModeFreeResources (res); + if (pres) + drmModeFreePlaneResources (pres); + if (plane) + drmModeFreePlane (plane); + if (props) + drmModeFreeObjectProperties (props); + drmSetClientCap (self->fd, DRM_CLIENT_CAP_UNIVERSAL_PLANES, 0); +} + +static GstStateChangeReturn +gst_kms_sink_change_state (GstElement * element, GstStateChange transition) +{ + GstKMSSink *self; + GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + gint fd; + + GST_DEBUG ("changing state: %s => %s", + gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)), + gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition))); + + self = GST_KMS_SINK (element); + fd = get_commit_fd (self); + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: + self->is_kmsproperty_set = FALSE; + memset (&self->hdr10meta, 0, sizeof (self->hdr10meta)); + break; + case GST_STATE_CHANGE_READY_TO_PAUSED: + { + guint i; + for (i = 0; i < DEFAULT_HOLD_BUFFER_NUM; i++) + self->hold_buf[i] = NULL; + break; + } + case GST_STATE_CHANGE_PAUSED_TO_PLAYING: + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + case GST_STATE_CHANGE_PLAYING_TO_PAUSED: + { + self->run_time = gst_element_get_start_time (element); + break; + } + case GST_STATE_CHANGE_PAUSED_TO_READY: + { + guint i; + gst_kms_sink_set_kmsproperty (self, 255, 0); + for (i = 0; i < DEFAULT_HOLD_BUFFER_NUM; i++) { + if (self->hold_buf[i]) + gst_buffer_unref (self->hold_buf[i]); + } +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + if (self->hdr10meta.eotf != 0) { +#else + if (self->hdr10meta.hdmi_metadata_type1.eotf != 0) { +#endif + guint blob_id = 0; + gint err = 0; +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + self->hdr10meta.eotf = 0; + drmModeCreatePropertyBlob (self->fd, &self->hdr10meta, 1, &blob_id); + err = drmModeObjectSetProperty (fd, self->conn_id, DRM_MODE_OBJECT_CONNECTOR, self->hdr_prop_id, blob_id); + drmModeDestroyPropertyBlob (self->fd, blob_id); +#else + err = drmModeObjectSetProperty (fd, self->conn_id, DRM_MODE_OBJECT_CONNECTOR, self->hdr_prop_id, blob_id); +#endif + if (err) + GST_ERROR_OBJECT (self, "reset blob property fail %d", err); + } + + break; + } + case GST_STATE_CHANGE_READY_TO_NULL: + if (self->run_time > 0) { + g_print ("Total showed frames (%"G_GUINT64_FORMAT"), playing for (%"GST_TIME_FORMAT"), fps (%.3f).\n", + self->frame_showed, GST_TIME_ARGS (self->run_time), + (gfloat)GST_SECOND * self->frame_showed / self->run_time); + } + + self->frame_showed = 0; + self->run_time = 0; + break; + default: + break; + } + return ret; } static GstBuffer * @@ -1877,6 +2249,143 @@ done: return buf; } +static void +gst_kms_sink_config_hdr10 (GstKMSSink *self, const GstCaps * caps) +{ + guint blob_id = 0; + int err; + gint i; + drmModeObjectPropertiesPtr props = NULL; + drmModePropertyPtr prop = NULL; + GstVideoMasteringDisplayInfo minfo; + GstVideoContentLightLevel cll; + gint fd = get_commit_fd (self); +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + const gchar *prop_name = "HDR_SOURCE_METADATA"; +#else + const gchar *prop_name = "HDR_OUTPUT_METADATA"; +#endif + + if (self->conn_id < 0) { + GST_ERROR_OBJECT (self, "no connector"); + return; + } + + gst_video_mastering_display_info_init (&minfo); + gst_video_content_light_level_init (&cll); + + if (caps) { + if (!gst_video_mastering_display_info_from_caps (&minfo, caps) + || !gst_video_content_light_level_from_caps (&cll, caps)) { + GST_INFO_OBJECT (self, "no HDR metadata present in caps"); + return; + } + } + +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + if (self->hdr10meta.eotf == 0) { +#else + if (self->hdr10meta.hdmi_metadata_type1.eotf == 0) { +#endif + GST_INFO_OBJECT (self, "redPrimary x=%d y=%d", minfo.display_primaries[0].x, + minfo.display_primaries[0].y); + GST_INFO_OBJECT (self, "greenPrimary x=%d y=%d", + minfo.display_primaries[1].x, minfo.display_primaries[1].y); + GST_INFO_OBJECT (self, "bluePrimary x=%d y=%d", + minfo.display_primaries[2].x, minfo.display_primaries[2].y); + GST_INFO_OBJECT (self, "whitePoint x=%d y=%d", minfo.white_point.x, + minfo.white_point.y); + GST_INFO_OBJECT (self, "maxMasteringLuminance %d", + minfo.max_display_mastering_luminance); + GST_INFO_OBJECT (self, "minMasteringLuminance %d", + minfo.min_display_mastering_luminance); + GST_INFO_OBJECT (self, "maxContentLightLevel %d", + cll.max_content_light_level); + GST_INFO_OBJECT (self, "maxFrameAverageLightLevel %d", + cll.max_frame_average_light_level); + + /* FIXME: better to use marcos instead of const value */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + self->hdr10meta.eotf = 2; + self->hdr10meta.type = 0; + self->hdr10meta.display_primaries_x [0] = minfo.display_primaries[0].x; + self->hdr10meta.display_primaries_x [1] = minfo.display_primaries[1].x; + self->hdr10meta.display_primaries_x [2] = minfo.display_primaries[2].x; + self->hdr10meta.display_primaries_y [0] = minfo.display_primaries[0].y; + self->hdr10meta.display_primaries_y [1] = minfo.display_primaries[1].y; + self->hdr10meta.display_primaries_y [2] = minfo.display_primaries[2].y; + self->hdr10meta.white_point_x = minfo.white_point.x; + self->hdr10meta.white_point_y = minfo.white_point.y; + self->hdr10meta.max_mastering_display_luminance = (minfo.max_display_mastering_luminance / 10000) & 0xffff;; + self->hdr10meta.min_mastering_display_luminance = minfo.min_display_mastering_luminance & 0xffff; + self->hdr10meta.max_fall = cll.max_frame_average_light_level; + self->hdr10meta.max_cll = cll.max_content_light_level; +#else + self->hdr10meta.metadata_type = 0; + self->hdr10meta.hdmi_metadata_type1.eotf = 2; + self->hdr10meta.hdmi_metadata_type1.metadata_type = 1; + self->hdr10meta.hdmi_metadata_type1.display_primaries[0].x = minfo.display_primaries[0].x; + self->hdr10meta.hdmi_metadata_type1.display_primaries[0].y = minfo.display_primaries[0].y; + self->hdr10meta.hdmi_metadata_type1.display_primaries[1].x = minfo.display_primaries[1].x; + self->hdr10meta.hdmi_metadata_type1.display_primaries[1].y = minfo.display_primaries[1].y; + self->hdr10meta.hdmi_metadata_type1.display_primaries[2].x = minfo.display_primaries[2].x; + self->hdr10meta.hdmi_metadata_type1.display_primaries[2].y = minfo.display_primaries[2].y; + self->hdr10meta.hdmi_metadata_type1.white_point.x = minfo.white_point.x; + self->hdr10meta.hdmi_metadata_type1.white_point.y = minfo.white_point.y; + self->hdr10meta.hdmi_metadata_type1.max_display_mastering_luminance = + (minfo.max_display_mastering_luminance / 10000) & 0xffff; + self->hdr10meta.hdmi_metadata_type1.min_display_mastering_luminance = + minfo.min_display_mastering_luminance & 0xffff; + self->hdr10meta.hdmi_metadata_type1.max_cll = cll.max_content_light_level; + self->hdr10meta.hdmi_metadata_type1.max_fall = cll.max_frame_average_light_level; +#endif + + if (!self->hdr_prop_id) { + props = drmModeObjectGetProperties (self->fd, self->conn_id, DRM_MODE_OBJECT_CONNECTOR); + for (i = 0; i < props->count_props; ++i) { + prop = drmModeGetProperty(self->fd, props->props[i]); + if (!strcmp(prop->name, prop_name)) { + GST_DEBUG_OBJECT (self, "found %s property on connector %d property id %d", prop_name, + self->conn_id, prop->prop_id); + self->hdr_prop_id = prop->prop_id; + } + drmModeFreeProperty (prop); + prop = NULL; + } + } + + if (self->hdr_prop_id == 0) { + GST_WARNING_OBJECT (self, "no %s property found", prop_name); + return; + } + + drmModeCreatePropertyBlob (self->fd, &self->hdr10meta, sizeof (self->hdr10meta), &blob_id); + GST_INFO_OBJECT (self, "create blob id %d", blob_id); + err = drmModeObjectSetProperty (fd, self->conn_id, DRM_MODE_OBJECT_CONNECTOR, self->hdr_prop_id, blob_id); + drmModeDestroyPropertyBlob (self->fd, blob_id); + if (err) { + GST_ERROR_OBJECT (self, "set blob property fail %d", err); + return; + } + } +} + +static gboolean +gst_kms_sink_check_scale_ratio (GstKMSSink * self, GstVideoRectangle dst, GstVideoRectangle src) +{ + gboolean can_scale = TRUE; + GST_INFO_OBJECT (self, "dst rectangle (%d, %d)-(%d x %d)", dst.x, dst.y, dst.w, dst.h); + GST_INFO_OBJECT (self, "src rectangle (%d, %d)-(%d x %d)", src.x, src.y, src.w, src.h); + + can_scale = (dst.w * self->downscale_ratio >= src.w + && dst.w <= src.w * self->upscale_ratio + && dst.h * self->downscale_ratio >= src.h + && dst.h <= src.h * self->upscale_ratio); + + GST_INFO_OBJECT (self, "can use hardware scale: %s", can_scale ? "TRUE" : "FALSE"); + return can_scale; +} + static GstFlowReturn gst_kms_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf) { @@ -1890,12 +2399,30 @@ gst_kms_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf) gint video_width, video_height; GstVideoRectangle dst = { 0, }; GstVideoRectangle result; + GstPhyMemMeta *phymemmeta = NULL; + guint64 dtrc_table_ofs; GstFlowReturn res; - + gboolean can_scale = TRUE; + guint32 fmt, alignment; + gint fd; + self = GST_KMS_SINK (vsink); + fd = get_commit_fd (self); res = GST_FLOW_ERROR; + if (!self->display_connected) { + GST_WARNING_OBJECT (self, "display not connected, drop this buffer"); + return GST_FLOW_OK; + } + + if (HAS_DPU()) { + fmt = gst_drm_format_from_video (GST_VIDEO_INFO_FORMAT (&self->vinfo)); + alignment = gst_drm_alignment_from_drm_format (fmt); + } else { + alignment = 1; + } + if (buf) { buffer = gst_kms_sink_get_input_buffer (self, buf); vinfo = &self->vinfo; @@ -1913,12 +2440,27 @@ gst_kms_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf) if (!buffer) return GST_FLOW_ERROR; + fb_id = gst_kms_memory_get_fb_id (gst_buffer_peek_memory (buffer, 0)); if (fb_id == 0) goto buffer_invalid; GST_TRACE_OBJECT (self, "displaying fb %d", fb_id); + if (!self->is_kmsproperty_set) { + phymemmeta = GST_PHY_MEM_META_GET (buffer); + if (phymemmeta) { + GST_DEBUG_OBJECT (self, "physical memory meta x_padding: %d y_padding: %d \ + RFC luma offset: %d chroma offset: %d", + phymemmeta->x_padding, phymemmeta->y_padding, phymemmeta->rfc_luma_offset, phymemmeta->rfc_chroma_offset); + dtrc_table_ofs = phymemmeta->rfc_luma_offset | ((guint64)phymemmeta->rfc_chroma_offset << 32); + gst_kms_sink_set_kmsproperty (self, self->global_alpha, dtrc_table_ofs); + } else + gst_kms_sink_set_kmsproperty (self, self->global_alpha, 0); + + self->is_kmsproperty_set = TRUE; + } + GST_OBJECT_LOCK (self); if (self->modesetting_enabled) { self->buffer_id = fb_id; @@ -1943,35 +2485,61 @@ gst_kms_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf) dst.h = self->render_rect.h; retry_set_plane: - gst_video_sink_center_rect (src, dst, &result, self->can_scale); + gst_video_sink_center_rect (src, dst, &result, can_scale); - result.x += self->render_rect.x; - result.y += self->render_rect.y; + result.x = GST_ROUND_DOWN_N (result.x + self->render_rect.x, alignment); + result.y = GST_ROUND_DOWN_N (result.y + self->render_rect.y, alignment);; + + if (result.x < 0 || result.y < 0) { + /* FIXME: need improve cropping handle when DTRC is not enable */ + if (!check_vsi_tile_enable (self, buf)) { + result.x = result.x < 0 ? 0 : result.x; + result.y = result.y < 0 ? 0 : result.y; + } + } if (crop) { src.w = crop->width; src.h = crop->height; } else { - src.w = video_width; - src.h = video_height; + src.w = GST_ROUND_DOWN_N (video_width, alignment); + src.h = GST_ROUND_DOWN_N (video_height, alignment); } - /* handle out of screen case */ - if ((result.x + result.w) > self->hdisplay) - result.w = self->hdisplay - result.x; + if (!gst_kms_sink_check_scale_ratio (self, result, src)) { + if (can_scale) { + can_scale = FALSE; + dst.w = MAX (self->hdisplay, src.w); + dst.h = MAX (self->vdisplay, src.h); + GST_WARNING_OBJECT (self, "try not scale"); + goto retry_set_plane; + } else + goto check_ratio_fail; + } - if ((result.y + result.h) > self->vdisplay) - result.h = self->vdisplay - result.y; + GST_TRACE_OBJECT (self, + "scaling result at (%i,%i) %ix%i sourcing at (%i,%i) %ix%i", + result.x, result.y, result.w, result.h, src.x, src.y, src.w, src.h); - if (result.w <= 0 || result.h <= 0) { - GST_WARNING_OBJECT (self, "video is out of display range"); - goto sync_frame; + /* handle out of screen case */ + if ((result.x + result.w) > self->hdisplay) { + gint crop_width = self->hdisplay - result.x; + if (crop_width > 0) + src.w = GST_ROUND_UP_2 (crop_width * src.w / result.w); + result.w = crop_width; } - /* to make sure it can be show when driver don't support scale */ - if (!self->can_scale) { - src.w = result.w; - src.h = result.h; + if ((result.y + result.h) > self->vdisplay) { + gint crop_height = self->vdisplay - result.y; + if (crop_height > 0) + src.h = GST_ROUND_UP_2 (crop_height * src.h / result.h); + result.h = crop_height; + } + + if (result.w <= 0 || result.h <= 0 || src.h <= 0 || src.w <= 0) { + GST_WARNING_OBJECT (self, "video is out of display range, use previous area"); + self->render_rect = self->last_rect; + goto done; } #ifdef HAVE_DRM_HDR /* Send the HDR infoframes if appropriate */ @@ -1982,17 +2550,14 @@ retry_set_plane: "drmModeSetPlane at (%i,%i) %ix%i sourcing at (%i,%i) %ix%i", result.x, result.y, result.w, result.h, src.x, src.y, src.w, src.h); - ret = drmModeSetPlane (self->fd, self->plane_id, self->crtc_id, fb_id, 0, + ret = drmModeSetPlane (fd, self->plane_id, self->crtc_id, fb_id, 0, result.x, result.y, result.w, result.h, /* source/cropping coordinates are given in Q16 */ src.x << 16, src.y << 16, src.w << 16, src.h << 16); if (ret) { - if (self->can_scale) { - self->can_scale = FALSE; - goto retry_set_plane; - } goto set_plane_failed; - } + } else + goto done; sync_frame: /* Wait for the previous frame to complete redraw */ @@ -2002,6 +2567,7 @@ sync_frame: } /* Save the rendered buffer and its metadata in case a redraw is needed */ +done: if (buffer != self->last_buffer) { gst_buffer_replace (&self->last_buffer, buffer); self->last_width = GST_VIDEO_SINK_WIDTH (self); @@ -2013,7 +2579,22 @@ sync_frame: GST_OBJECT_UNLOCK (self); res = GST_FLOW_OK; + self->frame_showed++; + self->last_rect = self->render_rect; + bail: + if (buf) { + guint i; + + if (self->hold_buf[DEFAULT_HOLD_BUFFER_NUM-1]) + gst_buffer_unref (self->hold_buf[DEFAULT_HOLD_BUFFER_NUM-1]); + + for (i = DEFAULT_HOLD_BUFFER_NUM - 1; i > 0; i--) + self->hold_buf[i] = self->hold_buf[i-1]; + + self->hold_buf[0] = gst_buffer_ref (buf); + } + gst_buffer_unref (buffer); return res; @@ -2041,6 +2622,13 @@ no_disp_ratio: ("Error calculating the output display ratio of the video.")); goto bail; } +check_ratio_fail: + { + GST_OBJECT_UNLOCK (self); + GST_ELEMENT_ERROR (self, RESOURCE, FAILED, (NULL), + ("Checking scale ratio fail.")); + goto bail; + } } static void @@ -2200,6 +2788,12 @@ gst_kms_sink_set_property (GObject * object, guint prop_id, case PROP_SKIP_VSYNC: sink->skip_vsync = g_value_get_boolean (value); break; + case PROP_GLOBAL_ALPHA: + sink->global_alpha = g_value_get_int (value); + break; + case PROP_FORCE_HANTROTILE: + sink->hantro_tile_enabled = g_value_get_boolean (value); + break; default: if (!gst_video_overlay_set_property (object, PROP_N, prop_id, value)) G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -2252,6 +2846,11 @@ gst_kms_sink_get_property (GObject * object, guint prop_id, break; case PROP_PLANE_PROPS: gst_value_set_structure (value, sink->plane_props); + case PROP_GLOBAL_ALPHA: + g_value_set_int (value, sink->global_alpha); + break; + case PROP_FORCE_HANTROTILE: + g_value_set_boolean (value, sink->hantro_tile_enabled); break; case PROP_FD: g_value_set_int (value, sink->fd); @@ -2285,14 +2884,24 @@ static void gst_kms_sink_init (GstKMSSink * sink) { sink->fd = -1; + sink->ctrl_fd = -1; sink->is_internal_fd = TRUE; sink->conn_id = -1; sink->plane_id = -1; + sink->primary_plane_id = -1; + sink->hdr_prop_id = 0; sink->can_scale = TRUE; + sink->scale_checked = FALSE; + sink->upscale_ratio = 1; + sink->downscale_ratio = 1; + sink->hantro_tile_enabled = FALSE; gst_poll_fd_init (&sink->pollfd); sink->poll = gst_poll_new (TRUE); gst_video_info_init (&sink->vinfo); sink->skip_vsync = FALSE; + sink->frame_showed = 0; + sink->run_time = 0; + sink->global_alpha = 0; #ifdef HAVE_DRM_HDR sink->no_infoframe = FALSE; @@ -2328,6 +2937,7 @@ gst_kms_sink_class_init (GstKMSSinkClass * klass) gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, caps)); gst_caps_unref (caps); + element_class->change_state = GST_DEBUG_FUNCPTR (gst_kms_sink_change_state); basesink_class->start = GST_DEBUG_FUNCPTR (gst_kms_sink_start); basesink_class->stop = GST_DEBUG_FUNCPTR (gst_kms_sink_stop); basesink_class->set_caps = GST_DEBUG_FUNCPTR (gst_kms_sink_set_caps); @@ -2396,6 +3006,16 @@ gst_kms_sink_class_init (GstKMSSinkClass * klass) g_param_spec_boolean ("force-modesetting", "Force modesetting", "When enabled, the sink try to configure the display mode", FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT); + + /** + * kmssink:force-hantrotile: + * + * If enable, the sink propose hantro tile modifier to VPU. + */ + g_properties[PROP_FORCE_HANTROTILE] = + g_param_spec_boolean ("force-hantrotile", "Force to use hantro tile", + "When enabled, the sink propose hantro tile modifier to VPU", FALSE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT); /** * kmssink:restore-crtc: @@ -2469,6 +3089,14 @@ gst_kms_sink_class_init (GstKMSSinkClass * klass) g_param_spec_boxed ("plane-properties", "Connector Plane", "Additional properties for the plane", GST_TYPE_STRUCTURE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + /** + * kmssink:global-alpha: + * + * configure global alpha on mscale + */ + g_properties[PROP_GLOBAL_ALPHA] = g_param_spec_int ("global-alpha", + "global alpha", "global alpha", 0, 255, 0, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT); /** * kmssink:fd: diff --git a/sys/kms/gstkmssink.h b/sys/kms/gstkmssink.h index 317f789ef9..913a9caeba 100644 --- a/sys/kms/gstkmssink.h +++ b/sys/kms/gstkmssink.h @@ -26,6 +26,9 @@ #ifndef __GST_KMS_SINK_H__ #define __GST_KMS_SINK_H__ +#include +#include + #include #include @@ -42,6 +45,8 @@ G_BEGIN_DECLS #define GST_IS_KMS_SINK_CLASS(klass) \ (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_KMS_SINK)) +#define DEFAULT_HOLD_BUFFER_NUM 2 + typedef struct _GstKMSSink GstKMSSink; typedef struct _GstKMSSinkClass GstKMSSinkClass; @@ -50,11 +55,17 @@ struct _GstKMSSink { /*< private >*/ gint fd; + gint ctrl_fd; gint conn_id; gint crtc_id; gint plane_id; + gint primary_plane_id; guint pipe; + /* fps print support */ + guint64 frame_showed; + GstClockTime run_time; + /* crtc data */ guint16 hdisplay, vdisplay; guint32 buffer_id; @@ -64,12 +75,31 @@ struct _GstKMSSink { gboolean has_prime_export; gboolean has_async_page_flip; gboolean can_scale; + gboolean scale_checked; + gint upscale_ratio; + gint downscale_ratio; + + /* global alpha */ + gboolean is_kmsproperty_set; + guint global_alpha; gboolean modesetting_enabled; gboolean restore_crtc; GstStructure *connector_props; GstStructure *plane_props; + gboolean display_connected; + gboolean hantro_tile_enabled; + + /* hdr10 support */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + struct hdr_static_metadata hdr10meta; +#else + struct hdr_output_metadata hdr10meta; +#endif + + gint hdr_prop_id; + GstVideoInfo vinfo; GstCaps *allowed_caps; GstBufferPool *pool; @@ -78,6 +108,7 @@ struct _GstKMSSink { guint last_width; guint last_height; GstBuffer *last_buffer; + GstBuffer *hold_buf[DEFAULT_HOLD_BUFFER_NUM]; GstMemory *tmp_kmsmem; gchar *devname; @@ -90,6 +121,7 @@ struct _GstKMSSink { /* render video rectangle */ GstVideoRectangle render_rect; + GstVideoRectangle last_rect; /* reconfigure info if driver doesn't scale */ GstVideoRectangle pending_rect; diff --git a/sys/kms/gstkmsutils.c b/sys/kms/gstkmsutils.c index cc719fcd64..e81a9ea182 100644 --- a/sys/kms/gstkmsutils.c +++ b/sys/kms/gstkmsutils.c @@ -27,7 +27,13 @@ #include "config.h" #endif +#include +#include + #include +#include +#include +#include #include "gstkmsutils.h" @@ -78,6 +84,13 @@ static const struct DEF_FMT (NV21, NV21), DEF_FMT (NV12, NV12), + /* 10bits/c YUV 4:2:0 */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + DEF_FMT (P010, NV12_10LE40), +#else + DEF_FMT (NV15, NV12_10LE40), +#endif + /* 16bits/p RGB */ DEF_FMT (RGB565, RGB16), DEF_FMT (BGR565, BGR16), @@ -151,6 +164,31 @@ gst_drm_bpp_from_drm (guint32 drmfmt) return bpp; } +guint32 +gst_drm_alignment_from_drm_format (guint32 drmfmt) +{ + guint32 alignment; + + switch (drmfmt) { + case DRM_FORMAT_YUV420: + case DRM_FORMAT_YVU420: + case DRM_FORMAT_YUV422: + case DRM_FORMAT_NV12: + case DRM_FORMAT_NV21: + case DRM_FORMAT_NV16: + case DRM_FORMAT_UYVY: + case DRM_FORMAT_YUYV: + case DRM_FORMAT_YVYU: + alignment = 2; + break; + default: + alignment = 1; + break; + } + + return alignment; +} + guint32 gst_drm_height_from_drm (guint32 drmfmt, guint32 height) { diff --git a/sys/kms/gstkmsutils.h b/sys/kms/gstkmsutils.h index 657007068d..bd19aefe2d 100644 --- a/sys/kms/gstkmsutils.h +++ b/sys/kms/gstkmsutils.h @@ -33,6 +33,7 @@ G_BEGIN_DECLS GstVideoFormat gst_video_format_from_drm (guint32 drmfmt); guint32 gst_drm_format_from_video (GstVideoFormat fmt); guint32 gst_drm_bpp_from_drm (guint32 drmfmt); +guint32 gst_drm_alignment_from_drm_format (guint32 drmfmt); guint32 gst_drm_height_from_drm (guint32 drmfmt, guint32 height); GstCaps * gst_kms_sink_caps_template_fill (void); void gst_video_calculate_device_ratio (guint dev_width, -- 2.25.1 From 5d46a1e5f0cfc83ce2115cd18e26b420dd898814 Mon Sep 17 00:00:00 2001 From: Elliot Chen Date: Tue, 12 Apr 2022 14:38:09 +0800 Subject: [PATCH 23/65] LF-5172 dashdemux: support adaptive playback for webm and single fragment mp4 dash stream 1. parse webm stream and support adaptive playback 2. support adaptive playback for single fragment mp4 stream 3. add data download control for webm and single fragment mp4 stream Signed-off-by: Elliot Chen --- ext/dash/gstdashdemux.c | 305 ++++++++++++++++++ ext/dash/gstdashdemux.h | 2 + ext/dash/gstmpdhelper.c | 8 + ext/dash/meson.build | 2 +- gst-libs/gst/matroska/gstmatroska.c | 460 ++++++++++++++++++++++++++++ gst-libs/gst/matroska/gstmatroska.h | 104 +++++++ gst-libs/gst/matroska/meson.build | 22 ++ gst-libs/gst/meson.build | 2 + 8 files changed, 904 insertions(+), 1 deletion(-) create mode 100755 gst-libs/gst/matroska/gstmatroska.c create mode 100755 gst-libs/gst/matroska/gstmatroska.h create mode 100755 gst-libs/gst/matroska/meson.build diff --git a/ext/dash/gstdashdemux.c b/ext/dash/gstdashdemux.c index 5471b83640..596e1784b6 100644 --- a/ext/dash/gstdashdemux.c +++ b/ext/dash/gstdashdemux.c @@ -424,6 +424,12 @@ static gboolean gst_dash_demux_poll_clock_drift (GstDashDemux * demux); static GTimeSpan gst_dash_demux_get_clock_compensation (GstDashDemux * demux); static GDateTime *gst_dash_demux_get_server_now_utc (GstDashDemux * demux); +static gboolean +gst_dash_demux_has_next_fragment(GstAdaptiveDemuxStream * stream); +static GstFlowReturn +gst_dash_demux_handle_matroska (GstAdaptiveDemux * demux, GstAdaptiveDemuxStream *stream, + GstAdapter *adapter); + #define SIDX(s) (&(s)->sidx_parser.sidx) static inline GstSidxBoxEntry * @@ -947,6 +953,7 @@ gst_dash_demux_setup_all_streams (GstDashDemux * demux) } gst_isoff_sidx_parser_init (&stream->sidx_parser); + gst_matroska_parser_init(&stream->matroska_parser); } return TRUE; @@ -1356,6 +1363,14 @@ gst_dash_demux_stream_update_fragment_info (GstAdaptiveDemuxStream * stream) * stream to a subsegment */ return GST_FLOW_OK; } + + /* single fragment stream */ + if (!gst_dash_demux_has_next_fragment(stream)) { + if (stream->fragment.index_uri) { + GST_DEBUG_OBJECT(stream->pad, "single fragment"); + return GST_FLOW_OK; + } + } } if (dashstream->moof_sync_samples @@ -1469,6 +1484,14 @@ gst_dash_demux_stream_update_fragment_info (GstAdaptiveDemuxStream * stream) dashstream->actual_position += entry->duration; } else { stream->fragment.range_end = fragment.range_end; + + /* single fragment stream */ + if (!gst_dash_demux_has_next_fragment(stream)) { + stream->fragment.range_end = + stream->fragment.range_start + entry->size - 1; + GST_DEBUG_OBJECT (stream->pad, "single fragment, update range_end, sidx = %d", + SIDX(dashstream)->entry_index); + } } } else { dashstream->actual_position = stream->fragment.timestamp = @@ -1633,6 +1656,7 @@ gst_dash_demux_stream_seek (GstAdaptiveDemuxStream * stream, gboolean forward, GST_LOG_OBJECT (stream->pad, "Segment index was changed, reset sidx parser"); gst_isoff_sidx_parser_clear (&dashstream->sidx_parser); + gst_matroska_parser_clear (&dashstream->matroska_parser); dashstream->sidx_base_offset = 0; dashstream->allow_sidx = TRUE; } @@ -1643,6 +1667,7 @@ gst_dash_demux_stream_seek (GstAdaptiveDemuxStream * stream, gboolean forward, GST_ERROR_OBJECT (stream->pad, "Couldn't find position in sidx"); dashstream->sidx_position = GST_CLOCK_TIME_NONE; gst_isoff_sidx_parser_clear (&dashstream->sidx_parser); + gst_matroska_parser_clear (&dashstream->matroska_parser); } dashstream->pending_seek_ts = GST_CLOCK_TIME_NONE; } else { @@ -2277,6 +2302,16 @@ gst_dash_demux_stream_advance_fragment (GstAdaptiveDemuxStream * stream) return ret; } +static gboolean +gst_dash_demux_has_next_fragment(GstAdaptiveDemuxStream * stream) +{ + GstDashDemux *dashdemux = GST_DASH_DEMUX_CAST (stream->demux); + GstDashDemuxStream *dashstream = (GstDashDemuxStream *) stream; + + return gst_mpd_client_has_next_segment (dashdemux->client, + dashstream->active_stream, stream->demux->segment.rate > 0.0); +} + static gboolean gst_dash_demux_stream_select_bitrate (GstAdaptiveDemuxStream * stream, guint64 bitrate) @@ -2294,6 +2329,17 @@ gst_dash_demux_stream_select_bitrate (GstAdaptiveDemuxStream * stream, goto end; } + if (!gst_dash_demux_has_next_fragment(stream)) { + GST_DEBUG_OBJECT (stream->pad, "single fragment stream, " + "connection_speed: %d, bitrate: %" G_GUINT64_FORMAT, + base_demux->connection_speed, bitrate); + + if (stream->downloading_index || stream->downloading_header) { + GST_DEBUG_OBJECT (stream->pad, "not media segment"); + return ret; + } + } + /* In key-frame trick mode don't change bitrates */ if (GST_ADAPTIVE_DEMUX_IN_TRICKMODE_KEY_UNITS (demux)) { GST_DEBUG_OBJECT (demux, "In key-frame trick mode, not changing bitrates"); @@ -2375,6 +2421,7 @@ gst_dash_demux_stream_select_bitrate (GstAdaptiveDemuxStream * stream, } gst_isoff_sidx_parser_clear (&dashstream->sidx_parser); + gst_matroska_parser_clear (&dashstream->matroska_parser); dashstream->sidx_base_offset = 0; dashstream->allow_sidx = TRUE; @@ -3617,6 +3664,12 @@ gst_dash_demux_data_received (GstAdaptiveDemux * demux, gst_adapter_push (dash_stream->adapter, buffer); buffer = NULL; + /* matroska stream */ + if ((ret = gst_dash_demux_handle_matroska(demux, stream, + dash_stream->adapter)) != GST_FLOW_NOT_SUPPORTED) { + return ret; + } + if (dash_stream->is_isobmff || stream->downloading_index) { /* SIDX index is also ISOBMMF */ ret = gst_dash_demux_handle_isobmff (demux, stream); @@ -4158,3 +4211,255 @@ gst_dash_demux_get_server_now_utc (GstDashDemux * demux) g_date_time_unref (client_now); return server_now; } + +static GstFlowReturn +gst_dash_demux_update_sidx_parser_by_matroska_parser(GstAdaptiveDemux *demux, + GstAdaptiveDemuxStream *stream) +{ + guint64 index = 0; + GstDashDemuxStream *dash_stream = (GstDashDemuxStream *)stream; + GstMatroskaParser *matroska_parser = &(dash_stream->matroska_parser); + GstSidxParser *sidx_parser = &dash_stream->sidx_parser; + GstFlowReturn res = GST_FLOW_OK; + GstSidxBox *sidx = NULL; + + /* Check matroska cues information list */ + if (!matroska_parser->array) { + GST_DEBUG_OBJECT(stream->pad, "the cues information array list is null "); + return res; + } + + /* update sidx parser */ + sidx_parser->sidx.first_offset = matroska_parser->segment_head_offset; + /* sidx base offset used in check subfragment boundary */ + dash_stream->sidx_base_offset = matroska_parser->segment_head_offset; + sidx_parser->sidx.entry_index = 0; + sidx_parser->sidx.entries_count = matroska_parser->cue_point_num; + sidx_parser->sidx.entries = + g_malloc (sizeof (GstSidxBoxEntry) * sidx_parser->sidx.entries_count); + GST_DEBUG_OBJECT(stream->pad, "matroska parser: segment_head_offset = %" G_GUINT64_FORMAT, + matroska_parser->segment_head_offset); + + /* Update sidx box information */ + while (index < sidx_parser->sidx.entries_count) { + GstSidxBoxEntry *sidx_entry = &sidx_parser->sidx.entries[index]; + GstMatroskaPointData *cue_entry = &g_array_index (matroska_parser->array, + GstMatroskaPointData, index); + sidx_entry->size = 0; + sidx_entry->offset = cue_entry->track_pos.cluster_pos; + sidx_entry->pts = cue_entry->cue_time * matroska_parser->time_scale; + sidx_entry->duration = -1; + + if (index >= 1) { + GstSidxBoxEntry *prev_sidx_entry = &sidx_parser->sidx.entries[index - 1]; + prev_sidx_entry->duration = (sidx_entry->pts - prev_sidx_entry->pts); + prev_sidx_entry->size = (sidx_entry->offset - prev_sidx_entry->offset); + GST_DEBUG_OBJECT(stream->pad, "sidx_entry[%" G_GUINT64_FORMAT "]: duration = %" G_GUINT64_FORMAT + ", size = %d", index - 1, prev_sidx_entry->duration, prev_sidx_entry->size); + } + + GST_DEBUG_OBJECT(stream->pad, "sidx_entry[%" G_GUINT64_FORMAT "]: offset = %" G_GUINT64_FORMAT + ", pts = %" G_GUINT64_FORMAT, index, sidx_entry->offset, sidx_entry->pts); + index++; + } + + GST_DEBUG_OBJECT(stream->pad, "free array, addr = %p", matroska_parser->array); + g_array_free(matroska_parser->array, TRUE); + matroska_parser->array = NULL; + + /* We might've cleared the index above */ + sidx = SIDX (dash_stream); + if (sidx->entries_count > 0) { + if (GST_CLOCK_TIME_IS_VALID (dash_stream->pending_seek_ts)) { + /* FIXME, preserve seek flags */ + if (gst_dash_demux_stream_sidx_seek (dash_stream, + demux->segment.rate >= 0, 0, dash_stream->pending_seek_ts, + NULL) != GST_FLOW_OK) { + GST_ERROR_OBJECT (stream->pad, "pending_seek_ts is valid, stream seek error"); + /* select fragment position */ + dash_stream->sidx_position = GST_CLOCK_TIME_NONE; + gst_isoff_sidx_parser_clear (&dash_stream->sidx_parser); + } else { + GST_DEBUG_OBJECT (stream->pad, "pending_seek_ts is valid, stream seek ok"); + } + dash_stream->pending_seek_ts = GST_CLOCK_TIME_NONE; + } else { + /* sidx seek */ + if (dash_stream->sidx_position == GST_CLOCK_TIME_NONE) { + SIDX (dash_stream)->entry_index = 0; + } else { + /* update sidx entry_index*/ + if (gst_dash_demux_stream_sidx_seek (dash_stream, + demux->segment.rate >= 0, GST_SEEK_FLAG_SNAP_BEFORE, + dash_stream->sidx_position, NULL) != GST_FLOW_OK) { + GST_ERROR_OBJECT (stream->pad, + "Couldn't find position in sidx"); + dash_stream->sidx_position = GST_CLOCK_TIME_NONE; + gst_isoff_sidx_parser_clear (&dash_stream->sidx_parser); + } else { + GST_DEBUG_OBJECT (stream->pad, + "stream sidx seek ok, sidx index = %d, sidx_position = %" G_GUINT64_FORMAT, + sidx_parser->sidx.entry_index, dash_stream->sidx_position); + } + } + dash_stream->sidx_position = + SIDX (dash_stream)->entries[SIDX (dash_stream)->entry_index]. + pts; + } + } + + sidx_parser->status = GST_ISOFF_SIDX_PARSER_FINISHED; + return res; +} + +static GstFlowReturn +gst_dash_demux_handle_matroska (GstAdaptiveDemux * demux, GstAdaptiveDemuxStream * stream, GstAdapter *adapter) +{ + GstBuffer * buffer = NULL; + GstDashDemuxStream *dash_stream = (GstDashDemuxStream *)stream; + GstMatroskaParser *matroska_parser = &(dash_stream->matroska_parser); + GstFlowReturn ret = GST_FLOW_OK; + GstMatroskaParserResult res = GST_MATROSKA_PARSER_OK; + + guint index_header_or_data = 0; + if (stream->downloading_index) + index_header_or_data = 1; + else if (stream->downloading_header) + index_header_or_data = 2; + else + index_header_or_data = 3; + + GST_DEBUG_OBJECT(stream->pad, "index_header_or_data = %d, rep idx = %d," + "sid index = %d, sidx_base_offset = %" G_GINT64_FORMAT ", stream current_offset = %" G_GUINT64_FORMAT, + index_header_or_data, + dash_stream->active_stream->representation_idx, + dash_stream->sidx_parser.sidx.entry_index, + dash_stream->sidx_base_offset, + dash_stream->current_offset); + + /* entry matroska parser */ + res = gst_matroska_parser_entry(matroska_parser, adapter); + if (res == GST_MATROSKA_PARSER_NOT_SUPPORTED) { + /* Not matroska format */ + return GST_FLOW_NOT_SUPPORTED; + } else { + if (res == GST_MATROSKA_PARSER_DONE) { + gst_dash_demux_update_sidx_parser_by_matroska_parser (demux, stream); + GST_DEBUG_OBJECT(stream->pad, "update sidx parser, consume = %" G_GUINT64_FORMAT, matroska_parser->consume); + } + } + + if ((dash_stream->sidx_parser.status == GST_ISOFF_SIDX_PARSER_FINISHED) && + (index_header_or_data == 3)) { + gsize available; + GST_DEBUG_OBJECT (stream->pad, "cluster data"); + + /* Not ISOBMFF but had a SIDX index. Does this even exist or work? */ + while (ret == GST_FLOW_OK + && ((available = gst_adapter_available (dash_stream->adapter)) > 0)) { + + gboolean advance = FALSE; + guint64 sidx_end_offset = + dash_stream->sidx_base_offset + + SIDX_CURRENT_ENTRY (dash_stream)->offset + + SIDX_CURRENT_ENTRY (dash_stream)->size; + gboolean has_next = gst_dash_demux_stream_has_next_subfragment (stream); + + if (dash_stream->current_offset + available < sidx_end_offset) { + buffer = gst_adapter_take_buffer (dash_stream->adapter, available); + } else { + if (!has_next && sidx_end_offset <= dash_stream->current_offset) { + /* Drain all bytes, since there might be trailing bytes at the end of subfragment */ + buffer = gst_adapter_take_buffer (dash_stream->adapter, available); + GST_DEBUG_OBJECT (stream->pad, "drain data"); + } else { + if (sidx_end_offset <= dash_stream->current_offset) { + /* This means a corrupted stream or a bug: ignoring bugs, it + * should only happen if the SIDX index is corrupt */ + GST_ERROR_OBJECT (stream->pad, "Invalid SIDX state"); + gst_adapter_clear (dash_stream->adapter); + ret = GST_FLOW_ERROR; + break; + } else { + buffer = + gst_adapter_take_buffer (dash_stream->adapter, + sidx_end_offset - dash_stream->current_offset); + advance = TRUE; + } + } + } + + GST_BUFFER_OFFSET (buffer) = dash_stream->current_offset; + GST_BUFFER_OFFSET_END (buffer) = + GST_BUFFER_OFFSET (buffer) + gst_buffer_get_size (buffer); + dash_stream->current_offset = GST_BUFFER_OFFSET_END (buffer); + + GST_DEBUG_OBJECT (stream->pad, "rep idx = %d, sidx idx = %d, sidx_end_offset = %" G_GUINT64_FORMAT + ", current_offset = %" G_GUINT64_FORMAT ", buffer offset end = %" G_GUINT64_FORMAT, + dash_stream->active_stream->representation_idx, + dash_stream->sidx_parser.sidx.entry_index, + sidx_end_offset, + GST_BUFFER_OFFSET (buffer), + GST_BUFFER_OFFSET_END (buffer)); + + ret = gst_adaptive_demux_stream_push_buffer (stream, buffer); + if (advance) { + if (has_next) { + GstFlowReturn new_ret; + GST_DEBUG_OBJECT (stream->pad, "has next subfragment, advance"); + new_ret = + gst_adaptive_demux_stream_advance_fragment (demux, stream, + SIDX_CURRENT_ENTRY (dash_stream)->duration); + + /* only overwrite if it was OK before */ + if (ret == GST_FLOW_OK) { + ret = new_ret; + + /* check whether need to change representation index */ + if (ret != GST_FLOW_OK) { + GST_DEBUG_OBJECT (stream->pad, "stop send data"); + return ret; + } + } + } + } else { + /* reach the last sub fragment boundary */ + break; + } + } + } else { + guint64 avail_len = gst_adapter_available (dash_stream->adapter); + + GST_DEBUG_OBJECT (stream->pad, "main header = %d: matroska parser offset = %" G_GUINT64_FORMAT + ", avail_len = %" G_GUINT64_FORMAT, + index_header_or_data, matroska_parser->offset, avail_len); + + /* If have parsed data, can only send parsed data */ + if (matroska_parser->offset) { + if (matroska_parser->offset > avail_len) { + GST_ERROR_OBJECT (stream->pad, "need to return, matroska parser offset: %" G_GUINT64_FORMAT + " > avail_len: %" G_GUINT64_FORMAT, + matroska_parser->offset, avail_len); + return ret; + } else { + avail_len = matroska_parser->offset; + /* Clear parser offset */ + matroska_parser->offset = 0; + } + } + + /* this should be the main header, just push it all */ + buffer = gst_adapter_take_buffer (dash_stream->adapter, + avail_len); + + GST_BUFFER_OFFSET (buffer) = dash_stream->current_offset; + GST_BUFFER_OFFSET_END (buffer) = + GST_BUFFER_OFFSET (buffer) + gst_buffer_get_size (buffer); + dash_stream->current_offset = GST_BUFFER_OFFSET_END (buffer); + ret = gst_adaptive_demux_stream_push_buffer (stream, buffer); + } + + return ret; +} + + diff --git a/ext/dash/gstdashdemux.h b/ext/dash/gstdashdemux.h index 6374fc94c5..bb869a1cc6 100644 --- a/ext/dash/gstdashdemux.h +++ b/ext/dash/gstdashdemux.h @@ -37,6 +37,7 @@ #include "gstmpdclient.h" #include #include +#include G_BEGIN_DECLS #define GST_TYPE_DASH_DEMUX \ @@ -72,6 +73,7 @@ struct _GstDashDemuxStream gint64 sidx_base_offset; gboolean allow_sidx; GstClockTime pending_seek_ts; + GstMatroskaParser matroska_parser; GstAdapter *adapter; /* current offset of the first byte in the adapter / last byte we pushed or diff --git a/ext/dash/gstmpdhelper.c b/ext/dash/gstmpdhelper.c index 26c78bab8d..9ed4c4692d 100644 --- a/ext/dash/gstmpdhelper.c +++ b/ext/dash/gstmpdhelper.c @@ -87,8 +87,12 @@ gst_mpd_helper_mimetype_to_stream_type (const gchar * mimeType) return GST_STREAM_TYPE_VIDEO; } else if (strcmp (mimeType, "video/mp4") == 0) { return GST_STREAM_TYPE_VIDEO; + } else if (strcmp (mimeType, "video/webm") == 0) { + return GST_STREAM_TYPE_VIDEO; } else if (strcmp (mimeType, "audio/mp4") == 0) { return GST_STREAM_TYPE_AUDIO; + } else if (strcmp (mimeType, "audio/webm") == 0) { + return GST_STREAM_TYPE_AUDIO; } else if (strcmp (mimeType, "text/vtt") == 0) { return GST_STREAM_TYPE_TEXT; } else @@ -104,8 +108,12 @@ gst_mpd_helper_mimetype_to_caps (const gchar * mimeType) return "video/mpegts, systemstream=(bool) true"; } else if (strcmp (mimeType, "video/mp4") == 0) { return "video/quicktime"; + } else if (strcmp (mimeType, "video/webm") == 0) { + return "video/webm"; } else if (strcmp (mimeType, "audio/mp4") == 0) { return "audio/x-m4a"; + } else if (strcmp (mimeType, "audio/webm") == 0) { + return "audio/webm"; } else if (strcmp (mimeType, "text/vtt") == 0) { return "application/x-subtitle-vtt"; } else diff --git a/ext/dash/meson.build b/ext/dash/meson.build index dbaf99e5e5..5558522890 100644 --- a/ext/dash/meson.build +++ b/ext/dash/meson.build @@ -47,7 +47,7 @@ if xml2_dep.found() include_directories : [configinc, libsinc], dependencies : [gstadaptivedemux_dep, gsturidownloader_dep, gsttag_dep, gstnet_dep, gstpbutils_dep, gstbase_dep, gstisoff_dep, - gio_dep, xml2_dep], + gio_dep, xml2_dep, gstmatroska_dep], install : true, install_dir : plugins_install_dir, ) diff --git a/gst-libs/gst/matroska/gstmatroska.c b/gst-libs/gst/matroska/gstmatroska.c new file mode 100755 index 0000000000..29f4e9ac4b --- /dev/null +++ b/gst-libs/gst/matroska/gstmatroska.c @@ -0,0 +1,460 @@ +/* GStreamer + * Copyright 2021-2022 NXP + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstmatroska.h" +#include "stdio.h" + +GST_DEBUG_CATEGORY_STATIC (gst_matroska_debug); +#define GST_CAT_DEFAULT gst_matroska_debug + +static gboolean initialized = FALSE; + +#define INITIALIZE_DEBUG_CATEGORY \ + if (!initialized) { \ + GST_DEBUG_CATEGORY_INIT (gst_matroska_debug, "matroska", 0, \ + "Matroska Format parsing library"); \ + initialized = TRUE; \ + } + +/* log definition */ +#define GST_MATROSKA_PARSER_DEBUG GST_LOG + +/* L0: EMBL header */ +#define GST_EBML_ID_HEADER 0x1A45DFA3 +/* L0: toplevel Segment */ +#define GST_MATROSKA_ID_SEGMENT 0x18538067 +#define GST_MATROSKA_ID_CLUSTER 0x1F43B675 + +/* L1: matroska top-level master IDs, childs of Segment */ +#define GST_MATROSKA_ID_SEEKHEAD 0x114D9B74 +#define GST_MATROSKA_ID_SEGMENTINFO 0x1549A966 +#define GST_MATROSKA_ID_TRACKS 0x1654AE6B +#define GST_MATROSKA_ID_CUES 0x1C53BB6B +#define GST_MATROSKA_ID_TAGS 0x1254C367 +#define GST_MATROSKA_ID_ATTACHMENTS 0x1941A469 +#define GST_MATROSKA_ID_CHAPTERS 0x1043A770 + +/* L2: time scale, child of segment information */ +#define GST_MATROSKA_ID_TIMECODESCALE 0x2AD7B1 +#define GST_MATROSKA_ID_DURATION 0x4489 +/* L2: cue point, child of cues */ +#define GST_MATROSKA_ID_POINTENTRY 0xBB +/* L3: cue time, child of cue point */ +#define GST_MATROSKA_ID_CUETIME 0xB3 +/* L3: cue track position , child of cue point */ +#define GST_MATROSKA_ID_CUETRACKPOSITION 0xB7 +/* L4: cue track, child of track position */ +#define GST_MATROSKA_ID_CUETRACK 0xF7 +/* L4: cue cluster position, child of track position */ +#define GST_MATROSKA_ID_CUECLUSTERPOSITION 0xF1 +#define GST_MATROSKA_ID_CUEBLOCKNUMBER 0x5378 + +void +gst_matroska_parser_init (GstMatroskaParser * parser) +{ + if (parser->array) { + GST_ERROR ("array list has not free yet, addr = %p", parser->array); + } + memset (parser, 0, sizeof (GstMatroskaParser)); +} + +void +gst_matroska_parser_clear (GstMatroskaParser * parser) +{ + if (parser->array) { + GST_ERROR ("free array list, addr = %p", parser->array); + g_array_free (parser->array, TRUE); + } + memset (parser, 0, sizeof (GstMatroskaParser)); +} + +static gint32 +gst_matroska_parser_read_len (guint8 data) +{ + guint i = 0; + + for (i = 0; i < 8; i++) { + if ((data & (0x80 >> i)) != 0) + break; + } + return (i + 1); +} + +static guint64 +gst_matroska_parser_read_data (guint8 * p, gint64 len) +{ + guint64 data = 0; + + if (!p || !len || (len > 8)) { + GST_ERROR ("param error, p = %p, len = %" G_GINT64_FORMAT, p, len); + return 0; + } + + while (len--) { + data <<= 8; + data |= *p++; + } + + return data; +} + +static guint64 +gst_matroska_parser_read_data_len (guint8 * p_buf, guint64 len) +{ + guint64 data = 0; + guint8 *p8 = p_buf; + + if (!p8 || !len || len > 8) { + GST_ERROR ("param error, p8 = %p , len = %" G_GUINT64_FORMAT, p8, len); + return 0; + } + + data = *p8++; + data -= (guint64) 1 << (8 - len); + while (--len) { + data = (data << 8) | (*p8++); + } + + return data; +} + +static guint64 +gst_matroska_read_one_ebml_info (guint8 * buf, gint64 size, + GstMatroskaEbmlInfo * p_info) +{ + gint64 len = 0; + gint64 bytes = 0; + guint64 rem_size = 0; + guint8 *p8 = NULL; + guint64 unknown_length[8] = { 0x7F, 0x3FFF, 0x1FFFFF, + 0x0FFFFFFF, 0x07FFFFFFFFLL, 0x03FFFFFFFFFFLL, + 0x01FFFFFFFFFFFFLL, 0x00FFFFFFFFFFFFFFLL + }; + + if (!buf || !size || !p_info) { + GST_MATROSKA_PARSER_DEBUG + ("read ebml info, buf: %p, size: %" G_GINT64_FORMAT ", p_info: %p", + buf, size, p_info); + return 0; + } + + rem_size = size; + p8 = buf; + + /* extract id field */ + len = gst_matroska_parser_read_len (*p8); + if ((rem_size <= len) || (len > 8)) { + if (len > 8) { + GST_ERROR ("id field error, size = %" G_GUINT64_FORMAT, len); + } + return 0; + } else { + p_info->id = gst_matroska_parser_read_data (p8, len); + p8 += len; + bytes += len; + rem_size -= len; + } + + /* extract size field */ + len = gst_matroska_parser_read_len (*p8); + if ((rem_size <= len) || (len > 8)) { + if (len > 8) { + GST_ERROR ("data size field error, size = %" G_GUINT64_FORMAT ", id = 0x%x", len, + p_info->id); + } + return 0; + } else { + p_info->size = gst_matroska_parser_read_data_len (p8, len); + } + /* check unknown length */ + if (unknown_length[len - 1] == p_info->size) { + p_info->size = 0x7FFFFFFFFFFFFFFFULL; + } + p8 += len; + bytes += len; + rem_size -= len; + + /* extract data field */ + p_info->data_buf = p8; + p_info->data_offset = (gint64) (p8 - buf); + bytes += p_info->size; + + return bytes; +} + +static GstMatroskaParserResult +gst_matroska_parser_check_id (GstMatroskaParser * parser, guint64 id, + guint32 len, GstAdapter * adapter) +{ + GstMatroskaParserResult res = GST_MATROSKA_PARSER_OK; + guint64 avail = gst_adapter_available (adapter); + + /* Check buffer length */ + if (avail < len) { + GST_MATROSKA_PARSER_DEBUG ("insufficient data, len = %" G_GUINT64_FORMAT, avail); + return GST_MATROSKA_PARSER_INSUFFICIENT_DATA; + } + + /* Check id */ + if (gst_matroska_parser_read_data ((guint8 *) gst_adapter_map (adapter, len), + len) != id) { + res = GST_MATROSKA_PARSER_NOT_SUPPORTED; + } + + gst_adapter_unmap (adapter); + return res; +} + +static GstMatroskaParserResult +gst_matroska_parser_extract_data (GstMatroskaParser * parser, + GstAdapter * adapter) +{ + GstMatroskaPointData point_data; + GstMatroskaEbmlInfo embl_info; + guint64 consume = 0; + guint64 offset = parser->offset; + GstMatroskaPointData *entry = NULL; + GstMatroskaParserResult res = GST_MATROSKA_PARSER_OK; + guint64 len = 0; + guint8 *p_buf = 0; + + INITIALIZE_DEBUG_CATEGORY; + len = gst_adapter_available (adapter); + p_buf = (guint8 *) gst_adapter_map (adapter, len); + while (TRUE) { + /* Read one ebml */ + consume = gst_matroska_read_one_ebml_info (p_buf + offset, len, &embl_info); + + /* Adjust some element type to parse sub element information */ + if (consume) { + if ((embl_info.id == GST_MATROSKA_ID_SEGMENT) || + (embl_info.id == GST_MATROSKA_ID_CUES) || + (embl_info.id == GST_MATROSKA_ID_CLUSTER)) { + if (consume > embl_info.size) { + consume -= embl_info.size; + } + } + } + + /* Check data integrity */ + if (!consume || (len < consume)) { + if (offset) { + parser->offset = offset; + } + gst_adapter_unmap (adapter); + + GST_MATROSKA_PARSER_DEBUG + ("insufficient data, offset = %" G_GUINT64_FORMAT + ", remain len = %" G_GUINT64_FORMAT + ", consume = %" G_GUINT64_FORMAT, + offset, len, consume); + return GST_MATROSKA_PARSER_OK; + } + + /* Get current cue to fill array list */ + if (parser->array && parser->cue_point_num) { + entry = &g_array_index (parser->array, + GstMatroskaPointData, (parser->cue_point_num - 1)); + } + + switch (embl_info.id) { + case GST_EBML_ID_HEADER: + { + GST_MATROSKA_PARSER_DEBUG ("id: embl header, size = %" G_GUINT64_FORMAT, + embl_info.size); + break; + } + case GST_MATROSKA_ID_SEGMENT: + { + parser->segment_offset = parser->consume; + /* update consume, need to parse childs id */ + consume = embl_info.data_offset; + GST_MATROSKA_PARSER_DEBUG ("id: segment, size = %" G_GUINT64_FORMAT, embl_info.size); + break; + } + case GST_MATROSKA_ID_SEEKHEAD: + { + parser->segment_head_offset = parser->consume; + GST_MATROSKA_PARSER_DEBUG + ("id: segment seek head, offset = %" G_GUINT64_FORMAT ", size = %" G_GUINT64_FORMAT, + parser->segment_head_offset, embl_info.size); + break; + } + case GST_MATROSKA_ID_SEGMENTINFO: + { + /* update consume, need to parse childs id */ + consume = embl_info.data_offset; + GST_MATROSKA_PARSER_DEBUG ("id: segment information, size = %" G_GUINT64_FORMAT, + embl_info.size); + break; + } + case GST_MATROSKA_ID_TIMECODESCALE: + { + parser->time_scale = + gst_matroska_parser_read_data (embl_info.data_buf, embl_info.size); + break; + } + case GST_MATROSKA_ID_DURATION: + { + parser->duration = + gst_matroska_parser_read_data (embl_info.data_buf, embl_info.size); + break; + } + case GST_MATROSKA_ID_CUES: + { + /* update consume, need to parse childs id */ + consume = embl_info.data_offset; + parser->len = parser->consume + consume + embl_info.size; + break; + } + case GST_MATROSKA_ID_POINTENTRY: + { + if (!parser->array) { + parser->array = + g_array_new (FALSE, TRUE, sizeof (GstMatroskaPointData)); + } + + memset (&point_data, 0, sizeof (GstMatroskaPointData)); + g_array_append_val (parser->array, point_data); + parser->cue_point_num++; + /* need to parse childs id */ + consume = embl_info.data_offset; + GST_MATROSKA_PARSER_DEBUG ("id: cue point, num = %" G_GUINT64_FORMAT ", size = %" G_GUINT64_FORMAT, + parser->cue_point_num, embl_info.size); + break; + } + case GST_MATROSKA_ID_CUETIME: + { + if (entry) { + entry->cue_time = + gst_matroska_parser_read_data (embl_info.data_buf, + embl_info.size); + } + break; + } + case GST_MATROSKA_ID_CUETRACKPOSITION: + { + /* need to parse childs id */ + consume = embl_info.data_offset; + GST_MATROSKA_PARSER_DEBUG ("id: track position, num = %" G_GUINT64_FORMAT ", size = %" G_GUINT64_FORMAT, + parser->cue_point_num, embl_info.size); + break; + } + case GST_MATROSKA_ID_CUECLUSTERPOSITION: + { + if (entry) { + entry->track_pos.cluster_pos = + gst_matroska_parser_read_data (embl_info.data_buf, + embl_info.size); + } + break; + } + case GST_MATROSKA_ID_CUETRACK: + { + if (entry) { + entry->track_pos.track = + gst_matroska_parser_read_data (embl_info.data_buf, + embl_info.size); + } + break; + } + case GST_MATROSKA_ID_CLUSTER: + { + GST_MATROSKA_PARSER_DEBUG ("id: cluster, offset = %" G_GUINT64_FORMAT, + parser->consume); + gst_adapter_unmap (adapter); + return GST_MATROSKA_PARSER_DONE; + } + default: + { + GST_MATROSKA_PARSER_DEBUG + ("unhandled id = 0x%x, size = %" G_GUINT64_FORMAT + ", data_offset = %" G_GUINT64_FORMAT + ", conume = %" G_GUINT64_FORMAT, + embl_info.id, embl_info.size, embl_info.data_offset, + parser->consume); + break; + } + } + len -= consume; + offset += consume; + parser->consume += consume; + + /* Get all cues information */ + if (parser->len) { + if (parser->consume >= parser->len) { + parser->offset = offset; + GST_MATROSKA_PARSER_DEBUG + ("get all cues data, offset in adapter = %" G_GUINT64_FORMAT + ", conume = %" G_GUINT64_FORMAT ", len = %" G_GUINT64_FORMAT, + parser->offset, parser->consume, parser->len); + res = GST_MATROSKA_PARSER_DONE; + break; + } + } + } + + gst_adapter_unmap (adapter); + return res; +} + +GstMatroskaParserResult +gst_matroska_parser_entry (GstMatroskaParser * parser, GstAdapter * adapter) +{ + GstMatroskaParserResult res = GST_MATROSKA_PARSER_OK; + + INITIALIZE_DEBUG_CATEGORY; + if (!parser || !adapter) { + GST_MATROSKA_PARSER_DEBUG ("gst_matroska_parser_entry: error param"); + return GST_MATROSKA_PARSER_NOT_SUPPORTED; + } + + switch (parser->status) { + case GST_MATROSKA_PARSER_STATUS_INIT: + { + gst_matroska_parser_init (parser); + parser->status = GST_MATROSKA_PARSER_STATUS_HEADER; + } + case GST_MATROSKA_PARSER_STATUS_HEADER: + { + res = + gst_matroska_parser_check_id (parser, GST_EBML_ID_HEADER, 4, adapter); + if (res != GST_MATROSKA_PARSER_OK) { + return res; + } + parser->status = GST_MATROSKA_PARSER_STATUS_DATA; + } + case GST_MATROSKA_PARSER_STATUS_DATA: + { + res = gst_matroska_parser_extract_data (parser, adapter); + if (res != GST_MATROSKA_PARSER_DONE) { + return res; + } + parser->status = GST_MATROSKA_PARSER_STATUS_FINISHED; + } + case GST_MATROSKA_PARSER_STATUS_FINISHED: + break; + default: + break; + } + return res; +} diff --git a/gst-libs/gst/matroska/gstmatroska.h b/gst-libs/gst/matroska/gstmatroska.h new file mode 100755 index 0000000000..ac8c855307 --- /dev/null +++ b/gst-libs/gst/matroska/gstmatroska.h @@ -0,0 +1,104 @@ +/* GStreamer + * Copyright 2021-2022 NXP + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +#ifndef __GST_MATROSKA_H__ +#define __GST_MATROSKA_H__ +#include +#include +#include + +G_BEGIN_DECLS +#ifndef GST_MATROSKA_API +# ifdef BUILDING_GST_MATROSKA +# define GST_MATROSKA_API GST_API_EXPORT /* from config.h */ +# else +# define GST_MATROSKA_API GST_API_IMPORT +# endif +#endif + typedef struct +{ + guint32 id; + guint64 size; + gint64 data_offset; + guint8 *data_buf; +} GstMatroskaEbmlInfo; + +typedef enum +{ + GST_MATROSKA_PARSER_OK, + GST_MATROSKA_PARSER_DONE, + GST_MATROSKA_PARSER_NOT_SUPPORTED, + GST_MATROSKA_PARSER_ERROR_PARAM, + GST_MATROSKA_PARSER_INSUFFICIENT_DATA, + GST_MATROSKA_PARSER_ERROR +} GstMatroskaParserResult; + +typedef enum +{ + GST_MATROSKA_PARSER_STATUS_INIT, + GST_MATROSKA_PARSER_STATUS_HEADER, + GST_MATROSKA_PARSER_STATUS_DATA, + GST_MATROSKA_PARSER_STATUS_FINISHED +} GstMatroskaParserStatus; + +typedef struct +{ + guint64 track; + guint64 cluster_pos; +} GstMatroskaTrackPosType; + +typedef struct +{ + guint64 cue_time; + GstMatroskaTrackPosType track_pos; +} GstMatroskaPointData; + +typedef struct _GstMatroskaParser +{ + /* total length before first cluster */ + guint64 len; + /* segment offset */ + guint64 segment_offset; + /* cluster address = segment_head_offset + cue cluster position */ + guint64 segment_head_offset; + /* unit of time scale is nanosecond */ + guint64 time_scale; + guint64 duration; + /* parsed length in adapter */ + guint64 offset; + /* current consume length */ + guint64 consume; + /* number of cue point */ + guint64 cue_point_num; + /* cue point data list */ + GArray *array; + GstMatroskaParserStatus status; + gboolean is_discard_ebml_header; + guint64 total_length; +} GstMatroskaParser; + +GST_MATROSKA_API void gst_matroska_parser_init (GstMatroskaParser * parser); + +GST_MATROSKA_API void gst_matroska_parser_clear (GstMatroskaParser * parser); + +GST_MATROSKA_API + GstMatroskaParserResult gst_matroska_parser_entry (GstMatroskaParser * + parser, GstAdapter * adapter); + +G_END_DECLS +#endif /* __GST_MATROSKA_H__ */ diff --git a/gst-libs/gst/matroska/meson.build b/gst-libs/gst/matroska/meson.build new file mode 100755 index 0000000000..71e717bfe7 --- /dev/null +++ b/gst-libs/gst/matroska/meson.build @@ -0,0 +1,22 @@ +matroska_sources = [ + 'gstmatroska.c', +] +matroska_headers = [ + 'gstmatroska.h', +] +install_headers(matroska_headers, subdir : 'gstreamer-1.0/gst/matroska') + +gstmatroska = library('gstmatroska-' + api_version, + matroska_sources, + c_args : gst_plugins_bad_args + ['-DGST_USE_UNSTABLE_API', '-DBUILDING_GST_MATROSKA', '-DG_LOG_DOMAIN="GStreamer-MATROSKA"'], + include_directories : [configinc, libsinc], + version : libversion, + soversion : soversion, + darwin_versions : osxversion, + install : true, + dependencies : [gstbase_dep], +) + +gstmatroska_dep = declare_dependency(link_with : gstmatroska, + include_directories : [libsinc], + dependencies : [gstbase_dep]) diff --git a/gst-libs/gst/meson.build b/gst-libs/gst/meson.build index 6207f3381a..23f797991d 100644 --- a/gst-libs/gst/meson.build +++ b/gst-libs/gst/meson.build @@ -25,3 +25,5 @@ subdir('wayland') subdir('webrtc') subdir('winrt') subdir('analytics') +subdir('matroska') + -- 2.25.1 From 0c0aae6968074818e677f657cfb85dd37981f75b Mon Sep 17 00:00:00 2001 From: Elliot Chen Date: Fri, 30 Sep 2022 14:42:15 +0800 Subject: [PATCH 24/65] LF-6236 adaptivedemux: add the feature to support changing subtitles Download subtitle data again when change subtitle Need to send subtitle streams EOS event after the audio or video stream has sent it. Signed-off-by: Elliot Chen --- gst-libs/gst/adaptivedemux/gstadaptivedemux.c | 75 ++++++++++++++++++- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/adaptivedemux/gstadaptivedemux.c b/gst-libs/gst/adaptivedemux/gstadaptivedemux.c index b783a21967..dd33435fd6 100644 --- a/gst-libs/gst/adaptivedemux/gstadaptivedemux.c +++ b/gst-libs/gst/adaptivedemux/gstadaptivedemux.c @@ -152,6 +152,12 @@ GST_DEBUG_CATEGORY (adaptivedemux_debug); #define GST_ADAPTIVE_DEMUX_SEGMENT_LOCK(d) g_mutex_lock (GST_ADAPTIVE_DEMUX_SEGMENT_GET_LOCK (d)) #define GST_ADAPTIVE_DEMUX_SEGMENT_UNLOCK(d) g_mutex_unlock (GST_ADAPTIVE_DEMUX_SEGMENT_GET_LOCK (d)) +#define SUBTITLE_TEMPL_CAPS \ + "application/x-subtitle; application/x-subtitle-sami; " \ + "application/x-subtitle-tmplayer; application/x-subtitle-mpl2; " \ + "application/x-subtitle-dks; application/x-subtitle-qttext;" \ + "application/x-subtitle-lrc; application/x-subtitle-vtt;" \ + "text/x-raw, format= { pango-markup, utf8 }" enum { PROP_0, @@ -1857,6 +1863,43 @@ gst_adaptive_demux_handle_seek_event (GstAdaptiveDemux * demux, GstPad * pad, return ret; } +static gboolean gst_adaptive_demux_is_subtitle_stream (GstAdaptiveDemuxStream *stream) +{ + gboolean ret = FALSE; + GstCaps *caps; + GstCaps *templ_caps = + gst_caps_from_string (SUBTITLE_TEMPL_CAPS); + + caps = gst_pad_get_current_caps (stream->pad); + ret = gst_caps_is_subset (caps, templ_caps); + gst_caps_unref (caps); + gst_caps_unref (templ_caps); + + return ret; +} + +static gboolean +gst_adaptive_demux_push_subtitle_src_event (GstAdaptiveDemux * demux, GstEvent * event) +{ + GList *iter; + gboolean ret = TRUE; + + for (iter = demux->streams; iter; iter = g_list_next (iter)) { + GstAdaptiveDemuxStream *stream = iter->data; + gst_event_ref (event); + + if (gst_adaptive_demux_is_subtitle_stream (stream)) { + if (stream->last_ret == GST_FLOW_EOS) { + ret = ret & gst_pad_push_event (stream->pad, event); + stream->last_ret = GST_FLOW_OK; + GST_DEBUG_OBJECT (stream->pad, "sending EOS"); + } + } + } + gst_event_unref (event); + return ret; +} + static gboolean gst_adaptive_demux_src_event (GstPad * pad, GstObject * parent, GstEvent * event) @@ -1886,13 +1929,32 @@ gst_adaptive_demux_src_event (GstPad * pad, GstObject * parent, stream = gst_adaptive_demux_find_stream_for_pad (demux, pad); if (stream) { + GST_DEBUG_OBJECT (stream->pad, "Received reconfigure event"); if (!stream->cancelled && gst_adaptive_demux_is_running (demux) && - stream->last_ret == GST_FLOW_NOT_LINKED) { + (stream->last_ret == GST_FLOW_NOT_LINKED || stream->last_ret == GST_FLOW_EOS)) { + if (stream->last_ret == GST_FLOW_EOS) { + if (!gst_adaptive_demux_is_subtitle_stream (stream)) { + gst_event_unref (event); + GST_MANIFEST_UNLOCK (demux); + return TRUE; + } else { + if (gst_task_get_state (stream->download_task) == GST_TASK_STARTED) { + GST_DEBUG_OBJECT (stream->pad, "downloading subtitle data, ignore it"); + gst_event_unref (event); + GST_MANIFEST_UNLOCK (demux); + return TRUE; + } + GST_DEBUG_OBJECT (stream->pad, "download subtitle data again"); + stream->restart_download = FALSE; + } + } else { + GST_DEBUG_OBJECT (stream->pad, "Restarting download loop"); + stream->restart_download = TRUE; + } stream->last_ret = GST_FLOW_OK; - stream->restart_download = TRUE; stream->need_header = TRUE; stream->discont = TRUE; - GST_DEBUG_OBJECT (stream->pad, "Restarting download loop"); + gst_task_start (stream->download_task); } gst_event_unref (event); @@ -3879,6 +3941,12 @@ gst_adaptive_demux_stream_download_loop (GstAdaptiveDemuxStream * stream) ret = GST_FLOW_OK; } } + + if (gst_adaptive_demux_is_subtitle_stream (stream)) { + GST_DEBUG_OBJECT (stream->pad, "not sending EOS"); + ret = GST_FLOW_OK; + } + break; case GST_FLOW_NOT_LINKED: @@ -3974,6 +4042,7 @@ end_of_manifest: if (demux->next_streams == NULL && demux->prepared_streams == NULL) { GST_DEBUG_OBJECT (stream->src, "Pushing EOS on pad"); gst_adaptive_demux_stream_push_event (stream, gst_event_new_eos ()); + gst_adaptive_demux_push_subtitle_src_event(demux, gst_event_new_eos ()); } else { GST_DEBUG_OBJECT (stream->src, "Stream is EOS, but we're switching fragments. Not sending."); -- 2.25.1 From 482727485442a2049f65bcb91fa23eb6edb6271f Mon Sep 17 00:00:00 2001 From: Elliot Chen Date: Thu, 4 Jan 2024 18:50:07 +0900 Subject: [PATCH 25/65] MMFMWK-9308 autovideoconvert2: select one or more elements to convert by rank It add only the matched elements. Firstly, it will pick one element that matches the caps on both sides whose rank is greater than the composite rank. Secondly, if none of those elements match the caps, it will then combine two elements from the elements picked earlier according to the rank. Lastly, if none of them match, it will pick the remain unpicked elements. Signed-off-by: Elliot Chen --- gst/autoselect/gstautoselect.c | 1862 +++++++++++++++++++++++++ gst/autoselect/gstautoselect.h | 107 ++ gst/autoselect/gstautovideoconvert2.c | 266 ++++ gst/autoselect/gstautovideoconvert2.h | 53 + gst/autoselect/meson.build | 15 + gst/autoselect/plugin.c | 43 + gst/meson.build | 2 +- meson_options.txt | 1 + 8 files changed, 2348 insertions(+), 1 deletion(-) create mode 100755 gst/autoselect/gstautoselect.c create mode 100755 gst/autoselect/gstautoselect.h create mode 100755 gst/autoselect/gstautovideoconvert2.c create mode 100755 gst/autoselect/gstautovideoconvert2.h create mode 100755 gst/autoselect/meson.build create mode 100755 gst/autoselect/plugin.c diff --git a/gst/autoselect/gstautoselect.c b/gst/autoselect/gstautoselect.c new file mode 100755 index 0000000000..9c9b847218 --- /dev/null +++ b/gst/autoselect/gstautoselect.c @@ -0,0 +1,1862 @@ +/* GStreamer + * + * Copyright 2024 NXP + * @author: Elliot Chen + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ +/** + * SECTION:element-autoselect + * @title: autoselect + * + * The #autoselect element has one sink and one source pad. It will look for + * other elements that also have one sink and one source pad. + * + * The #autoselect make single or multiple elements to perform conversion + * and add only the selected element. If the caps change, it may change the + * selected element and remove thecurrent one from autoselect if the current + * one no longer matches the caps. + * + * Firstly, it will pick an element that matches the caps on both sides whose + * rank is greater than the composite rank level. + * + * Secondly, if none of the those elements match the caps whose rank is higher + * than composite rank, it will then try to pick and combine two elements (first + * element and last element) from the previous elements which have been tried + * before according to the rank and check whether they match and add them. + * + * Lastly, if none of the composite elements match the caps, it will try to + * pick the remain elements whose rank are equal to or less than composite rank. + * + * If single element can perform conversion, this single element is the first + * element in the diagram and there is no last element. The first element src pad + * connects #internal_sinkpad directly. + * +---------------------------------------------------------------------------+ + * | autoselect | + * | +---------------+ +---------------+ | + * | | first element | | last element | | + * | internal_srcpad-sink src-sink src-internal_sinkpad | + * | +---------------+ +---------------+ | + * sink-+ +-src + * +---------------------------------------------------------------------------+ + * + * The list of element it will look into can be specified in the + * #GstAutoSelect:factories property, otherwise it will look at all available + * elements. + */ + + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstautoselect.h" + +#include +#define DEFAULT_COMPOSITE_RANK (GST_RANK_PRIMARY) +#define DEFAULT_LOWEST_SELECT_RANK 0 + +GST_DEBUG_CATEGORY (autoselect_debug); +#define GST_CAT_DEFAULT (autoselect_debug) + +#define GST_AUTOSELECT_LOCK(ac) GST_OBJECT_LOCK (ac) +#define GST_AUTOSELECT_UNLOCK(ac) GST_OBJECT_UNLOCK (ac) + +/* elementfactory information */ +static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GstStaticPadTemplate sink_internal_template = +GST_STATIC_PAD_TEMPLATE ("sink_internal", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GstStaticPadTemplate src_internal_template = +GST_STATIC_PAD_TEMPLATE ("src_internal", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +/* GstAutoSelect signals and args */ +enum +{ + /* FILL ME */ + LAST_SIGNAL +}; + +enum +{ + PROP_0, + PROP_FACTORIES, + PROP_COMPOSITE_RANK, + PROP_SELECT_ELEMENT, + PROP_LOWEST_RANK +}; + +static void gst_auto_select_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_auto_select_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void gst_auto_select_dispose (GObject * object); + +static GstElement *gst_auto_select_get_subelement (GstAutoSelect * + autoselect, GstPadDirection dir); +static GstPad *gst_auto_select_get_internal_sinkpad (GstAutoSelect * + autoselect); +static GstPad *gst_auto_select_get_internal_srcpad (GstAutoSelect * autoselect); + +static GstIterator *gst_auto_select_iterate_internal_links (GstPad * pad, + GstObject * parent); +static gboolean +gst_auto_select_check_caps_negotiation (GstAutoSelect * autoselect, + GstCaps * caps, GstEvent * event); +static gboolean gst_auto_select_sink_setcaps (GstAutoSelect * autoselect, + GstCaps * caps, GstEvent * event); +static GstCaps *gst_auto_select_getcaps (GstAutoSelect * autoselect, + GstCaps * filter, GstPadDirection dir); +static GstFlowReturn gst_auto_select_sink_chain (GstPad * pad, + GstObject * parent, GstBuffer * buffer); +static GstFlowReturn gst_auto_select_sink_chain_list (GstPad * pad, + GstObject * parent, GstBufferList * list); +static gboolean gst_auto_select_sink_event (GstPad * pad, GstObject * parent, + GstEvent * event); +static gboolean gst_auto_select_sink_query (GstPad * pad, GstObject * parent, + GstQuery * query); + +static gboolean gst_auto_select_src_event (GstPad * pad, GstObject * parent, + GstEvent * event); +static gboolean gst_auto_select_src_query (GstPad * pad, GstObject * parent, + GstQuery * query); + +static GstFlowReturn gst_auto_select_internal_sink_chain (GstPad * pad, + GstObject * parent, GstBuffer * buffer); +static GstFlowReturn gst_auto_select_internal_sink_chain_list (GstPad * pad, + GstObject * parent, GstBufferList * list); +static gboolean gst_auto_select_internal_sink_event (GstPad * pad, + GstObject * parent, GstEvent * event); +static gboolean gst_auto_select_internal_sink_query (GstPad * pad, + GstObject * parent, GstQuery * query); + +static gboolean gst_auto_select_internal_src_event (GstPad * pad, + GstObject * parent, GstEvent * event); +static gboolean gst_auto_select_internal_src_query (GstPad * pad, + GstObject * parent, GstQuery * query); + +static void gst_auto_select_load_factories (GstAutoSelect * autoselect); + +static GList *gst_auto_select_copy_factories_list (GstAutoSelect * autoselect); +static gboolean +gst_auto_select_update_composite_factories_list (GstAutoSelect * autoselect, + GList ** factories_list, GstElementFactory * factory); +static gboolean +gst_auto_select_construct_composite_elements (GstAutoSelect * autoselect, + GList * first_factories, GList * last_factories, + GstElementFactory * current_factory); +static gboolean gst_auto_select_check_current_factory (GstAutoSelect * + autoselect, GstElementFactory * factory); +static gboolean gst_auto_select_construct_single_element (GstAutoSelect * + autoselect, GstElementFactory * current_factory); +static gboolean gst_auto_select_add_element (GstAutoSelect * autoselect, + GList * elem_list); +static gboolean gst_auto_select_activate_element (GstAutoSelect * autoselect, + GList * elem_list); + +static GQuark internal_srcpad_quark = 0; +static GQuark internal_sinkpad_quark = 0; +static GQuark parent_quark = 0; + +G_DEFINE_TYPE (GstAutoSelect, gst_auto_select, GST_TYPE_BIN); +GST_ELEMENT_REGISTER_DEFINE (autoselect, "autoselect", + GST_RANK_NONE, GST_TYPE_AUTO_SELECT); + +static void +gst_auto_select_class_init (GstAutoSelectClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstElementClass *gstelement_class = (GstElementClass *) klass; + + GST_DEBUG_CATEGORY_INIT (autoselect_debug, "autoselect", 0, + "Auto select based on caps"); + + internal_srcpad_quark = g_quark_from_static_string ("internal_srcpad"); + internal_sinkpad_quark = g_quark_from_static_string ("internal_sinkpad"); + parent_quark = g_quark_from_static_string ("parent"); + + + gst_element_class_add_static_pad_template (gstelement_class, &srctemplate); + gst_element_class_add_static_pad_template (gstelement_class, &sinktemplate); + + gst_element_class_set_static_metadata (gstelement_class, + "Auto select element(s) based on caps", "Generic/Bin", + "Selects the right element(s) based on the caps", + "Elliot Chen "); + + gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_auto_select_dispose); + + gobject_class->set_property = gst_auto_select_set_property; + gobject_class->get_property = gst_auto_select_get_property; + klass->update_composite_factories_list = + GST_DEBUG_FUNCPTR (gst_auto_select_update_composite_factories_list); + klass->construct_composite_elements = + GST_DEBUG_FUNCPTR (gst_auto_select_construct_composite_elements); + klass->check_current_factory = + GST_DEBUG_FUNCPTR (gst_auto_select_check_current_factory); + klass->construct_single_element = + GST_DEBUG_FUNCPTR (gst_auto_select_construct_single_element); + klass->add_element = GST_DEBUG_FUNCPTR (gst_auto_select_add_element); + klass->activate_element = + GST_DEBUG_FUNCPTR (gst_auto_select_activate_element); + + g_object_class_install_property (gobject_class, PROP_FACTORIES, + g_param_spec_pointer ("factories", + "GList of GstElementFactory", + "GList of GstElementFactory objects to pick from (the element takes" + " ownership of the list (NULL means it will go through all possible" + " elements), can only be set once", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_COMPOSITE_RANK, + g_param_spec_int ("composite-rank", "composite rank", + "combine multiple elements if none of the single elements" + " match the caps whose rank is above the composite rank", + 0, G_MAXINT, DEFAULT_COMPOSITE_RANK, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_SELECT_ELEMENT, + g_param_spec_pointer ("select-element-list", + "GList of the current selected element(s)", + "GList of the current selected element(s)", + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_LOWEST_RANK, + g_param_spec_int ("lowest-rank", "lowest rank", + "the factory can be selected in GList of GstElementFactory " + " whose rank need be equal to or greater than the lowest rank", + 0, G_MAXINT, DEFAULT_LOWEST_SELECT_RANK, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); +} + +static void +gst_auto_select_init (GstAutoSelect * autoselect) +{ + autoselect->sinkpad = + gst_pad_new_from_static_template (&sinktemplate, "sink"); + autoselect->srcpad = gst_pad_new_from_static_template (&srctemplate, "src"); + + gst_pad_set_chain_function (autoselect->sinkpad, + GST_DEBUG_FUNCPTR (gst_auto_select_sink_chain)); + gst_pad_set_chain_list_function (autoselect->sinkpad, + GST_DEBUG_FUNCPTR (gst_auto_select_sink_chain_list)); + gst_pad_set_event_function (autoselect->sinkpad, + GST_DEBUG_FUNCPTR (gst_auto_select_sink_event)); + gst_pad_set_query_function (autoselect->sinkpad, + GST_DEBUG_FUNCPTR (gst_auto_select_sink_query)); + gst_pad_set_iterate_internal_links_function (autoselect->sinkpad, + GST_DEBUG_FUNCPTR (gst_auto_select_iterate_internal_links)); + + gst_pad_set_event_function (autoselect->srcpad, + GST_DEBUG_FUNCPTR (gst_auto_select_src_event)); + gst_pad_set_query_function (autoselect->srcpad, + GST_DEBUG_FUNCPTR (gst_auto_select_src_query)); + gst_pad_set_iterate_internal_links_function (autoselect->srcpad, + GST_DEBUG_FUNCPTR (gst_auto_select_iterate_internal_links)); + + gst_element_add_pad (GST_ELEMENT (autoselect), autoselect->sinkpad); + gst_element_add_pad (GST_ELEMENT (autoselect), autoselect->srcpad); + + autoselect->composite_rank = DEFAULT_COMPOSITE_RANK; + autoselect->lowest_rank = DEFAULT_LOWEST_SELECT_RANK; + autoselect->is_composite = FALSE; +} + +static void +gst_auto_select_dispose (GObject * object) +{ + GstAutoSelect *autoselect = GST_AUTO_SELECT (object); + + if (autoselect->first_subelement) + g_clear_object (&autoselect->first_subelement); + + if (autoselect->last_subelement) + g_clear_object (&autoselect->last_subelement); + + g_clear_object (&autoselect->current_internal_sinkpad); + g_clear_object (&autoselect->current_internal_srcpad); + g_list_free (autoselect->elem_list); + + for (;;) { + GList *factories = g_atomic_pointer_get (&autoselect->factories); + + if (g_atomic_pointer_compare_and_exchange (&autoselect->factories, + factories, NULL)) { + gst_plugin_feature_list_free (factories); + break; + } + } + + G_OBJECT_CLASS (gst_auto_select_parent_class)->dispose (object); +} + +static void +gst_auto_select_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstAutoSelect *autoselect = GST_AUTO_SELECT (object); + GstState cur_state; + + gst_element_get_state (GST_ELEMENT (autoselect), &cur_state, NULL, 0); + if (cur_state == GST_STATE_NULL) { + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + case PROP_FACTORIES: + GList * old_factories = g_atomic_pointer_get (&autoselect->factories); + GList *new_factories = g_value_get_pointer (value); + + new_factories = g_list_copy (new_factories); + if (g_atomic_pointer_compare_and_exchange (&autoselect->factories, + old_factories, new_factories)) { + if (old_factories) { + GST_AUTOSELECT_LOCK (autoselect); + g_list_free (old_factories); + GST_AUTOSELECT_UNLOCK (autoselect); + } + } else { + g_list_free (new_factories); + } + break; + case PROP_COMPOSITE_RANK: + autoselect->composite_rank = g_value_get_int (value); + GST_DEBUG_OBJECT (object, "composite rank set to %d", + autoselect->composite_rank); + break; + case PROP_LOWEST_RANK: + autoselect->lowest_rank = g_value_get_int (value); + GST_DEBUG_OBJECT (object, "lowest select rank set to %d", + autoselect->lowest_rank); + break; + } + } else { + GST_WARNING_OBJECT (object, "Can not set property because the element" + "is not in the NULL state or initial state"); + } +} + +static void +gst_auto_select_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstAutoSelect *autoselect = GST_AUTO_SELECT (object); + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + case PROP_FACTORIES: + g_value_set_pointer (value, + g_atomic_pointer_get (&autoselect->factories)); + break; + case PROP_COMPOSITE_RANK: + g_value_set_int (value, autoselect->composite_rank); + break; + case PROP_SELECT_ELEMENT: + g_value_set_pointer (value, + g_atomic_pointer_get (&autoselect->elem_list)); + break; + case PROP_LOWEST_RANK: + g_value_set_int (value, autoselect->lowest_rank); + break; + } +} + + +static GstElement * +gst_auto_select_get_element_by_type (GstAutoSelect * autoselect, GType type) +{ + GList *item; + GstBin *bin = GST_BIN (autoselect); + GstElement *element = NULL; + + g_return_val_if_fail (type != 0, NULL); + + GST_OBJECT_LOCK (autoselect); + + for (item = bin->children; item; item = item->next) { + if (G_TYPE_CHECK_INSTANCE_TYPE (item->data, type)) { + element = gst_object_ref (item->data); + break; + } + } + + GST_OBJECT_UNLOCK (autoselect); + + return element; +} + +/** + * get_pad_by_direction: + * @element: The Element + * @direction: The direction + * + * Gets a #GstPad that goes in the requested direction. I will return NULL + * if there is no pad or if there is more than one pad in this direction + */ + +static GstPad * +get_pad_by_direction (GstElement * element, GstPadDirection direction) +{ + GstIterator *iter = gst_element_iterate_pads (element); + GstPad *selected_pad = NULL; + gboolean done; + GValue item = { 0, }; + + if (!iter) + return NULL; + + done = FALSE; + while (!done) { + switch (gst_iterator_next (iter, &item)) { + case GST_ITERATOR_OK: + { + GstPad *pad = g_value_get_object (&item); + + if (gst_pad_get_direction (pad) == direction) { + /* We check if there is more than one pad in this direction, + * if there is, we return NULL so that the element is refused + */ + if (selected_pad) { + done = TRUE; + gst_object_unref (selected_pad); + selected_pad = NULL; + } else { + selected_pad = g_object_ref (pad); + } + } + g_value_unset (&item); + } + break; + case GST_ITERATOR_RESYNC: + if (selected_pad) { + gst_object_unref (selected_pad); + selected_pad = NULL; + } + gst_iterator_resync (iter); + break; + case GST_ITERATOR_ERROR: + GST_ERROR ("Error iterating pads of element %s", + GST_OBJECT_NAME (element)); + gst_object_unref (selected_pad); + selected_pad = NULL; + done = TRUE; + break; + case GST_ITERATOR_DONE: + done = TRUE; + break; + } + } + g_value_unset (&item); + gst_iterator_free (iter); + + if (!selected_pad) + GST_ERROR ("Did not find pad of direction %d in %s", + direction, GST_OBJECT_NAME (element)); + + return selected_pad; +} + +static GstElement * +gst_auto_select_get_subelement (GstAutoSelect * autoselect, GstPadDirection dir) +{ + GstElement *element = NULL; + + GST_AUTOSELECT_LOCK (autoselect); + if (autoselect->is_composite) { + if (dir == GST_PAD_SINK) { + if (autoselect->first_subelement) + element = gst_object_ref (autoselect->first_subelement); + } else { + if (autoselect->last_subelement) + element = gst_object_ref (autoselect->last_subelement); + } + } else { + if (autoselect->first_subelement) + element = gst_object_ref (autoselect->first_subelement); + } + GST_AUTOSELECT_UNLOCK (autoselect); + + return element; +} + +static GstPad * +gst_auto_select_get_internal_sinkpad (GstAutoSelect * autoselect) +{ + GstPad *pad = NULL; + + GST_AUTOSELECT_LOCK (autoselect); + if (autoselect->current_internal_sinkpad) + pad = gst_object_ref (autoselect->current_internal_sinkpad); + GST_AUTOSELECT_UNLOCK (autoselect); + + return pad; +} + +static GstPad * +gst_auto_select_get_internal_srcpad (GstAutoSelect * autoselect) +{ + GstPad *pad = NULL; + + GST_AUTOSELECT_LOCK (autoselect); + if (autoselect->current_internal_srcpad) + pad = gst_object_ref (autoselect->current_internal_srcpad); + GST_AUTOSELECT_UNLOCK (autoselect); + + return pad; +} + +static void +gst_auto_select_remove_elements (GstBin * bin) +{ + GstIterator *iter = NULL; + GValue value = { 0 }; + GstElement *elem = NULL; + gboolean done = FALSE; + + iter = gst_bin_iterate_elements (bin); + while (!done) { + switch (gst_iterator_next (iter, &value)) { + case GST_ITERATOR_OK: + elem = (GstElement *) g_value_get_object (&value); + GST_DEBUG_OBJECT (bin, "remove element %s from bin", + GST_OBJECT_NAME (elem)); + gst_bin_remove (bin, elem); + gst_element_set_state (GST_ELEMENT (elem), GST_STATE_NULL); + /* Iterator increased the element refcount, so unref */ + g_value_unset (&value); + break; + case GST_ITERATOR_RESYNC: + gst_iterator_resync (iter); + break; + case GST_ITERATOR_ERROR: + GST_WARNING_OBJECT (bin, "error in iterating elements"); + done = TRUE; + break; + case GST_ITERATOR_DONE: + done = TRUE; + break; + } + } + gst_iterator_free (iter); +} + +static gboolean +gst_auto_select_add_element (GstAutoSelect * autoselect, GList * elem_list) +{ + GstElement *current_elem = NULL; + GstElement *first_elem = NULL; + GstElement *last_elem = NULL; + GstPad *internal_sinkpad = NULL; + GstPad *internal_srcpad = NULL; + GPtrArray *pad_array = NULL; + GstPadLinkReturn padlinkret; + GList *select_list = NULL; + + if (!elem_list) { + GST_DEBUG_OBJECT (autoselect, "No valid element list"); + return FALSE; + } + + /* Add the element to bin and get the pad */ + GST_DEBUG_OBJECT (autoselect, "Start trying to add element"); + for (select_list = elem_list; select_list; + select_list = g_list_next (select_list)) { + GstPad *sink_pad, *src_pad; + current_elem = GST_ELEMENT (select_list->data); + + if (!gst_bin_add (GST_BIN (autoselect), current_elem)) { + GST_WARNING_OBJECT (autoselect, "Could not add element %s to the bin", + GST_OBJECT_NAME (current_elem)); + gst_object_unref (current_elem); + goto add_element_failed; + } + GST_DEBUG_OBJECT (autoselect, "Trying to add element %s", + GST_OBJECT_NAME (current_elem)); + + /* Check and get pad from the selected element */ + sink_pad = get_pad_by_direction (current_elem, GST_PAD_SINK); + src_pad = get_pad_by_direction (current_elem, GST_PAD_SRC); + if (!sink_pad || !src_pad) { + GST_WARNING_OBJECT (autoselect, + "Could not find matched sink or src pad in %s", + GST_OBJECT_NAME (current_elem)); + if (sink_pad) + gst_object_unref (sink_pad); + if (src_pad) + gst_object_unref (src_pad); + goto link_pad_failed; + } + + if (!pad_array) { + pad_array = + g_ptr_array_new_with_free_func ((GDestroyNotify) gst_object_unref); + } + g_ptr_array_add (pad_array, sink_pad); + g_ptr_array_add (pad_array, src_pad); + + if (first_elem == NULL) { + first_elem = current_elem; + } + last_elem = current_elem; + } + + /* Create internal pad */ + internal_sinkpad = + gst_pad_new_from_static_template (&sink_internal_template, + "sink_internal"); + internal_srcpad = + gst_pad_new_from_static_template (&src_internal_template, "src_internal"); + + if (!internal_sinkpad || !internal_srcpad) { + GST_ERROR_OBJECT (autoselect, "Could not create internal pads"); + if (internal_sinkpad) + gst_object_unref (internal_sinkpad); + if (internal_srcpad) + gst_object_unref (internal_srcpad); + goto link_pad_failed; + } + + g_object_weak_ref (G_OBJECT (first_elem), (GWeakNotify) gst_object_unref, + internal_srcpad); + g_object_weak_ref (G_OBJECT (last_elem), (GWeakNotify) gst_object_unref, + internal_sinkpad); + + gst_pad_set_active (internal_sinkpad, TRUE); + gst_pad_set_active (internal_srcpad, TRUE); + + g_object_set_qdata (G_OBJECT (internal_srcpad), parent_quark, autoselect); + g_object_set_qdata (G_OBJECT (internal_sinkpad), parent_quark, autoselect); + + gst_pad_set_chain_function (internal_sinkpad, + GST_DEBUG_FUNCPTR (gst_auto_select_internal_sink_chain)); + gst_pad_set_chain_list_function (internal_sinkpad, + GST_DEBUG_FUNCPTR (gst_auto_select_internal_sink_chain_list)); + gst_pad_set_event_function (internal_sinkpad, + GST_DEBUG_FUNCPTR (gst_auto_select_internal_sink_event)); + gst_pad_set_query_function (internal_sinkpad, + GST_DEBUG_FUNCPTR (gst_auto_select_internal_sink_query)); + + gst_pad_set_event_function (internal_srcpad, + GST_DEBUG_FUNCPTR (gst_auto_select_internal_src_event)); + gst_pad_set_query_function (internal_srcpad, + GST_DEBUG_FUNCPTR (gst_auto_select_internal_src_query)); + + /* Try to link the pad */ + GstPad *first_elem_sink_pad = g_ptr_array_index (pad_array, 0); + padlinkret = gst_pad_link_full (internal_srcpad, first_elem_sink_pad, + GST_PAD_LINK_CHECK_NOTHING); + if (GST_PAD_LINK_FAILED (padlinkret)) { + GST_WARNING_OBJECT (autoselect, "Could not links pad %s:%s to %s:%s" + " for reason %d", + GST_DEBUG_PAD_NAME (internal_srcpad), + GST_DEBUG_PAD_NAME (first_elem_sink_pad), padlinkret); + goto link_pad_failed; + } + + GstPad *last_elem_src_pad = g_ptr_array_index (pad_array, pad_array->len - 1); + padlinkret = gst_pad_link_full (last_elem_src_pad, internal_sinkpad, + GST_PAD_LINK_CHECK_NOTHING); + if (GST_PAD_LINK_FAILED (padlinkret)) { + GST_WARNING_OBJECT (autoselect, "Could not links pad %s:%s to %s:%s" + " for reason %d", + GST_DEBUG_PAD_NAME (last_elem_src_pad), + GST_DEBUG_PAD_NAME (internal_sinkpad), padlinkret); + goto link_pad_failed; + } + + /* Connect all the elements */ + if (first_elem != last_elem) { + for (guint i = 1; i < (pad_array->len - 1); i = i + 2) { + GstPad *src_pad = g_ptr_array_index (pad_array, i); + GstPad *sink_pad = g_ptr_array_index (pad_array, i + 1); + padlinkret = gst_pad_link (src_pad, sink_pad); + if (GST_PAD_LINK_FAILED (padlinkret)) { + GST_WARNING_OBJECT (autoselect, "Could not links pad %s:%s to %s:%s" + " for reason %d", + GST_DEBUG_PAD_NAME (src_pad), + GST_DEBUG_PAD_NAME (sink_pad), padlinkret); + goto link_pad_failed; + } + } + } + g_ptr_array_free (pad_array, TRUE); + + /* Record the internal pad */ + g_object_set_qdata (G_OBJECT (first_elem), + internal_srcpad_quark, internal_srcpad); + g_object_set_qdata (G_OBJECT (last_elem), + internal_sinkpad_quark, internal_sinkpad); + + for (select_list = elem_list; select_list; + select_list = g_list_next (select_list)) { + current_elem = GST_ELEMENT (select_list->data); + gst_element_sync_state_with_parent (current_elem); + } + + return TRUE; + +link_pad_failed: + g_ptr_array_free (pad_array, TRUE); +add_element_failed: + g_list_free (elem_list); + gst_auto_select_remove_elements (GST_BIN (autoselect)); + return FALSE; +} + +static gboolean +gst_auto_select_make_composite_elements (GstAutoSelect * autoselect, + GList * first_factories, GList * last_factories) +{ + GList *first_list; + GList *last_list; + GList *elem_list = NULL; + GstElement *first_elem = NULL; + GstElement *last_elem = NULL; + GstElementFactory *first_loaded_factory = NULL; + GstElementFactory *last_loaded_factory = NULL; + gboolean ret = FALSE; + GstAutoSelectClass *klass = GST_AUTO_SELECT_GET_CLASS (autoselect); + + if (!first_factories || !last_factories) { + GST_DEBUG_OBJECT (autoselect, + "No valid factories to make composite element"); + return FALSE; + } + + for (first_list = first_factories; first_list; + first_list = g_list_next (first_list)) { + GstElementFactory *first_factory = GST_ELEMENT_FACTORY (first_list->data); + first_loaded_factory = + GST_ELEMENT_FACTORY (gst_plugin_feature_load (GST_PLUGIN_FEATURE + (first_factory))); + if (!first_loaded_factory) + continue; + + for (last_list = last_factories; last_list; + last_list = g_list_next (last_list)) { + GstElementFactory *last_factory = GST_ELEMENT_FACTORY (last_list->data); + last_loaded_factory = + GST_ELEMENT_FACTORY (gst_plugin_feature_load (GST_PLUGIN_FEATURE + (last_factory))); + + if (!last_loaded_factory || first_loaded_factory == last_loaded_factory) { + /* If the first and last factory are the same, need ignore it */ + continue; + } + + /* Create first element */ + first_elem = gst_auto_select_get_element_by_type (autoselect, + gst_element_factory_get_element_type (first_loaded_factory)); + if (first_elem) { + GST_WARNING_OBJECT (autoselect, + "first element %s has been added to the bin", + GST_OBJECT_NAME (first_elem)); + } else { + + first_elem = gst_element_factory_create (first_loaded_factory, NULL); + if (!first_elem) { + GST_WARNING_OBJECT (autoselect, "Failed to create first element"); + gst_object_unref (first_loaded_factory); + continue; + } + } + + /* Create last element */ + last_elem = gst_auto_select_get_element_by_type (autoselect, + gst_element_factory_get_element_type (last_loaded_factory)); + + if (last_elem) { + GST_WARNING_OBJECT (autoselect, + "last element %s has been added to the bin", + GST_OBJECT_NAME (last_elem)); + } else { + last_elem = gst_element_factory_create (last_loaded_factory, NULL); + if (!last_elem) { + GST_WARNING_OBJECT (autoselect, "Failed to create last element"); + gst_object_unref (last_loaded_factory); + continue; + } + } + gst_object_unref (last_loaded_factory); + + /* add the elements */ + elem_list = NULL; + elem_list = g_list_append (elem_list, first_elem); + elem_list = g_list_append (elem_list, last_elem); + ret = klass->add_element (autoselect, elem_list); + if (ret) { + ret = klass->activate_element (autoselect, elem_list); + if (ret) + break; + } + } + + gst_object_unref (first_loaded_factory); + if (ret) + break; + } + return ret; +} + +static gboolean +gst_auto_select_construct_single_element (GstAutoSelect * autoselect, + GstElementFactory * factory) +{ + GstElement *element = NULL; + GList *elem_list = NULL; + GstElementFactory *loaded_factory = + GST_ELEMENT_FACTORY (gst_plugin_feature_load (GST_PLUGIN_FEATURE + (factory))); + GstAutoSelectClass *klass = GST_AUTO_SELECT_GET_CLASS (autoselect); + + if (!loaded_factory) + return FALSE; + + element = gst_auto_select_get_element_by_type (autoselect, + gst_element_factory_get_element_type (loaded_factory)); + gst_object_unref (loaded_factory); + + if (element) { + GST_WARNING_OBJECT (autoselect, "element %s has been added to the bin", + GST_OBJECT_NAME (element)); + } else { + element = gst_element_factory_create (factory, NULL); + } + + if (!element) + return FALSE; + + elem_list = g_list_append (elem_list, element); + if (!klass->add_element (autoselect, elem_list)) { + return FALSE; + } + + return klass->activate_element (autoselect, elem_list); +} + +/* + * This function checks if there is one and only one pad template on the + * factory that can accept the given caps. If there is one and only one, + * it returns TRUE, otherwise, its FALSE + */ + +static gboolean +factory_can_intersect (GstAutoSelect * autoselect, + GstElementFactory * factory, GstPadDirection direction, GstCaps * caps) +{ + const GList *templates; + gint has_direction = FALSE; + gboolean ret = FALSE; + + g_return_val_if_fail (factory != NULL, FALSE); + g_return_val_if_fail (caps != NULL, FALSE); + + templates = gst_element_factory_get_static_pad_templates (factory); + + while (templates) { + GstStaticPadTemplate *template = (GstStaticPadTemplate *) templates->data; + + if (template->direction == direction) { + GstCaps *tmpl_caps = NULL; + gboolean intersect; + + /* If there is more than one pad in this direction, we return FALSE + * Only transform elements (with one sink and one source pad) + * are accepted + */ + if (has_direction) { + GST_DEBUG_OBJECT (autoselect, "Factory %p" + " has more than one static template with dir %d", + template, direction); + return FALSE; + } + has_direction = TRUE; + + tmpl_caps = gst_static_caps_get (&template->static_caps); + intersect = gst_caps_can_intersect (tmpl_caps, caps); + GST_DEBUG_OBJECT (autoselect, "Factories %" GST_PTR_FORMAT + " static caps %" GST_PTR_FORMAT " and caps %" GST_PTR_FORMAT + " can%s intersect", factory, tmpl_caps, caps, + intersect ? "" : " not"); + gst_caps_unref (tmpl_caps); + + ret |= intersect; + } + templates = g_list_next (templates); + } + + return ret; +} + +static gboolean +sticky_event_push (GstPad * pad, GstEvent ** event, gpointer user_data) +{ + GstAutoSelect *autoselect = GST_AUTO_SELECT (user_data); + + gst_event_ref (*event); + gst_pad_push_event (autoselect->current_internal_srcpad, *event); + + return TRUE; +} + +static gboolean +gst_auto_select_activate_element (GstAutoSelect * autoselect, GList * elem_list) +{ + gboolean ret = FALSE; + GList *select_list = NULL; + GstElement *first_elem = NULL; + GstElement *last_elem = NULL; + GstPad *internal_srcpad = NULL; + GstPad *internal_sinkpad = NULL; + + if (!elem_list) { + GST_DEBUG_OBJECT (autoselect, "No valid element"); + return ret; + } + + select_list = elem_list; + first_elem = GST_ELEMENT (select_list->data); + internal_srcpad = g_object_get_qdata (G_OBJECT (first_elem), + internal_srcpad_quark); + + select_list = g_list_next (select_list); + if (select_list) { + select_list = g_list_last (elem_list); + last_elem = GST_ELEMENT (select_list->data); + internal_sinkpad = g_object_get_qdata (G_OBJECT (last_elem), + internal_sinkpad_quark); + } else { + internal_sinkpad = g_object_get_qdata (G_OBJECT (first_elem), + internal_sinkpad_quark); + } + + /* Check the first elements can really accept caps */ + if (autoselect->caps) { + if (!gst_pad_peer_query_accept_caps (internal_srcpad, autoselect->caps)) { + GST_DEBUG_OBJECT (autoselect, "Could not set %s:%s to %" + GST_PTR_FORMAT, GST_DEBUG_PAD_NAME (internal_srcpad), + autoselect->caps); + goto activate_element_failed; + } else { + ret = TRUE; + } + } + + GST_AUTOSELECT_LOCK (autoselect); + gst_object_replace ((GstObject **) & autoselect->first_subelement, + GST_OBJECT (first_elem)); + if (last_elem) { + gst_object_replace ((GstObject **) & autoselect->last_subelement, + GST_OBJECT (last_elem)); + } + gst_object_replace ((GstObject **) & autoselect->current_internal_srcpad, + GST_OBJECT (internal_srcpad)); + gst_object_replace ((GstObject **) & autoselect->current_internal_sinkpad, + GST_OBJECT (internal_sinkpad)); + autoselect->elem_list = elem_list; + GST_AUTOSELECT_UNLOCK (autoselect); + + gst_pad_sticky_events_foreach (autoselect->sinkpad, sticky_event_push, + autoselect); + + /* If there are multiple elements in the + * bin, we need check caps negotiation result + */ + if (last_elem) + ret = + gst_auto_select_check_caps_negotiation (autoselect, autoselect->caps, + autoselect->event); + + if (ret) + gst_pad_push_event (autoselect->sinkpad, gst_event_new_reconfigure ()); + return ret; + +activate_element_failed: + g_list_free (elem_list); + gst_auto_select_remove_elements (GST_BIN (autoselect)); + return ret; +} + +static GstIterator * +gst_auto_select_iterate_internal_links (GstPad * pad, GstObject * parent) +{ + GstAutoSelect *autoselect = GST_AUTO_SELECT (parent); + GstIterator *it = NULL; + GstPad *internal; + + if (pad == autoselect->sinkpad) + internal = gst_auto_select_get_internal_srcpad (autoselect); + else + internal = gst_auto_select_get_internal_sinkpad (autoselect); + + if (internal) { + GValue val = { 0, }; + + g_value_init (&val, GST_TYPE_PAD); + g_value_take_object (&val, internal); + + it = gst_iterator_new_single (GST_TYPE_PAD, &val); + g_value_unset (&val); + } + + return it; +} + +static gboolean +gst_auto_select_check_caps_negotiation (GstAutoSelect * autoselect, + GstCaps * caps, GstEvent * event) +{ + GList *item; + GstElement *curr_elem; + GstPad *sink_pad = NULL; + GstPad *peer = NULL; + gboolean is_first_elem = TRUE; + + for (item = autoselect->elem_list; item; item = item->next) { + curr_elem = item->data; + + sink_pad = get_pad_by_direction (curr_elem, GST_PAD_SINK); + if (!sink_pad) { + GST_WARNING_OBJECT (autoselect, "Could not find matched sink pad in %s", + GST_OBJECT_NAME (curr_elem)); + goto check_failed; + } + if (is_first_elem) { + is_first_elem = FALSE; + peer = gst_pad_get_peer (sink_pad); + if (!peer) { + GST_WARNING_OBJECT (autoselect, "Could not get peer pad, %s:%s", + GST_DEBUG_PAD_NAME (sink_pad)); + goto check_failed; + } + + /* Send caps event to trigger caps negotiation + * If there are multiple elements in the bin, + * we can get all the element sink caps to + * check caps negotiation result + */ + gst_event_ref (event); + if (!gst_pad_push_event (peer, event)) { + GST_DEBUG_OBJECT (autoselect, "Could not send gap event, %s:%s", + GST_DEBUG_PAD_NAME (peer)); + gst_object_unref (peer); + goto check_failed; + } + gst_object_unref (peer); + } + + if (!gst_pad_get_current_caps (sink_pad)) { + GST_DEBUG_OBJECT (autoselect, "Could not get caps, %s:%s", + GST_DEBUG_PAD_NAME (sink_pad)); + gst_object_unref (sink_pad); + goto check_failed; + } + gst_object_unref (sink_pad); + } + + gst_event_unref (event); + return TRUE; +check_failed: + GST_AUTOSELECT_LOCK (autoselect); + autoselect->first_subelement = NULL; + autoselect->last_subelement = NULL; + autoselect->current_internal_sinkpad = NULL; + autoselect->current_internal_srcpad = NULL; + g_list_free (autoselect->elem_list); + autoselect->elem_list = NULL; + GST_AUTOSELECT_UNLOCK (autoselect); + gst_auto_select_remove_elements (GST_BIN (autoselect)); + return FALSE; +} + +static gboolean +gst_auto_select_update_composite_factories_list (GstAutoSelect * autoselect, + GList ** factories_list, GstElementFactory * factory) +{ + guint rank = gst_plugin_feature_get_rank ((GstPluginFeature *) factory); + + if (rank > autoselect->composite_rank) { + GST_LOG_OBJECT (autoselect, + "Add Factory %s to list", + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE (factory))); + /* Record the factory that only accept sink pad */ + *factories_list = g_list_append (*factories_list, factory); + return TRUE; + } + return FALSE; +} + +static gboolean +gst_auto_select_construct_composite_elements (GstAutoSelect * autoselect, + GList * first_factories, GList * last_factories, + GstElementFactory * current_factory) +{ + guint curent_rank = + gst_plugin_feature_get_rank ((GstPluginFeature *) current_factory); + + if (curent_rank <= autoselect->composite_rank + && first_factories && last_factories) { + /* Try to make and combine two elements if no + * single element currently matched whose rank + * is greater than the composite rank level. + */ + autoselect->is_composite = + gst_auto_select_make_composite_elements (autoselect, + first_factories, last_factories); + + if (!autoselect->is_composite) { + GST_LOG_OBJECT (autoselect, "Can not make composite elements"); + } + return TRUE; + } + + return FALSE; +} + +static gboolean +gst_auto_select_check_current_factory (GstAutoSelect * autoselect, + GstElementFactory * factory) +{ + guint rank = gst_plugin_feature_get_rank ((GstPluginFeature *) factory); + + /* Check current factory rank and ignore it + * if it's lower than the specified lowest rank. + */ + if (rank < autoselect->lowest_rank) { + GST_LOG_OBJECT (autoselect, + "Factory %s is ignored because the rank is litter than the specified rank", + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE (factory))); + return FALSE; + } + + return TRUE; +} + +static gboolean +gst_auto_select_sink_setcaps (GstAutoSelect * autoselect, GstCaps * caps, + GstEvent * event) +{ + GList *elem; + GstCaps *other_caps = NULL; + GList *factories = NULL; + GstCaps *current_caps; + GList *first_factories = NULL; + GList *last_factories = NULL; + gboolean has_sent_event = FALSE; + GstAutoSelectClass *klass = GST_AUTO_SELECT_GET_CLASS (autoselect); + + g_return_val_if_fail (autoselect != NULL, FALSE); + + /* Check the caps and return if it's the same as the current caps */ + current_caps = gst_pad_get_current_caps (autoselect->sinkpad); + if (current_caps) { + if (gst_caps_is_equal_fixed (caps, current_caps)) { + GST_DEBUG_OBJECT (autoselect, "Got the same caps %" GST_PTR_FORMAT, caps); + gst_caps_unref (current_caps); + goto done; + } + gst_caps_unref (current_caps); + } + + /* Check whether we can set the new caps with the current element(s) */ + if (caps && (autoselect->is_composite || autoselect->first_subelement)) { + if (gst_pad_peer_query_accept_caps (autoselect->current_internal_srcpad, + caps)) { + /* If there is one element, it can really accept caps */ + if (!autoselect->is_composite) { + goto done; + } + + /* Need check multiple elements caps negotiation result */ + if (gst_auto_select_check_caps_negotiation (autoselect, caps, event)) { + has_sent_event = TRUE; + goto done; + } + } + } + + /* Start to construct single or composite elements with factories list */ + autoselect->caps = caps; + autoselect->event = event; + other_caps = gst_pad_peer_query_caps (autoselect->srcpad, NULL); + factories = gst_auto_select_copy_factories_list (autoselect); + if (!factories) + goto done; + + for (elem = factories; elem; elem = g_list_next (elem)) { + GstElementFactory *factory = GST_ELEMENT_FACTORY (elem->data); + + /* If no single element currently is selected, + * check whether we need to construct composite elements now. + */ + if (klass->construct_composite_elements (autoselect, + first_factories, last_factories, factory)) { + g_list_free (first_factories); + g_list_free (last_factories); + first_factories = NULL; + last_factories = NULL; + + if (autoselect->is_composite) { + has_sent_event = TRUE; + break; + } + } + + /* Check the current factory and ignore it if the + * conditions don't match. + */ + if (!klass->check_current_factory (autoselect, factory)) { + continue; + } + + /* Let's first check if these caps have any chance of success + * according to the static pad templates on the factory + */ + if (!factory_can_intersect (autoselect, factory, GST_PAD_SINK, caps)) { + GST_LOG_OBJECT (autoselect, "Factory %s does not accept sink caps %" + GST_PTR_FORMAT, + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE (factory)), caps); + + if (other_caps != NULL) { + /* Record the factory that only accept src pad */ + if (factory_can_intersect (autoselect, factory, GST_PAD_SRC, + other_caps)) { + if (klass->update_composite_factories_list (autoselect, + &last_factories, factory)) { + GST_LOG_OBJECT (autoselect, + "Factory %s can only accept src caps %" GST_PTR_FORMAT, + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE (factory)), + other_caps); + } + } + } + continue; + } + + if (other_caps != NULL) { + if (!factory_can_intersect (autoselect, factory, GST_PAD_SRC, other_caps)) { + GST_LOG_OBJECT (autoselect, + "Factory %s does not accept src caps %" GST_PTR_FORMAT, + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE (factory)), + other_caps); + + /* Record the factory that only accept sink pad */ + if (klass->update_composite_factories_list (autoselect, + &first_factories, factory)) { + GST_LOG_OBJECT (autoselect, + "Factory %s can only accept sink caps %" GST_PTR_FORMAT, + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE (factory)), caps); + } + continue; + } + } + + /* Try to construct single element */ + if (!klass->construct_single_element (autoselect, factory)) { + /* add the factory to the two lists together and it can be + * selected again when constructing composite elements. + */ + first_factories = g_list_append (first_factories, factory); + last_factories = g_list_append (last_factories, factory); + } else { + break; + } + } + +done: + if (factories) + g_list_free (factories); + + if (other_caps) + gst_caps_unref (other_caps); + + if (first_factories) + g_list_free (first_factories); + if (last_factories) + g_list_free (last_factories); + + if (autoselect->is_composite || autoselect->first_subelement) { + GST_DEBUG_OBJECT (autoselect, "Could set %s:%s to %" GST_PTR_FORMAT, + GST_DEBUG_PAD_NAME (autoselect->current_internal_srcpad), caps); + + /* Need send caps event if it hasn't been sent */ + if (!has_sent_event) { + return gst_pad_push_event (autoselect->current_internal_srcpad, event); + } + return TRUE; + } else { + gst_event_unref (event); + GST_DEBUG_OBJECT (autoselect, "Could not find a matching element for caps"); + return FALSE; + } +} + +/* + * This function filters the pad pad templates, taking only transform element + * (with one sink and one src pad) + */ + +static gboolean +gst_auto_select_default_filter_func (GstPluginFeature * feature, + gpointer user_data) +{ + GstElementFactory *factory = NULL; + const GList *static_pad_templates, *tmp; + GstStaticPadTemplate *src = NULL, *sink = NULL; + + if (!GST_IS_ELEMENT_FACTORY (feature)) + return FALSE; + + factory = GST_ELEMENT_FACTORY (feature); + + static_pad_templates = gst_element_factory_get_static_pad_templates (factory); + + for (tmp = static_pad_templates; tmp; tmp = g_list_next (tmp)) { + GstStaticPadTemplate *template = tmp->data; + GstCaps *caps; + + if (template->presence == GST_PAD_SOMETIMES) + return FALSE; + + if (template->presence != GST_PAD_ALWAYS) + continue; + + switch (template->direction) { + case GST_PAD_SRC: + if (src) + return FALSE; + src = template; + break; + case GST_PAD_SINK: + if (sink) + return FALSE; + sink = template; + break; + default: + return FALSE; + } + + caps = gst_static_pad_template_get_caps (template); + + if (gst_caps_is_any (caps) || gst_caps_is_empty (caps)) + return FALSE; + } + + if (!src || !sink) + return FALSE; + + return TRUE; +} + +/* function used to sort element features + * Copy-pasted from decodebin */ +static gint +compare_ranks (GstPluginFeature * f1, GstPluginFeature * f2) +{ + gint diff; + const gchar *rname1, *rname2; + + diff = gst_plugin_feature_get_rank (f2) - gst_plugin_feature_get_rank (f1); + if (diff != 0) + return diff; + + rname1 = gst_plugin_feature_get_name (f1); + rname2 = gst_plugin_feature_get_name (f2); + + diff = strcmp (rname2, rname1); + + return diff; +} + +static void +gst_auto_select_load_factories (GstAutoSelect * autoselect) +{ + GList *all_factories; + + all_factories = + gst_registry_feature_filter (gst_registry_get (), + gst_auto_select_default_filter_func, FALSE, NULL); + + all_factories = g_list_sort (all_factories, (GCompareFunc) compare_ranks); + + g_assert (all_factories); + + if (!g_atomic_pointer_compare_and_exchange (&autoselect->factories, + (GList *) NULL, all_factories)) { + gst_plugin_feature_list_free (all_factories); + } +} + +/* In this case, we should almost always have an internal element, because + * set_caps() should have been called first + */ + +static GstFlowReturn +gst_auto_select_sink_chain (GstPad * pad, GstObject * parent, + GstBuffer * buffer) +{ + GstFlowReturn ret = GST_FLOW_NOT_NEGOTIATED; + GstAutoSelect *autoselect = GST_AUTO_SELECT (parent); + + if (autoselect->current_internal_srcpad) { + ret = gst_pad_push (autoselect->current_internal_srcpad, buffer); + if (ret != GST_FLOW_OK) + GST_DEBUG_OBJECT (autoselect, + "Child element %" GST_PTR_FORMAT "returned flow %s", + autoselect->first_subelement, gst_flow_get_name (ret)); + } else { + GST_ERROR_OBJECT (autoselect, "Got buffer without an negotiated element," + " returning not-negotiated"); + gst_buffer_unref (buffer); + } + + return ret; +} + +static GstFlowReturn +gst_auto_select_sink_chain_list (GstPad * pad, GstObject * parent, + GstBufferList * list) +{ + GstFlowReturn ret = GST_FLOW_NOT_NEGOTIATED; + GstAutoSelect *autoselect = GST_AUTO_SELECT (parent); + + if (autoselect->current_internal_srcpad) { + ret = gst_pad_push_list (autoselect->current_internal_srcpad, list); + if (ret != GST_FLOW_OK) + GST_DEBUG_OBJECT (autoselect, + "Child element %" GST_PTR_FORMAT "returned flow %s", + autoselect->first_subelement, gst_flow_get_name (ret)); + } else { + GST_ERROR_OBJECT (autoselect, "Got buffer without an negotiated element," + " returning not-negotiated"); + gst_buffer_list_unref (list); + } + + return ret; +} + +static gboolean +gst_auto_select_sink_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + gboolean ret = TRUE; + GstAutoSelect *autoselect = GST_AUTO_SELECT (parent); + GstPad *internal_srcpad; + + if (GST_EVENT_TYPE (event) == GST_EVENT_CAPS) { + GstCaps *caps; + + gst_event_parse_caps (event, &caps); + return gst_auto_select_sink_setcaps (autoselect, caps, event); + } + + internal_srcpad = gst_auto_select_get_internal_srcpad (autoselect); + if (internal_srcpad) { + ret = gst_pad_push_event (internal_srcpad, event); + gst_object_unref (internal_srcpad); + } else { + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_FLUSH_STOP: + case GST_EVENT_FLUSH_START: + ret = gst_pad_push_event (autoselect->srcpad, event); + break; + default: + gst_event_unref (event); + ret = TRUE; + break; + } + } + + return ret; +} + +/* TODO Properly test that this code works well for queries */ +static gboolean +gst_auto_select_sink_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + gboolean ret = TRUE; + GstAutoSelect *autoselect = GST_AUTO_SELECT (parent); + GstElement *subelement; + + if (GST_QUERY_TYPE (query) == GST_QUERY_CAPS) { + GstCaps *filter, *caps; + + gst_query_parse_caps (query, &filter); + caps = gst_auto_select_getcaps (autoselect, filter, GST_PAD_SINK); + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + + return TRUE; + } + + subelement = gst_auto_select_get_subelement (autoselect, GST_PAD_SINK); + if (subelement) { + GstPad *sub_sinkpad = get_pad_by_direction (subelement, GST_PAD_SINK); + + ret = gst_pad_query (sub_sinkpad, query); + + gst_object_unref (sub_sinkpad); + gst_object_unref (subelement); + + if (ret && GST_QUERY_TYPE (query) == GST_QUERY_ACCEPT_CAPS) { + gboolean res; + gst_query_parse_accept_caps_result (query, &res); + + if (!res) + goto ignore_acceptcaps_failure; + } + return ret; + } + +ignore_acceptcaps_failure: + + if (GST_QUERY_TYPE (query) == GST_QUERY_ACCEPT_CAPS) { + GstCaps *caps; + GstCaps *accept_caps; + + gst_query_parse_accept_caps (query, &accept_caps); + + caps = gst_auto_select_getcaps (autoselect, accept_caps, GST_PAD_SINK); + gst_query_set_accept_caps_result (query, + gst_caps_can_intersect (caps, accept_caps)); + gst_caps_unref (caps); + + return TRUE; + } + + GST_WARNING_OBJECT (autoselect, "Got query %s while no element was" + " selected, letting through", + gst_query_type_get_name (GST_QUERY_TYPE (query))); + return gst_pad_peer_query (autoselect->srcpad, query); +} + +static GList * +gst_auto_select_copy_factories_list (GstAutoSelect * autoselect) +{ + GList *factories; + + GST_AUTOSELECT_LOCK (autoselect); + factories = g_atomic_pointer_get (&autoselect->factories); + if (factories) + factories = g_list_copy (factories); + GST_AUTOSELECT_UNLOCK (autoselect); + + if (!factories) { + GST_WARNING_OBJECT (autoselect, + "No factory list information, create it itself"); + gst_auto_select_load_factories (autoselect); + GST_AUTOSELECT_LOCK (autoselect); + factories = g_atomic_pointer_get (&autoselect->factories); + if (factories) + factories = g_list_copy (factories); + GST_AUTOSELECT_UNLOCK (autoselect); + } + + return factories; +} + +/** + * gst_auto_select_getcaps: + * @pad: the sink #GstPad + * + * This function returns the union of the caps of all the possible element + * factories, based on the static pad templates. + * It also checks does a getcaps on the downstream element and ignores all + * factories whose static caps can not satisfy it. + * + * It does not try to use each elements getcaps() function + */ + +static GstCaps * +gst_auto_select_getcaps (GstAutoSelect * autoselect, GstCaps * filter, + GstPadDirection dir) +{ + GstCaps *caps = NULL, *other_caps = NULL; + GList *elem; + GList *factories = NULL; + + caps = gst_caps_new_empty (); + + if (dir == GST_PAD_SINK) + other_caps = gst_pad_peer_query_caps (autoselect->srcpad, NULL); + else + other_caps = gst_pad_peer_query_caps (autoselect->sinkpad, NULL); + + GST_DEBUG_OBJECT (autoselect, + "Lets find all the element that can fit here with src caps %" + GST_PTR_FORMAT, other_caps); + + if (other_caps && gst_caps_is_empty (other_caps)) { + goto out; + } + + factories = gst_auto_select_copy_factories_list (autoselect); + if (!factories) + return caps; + + for (elem = factories; elem; elem = g_list_next (elem)) { + GstElementFactory *factory = GST_ELEMENT_FACTORY (elem->data); + + if (filter) { + if (!factory_can_intersect (autoselect, factory, dir, filter)) { + GST_LOG_OBJECT (autoselect, + "Factory %s does not accept filter caps %" GST_PTR_FORMAT, + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE (factory)), filter); + continue; + } + } + if (other_caps != NULL) { + if (!factory_can_intersect (autoselect, factory, + dir == GST_PAD_SINK ? GST_PAD_SRC : GST_PAD_SINK, other_caps)) { + GST_LOG_OBJECT (autoselect, + "Factory %s does not accept other caps %" GST_PTR_FORMAT, + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE (factory)), + other_caps); + continue; + } + } + + const GList *tmp; + for (tmp = gst_element_factory_get_static_pad_templates (factory); + tmp; tmp = g_list_next (tmp)) { + GstStaticPadTemplate *template = tmp->data; + if (template->direction == dir) { + GstCaps *static_caps = gst_static_pad_template_get_caps (template); + + if (static_caps) { + GstCaps *intersection; + if (filter) { + intersection = gst_caps_intersect_full (filter, static_caps, + GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (static_caps); + caps = gst_caps_merge (caps, intersection); + } else { + caps = gst_caps_merge (caps, static_caps); + } + } + + /* Early out, any is absorbing */ + if (gst_caps_is_any (caps)) + goto out; + } + } + } + + GST_DEBUG_OBJECT (autoselect, + "Pad dir: %d, Returning unioned caps %" GST_PTR_FORMAT, dir, caps); + +out: + if (factories) + g_list_free (factories); + if (other_caps) + gst_caps_unref (other_caps); + + return caps; +} + + + +static gboolean +gst_auto_select_src_event (GstPad * pad, GstObject * parent, GstEvent * event) +{ + gboolean ret = TRUE; + GstAutoSelect *autoselect = GST_AUTO_SELECT (parent); + GstPad *internal_sinkpad; + + if (GST_EVENT_TYPE (event) == GST_EVENT_RECONFIGURE) + gst_pad_push_event (autoselect->sinkpad, gst_event_ref (event)); + + internal_sinkpad = gst_auto_select_get_internal_sinkpad (autoselect); + if (internal_sinkpad) { + ret = gst_pad_push_event (internal_sinkpad, event); + gst_object_unref (internal_sinkpad); + } else if (GST_EVENT_TYPE (event) != GST_EVENT_RECONFIGURE) { + GST_WARNING_OBJECT (autoselect, + "Got upstream event while no element was selected," "forwarding."); + ret = gst_pad_push_event (autoselect->sinkpad, event); + } else + gst_event_unref (event); + + return ret; +} + +/* TODO Properly test that this code works well for queries */ +static gboolean +gst_auto_select_src_query (GstPad * pad, GstObject * parent, GstQuery * query) +{ + gboolean ret = TRUE; + GstAutoSelect *autoselect = GST_AUTO_SELECT (parent); + GstElement *subelement; + + if (GST_QUERY_TYPE (query) == GST_QUERY_CAPS) { + GstCaps *filter, *caps; + + gst_query_parse_caps (query, &filter); + caps = gst_auto_select_getcaps (autoselect, filter, GST_PAD_SRC); + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + + return TRUE; + } + + subelement = gst_auto_select_get_subelement (autoselect, GST_PAD_SRC); + if (subelement) { + GstPad *sub_srcpad = get_pad_by_direction (subelement, GST_PAD_SRC); + + ret = gst_pad_query (sub_srcpad, query); + + gst_object_unref (sub_srcpad); + gst_object_unref (subelement); + } else { + GST_WARNING_OBJECT (autoselect, + "Got upstream query of type %s while no element was selected," + " forwarding.", gst_query_type_get_name (GST_QUERY_TYPE (query))); + ret = gst_pad_peer_query (autoselect->sinkpad, query); + } + + return ret; +} + +static GstFlowReturn +gst_auto_select_internal_sink_chain (GstPad * pad, GstObject * parent, + GstBuffer * buffer) +{ + GstAutoSelect *autoselect = + GST_AUTO_SELECT (g_object_get_qdata (G_OBJECT (pad), + parent_quark)); + + return gst_pad_push (autoselect->srcpad, buffer); +} + +static GstFlowReturn +gst_auto_select_internal_sink_chain_list (GstPad * pad, GstObject * parent, + GstBufferList * list) +{ + GstAutoSelect *autoselect = + GST_AUTO_SELECT (g_object_get_qdata (G_OBJECT (pad), + parent_quark)); + + return gst_pad_push_list (autoselect->srcpad, list); +} + +static gboolean +gst_auto_select_internal_sink_event (GstPad * pad, GstObject * parent, + GstEvent * event) +{ + GstAutoSelect *autoselect = + GST_AUTO_SELECT (g_object_get_qdata (G_OBJECT (pad), + parent_quark)); + gboolean drop = FALSE; + + GST_AUTOSELECT_LOCK (autoselect); + if (autoselect->current_internal_sinkpad != pad) { + drop = TRUE; + } + GST_AUTOSELECT_UNLOCK (autoselect); + + if (drop) { + gst_event_unref (event); + return TRUE; + } + + return gst_pad_push_event (autoselect->srcpad, event); +} + +static gboolean +gst_auto_select_internal_sink_query (GstPad * pad, GstObject * parent, + GstQuery * query) +{ + GstAutoSelect *autoselect = + GST_AUTO_SELECT (g_object_get_qdata (G_OBJECT (pad), + parent_quark)); + + if (!gst_pad_peer_query (autoselect->srcpad, query)) { + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CAPS: + { + GstCaps *filter; + + gst_query_parse_caps (query, &filter); + if (filter) { + gst_query_set_caps_result (query, filter); + } else { + filter = gst_caps_new_any (); + gst_query_set_caps_result (query, filter); + gst_caps_unref (filter); + } + return TRUE; + } + case GST_QUERY_ACCEPT_CAPS: + gst_query_set_accept_caps_result (query, TRUE); + return TRUE; + default: + return FALSE; + } + } + + return TRUE; +} + +static gboolean +gst_auto_select_internal_src_event (GstPad * pad, GstObject * parent, + GstEvent * event) +{ + GstAutoSelect *autoselect = + GST_AUTO_SELECT (g_object_get_qdata (G_OBJECT (pad), + parent_quark)); + gboolean drop = FALSE; + + GST_AUTOSELECT_LOCK (autoselect); + if (autoselect->current_internal_srcpad != pad) { + drop = TRUE; + } + GST_AUTOSELECT_UNLOCK (autoselect); + + if (drop) { + GST_DEBUG_OBJECT (autoselect, "Dropping event %" GST_PTR_FORMAT, event); + gst_event_unref (event); + return TRUE; + } + + return gst_pad_push_event (autoselect->sinkpad, event); +} + +static gboolean +gst_auto_select_internal_src_query (GstPad * pad, GstObject * parent, + GstQuery * query) +{ + GstAutoSelect *autoselect = + GST_AUTO_SELECT (g_object_get_qdata (G_OBJECT (pad), + parent_quark)); + + return gst_pad_peer_query (autoselect->sinkpad, query); +} diff --git a/gst/autoselect/gstautoselect.h b/gst/autoselect/gstautoselect.h new file mode 100755 index 0000000000..a61a2b455d --- /dev/null +++ b/gst/autoselect/gstautoselect.h @@ -0,0 +1,107 @@ +/* GStreamer + * + * Copyright 2024 NXP + * @author: Elliot Chen + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + + +#ifndef __GST_AUTO_SELECT_H__ +#define __GST_AUTO_SELECT_H__ + +#include +#include + +G_BEGIN_DECLS +#define GST_TYPE_AUTO_SELECT (gst_auto_select_get_type()) +#define GST_AUTO_SELECT(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_AUTO_SELECT,GstAutoSelect)) +#define GST_AUTO_SELECT_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_AUTO_SELECT,GstAutoSelectClass)) +#define GST_AUTO_SELECT_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_AUTO_SELECT,GstAutoSelectClass)) +#define GST_IS_AUTO_SELECT(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_AUTO_SELECT)) +#define GST_IS_AUTO_SELECT_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_AUTO_SELECT)) +typedef struct _GstAutoSelect GstAutoSelect; +typedef struct _GstAutoSelectClass GstAutoSelectClass; + +struct _GstAutoSelect +{ + /*< private > */ + GstBin bin; + + GList *factories; + + GstPad *sinkpad; + GstPad *srcpad; + + /* Have to be set all at once + * Protected by the object lock and the stream lock + * Both must be held to modify these + */ + GstPad *current_internal_srcpad; + GstPad *current_internal_sinkpad; + + /* record all the elements which + * have beed added + */ + GList *elem_list; + /* whether there are multiple elements */ + gboolean is_composite; + /* Used to trigger trying to combine + * multiple elements + */ + guint composite_rank; + /* The first element in the bin */ + GstElement *first_subelement; + /* The last element in the bin */ + GstElement *last_subelement; + /* Only select the factory whose rank is + * equal to or greater than this value, + * otherwise the factory will be ignored. + */ + guint lowest_rank; + + GstCaps *caps; + GstEvent *event; +}; + +struct _GstAutoSelectClass +{ + GstBinClass parent_class; + + gboolean (*check_current_factory) (GstAutoSelect * autoselect, + GstElementFactory * factory); + + gboolean (*construct_single_element) (GstAutoSelect * autoselect, + GstElementFactory * current_factory); + + gboolean (*update_composite_factories_list) (GstAutoSelect * autoselect, + GList ** factories_list, GstElementFactory * factory); + + gboolean (*construct_composite_elements) (GstAutoSelect * autoselect, + GList * first_factories, GList * last_factories, + GstElementFactory * current_factory); + + gboolean (*add_element) (GstAutoSelect * autoselect, GList * elem_list); + + gboolean (*activate_element) (GstAutoSelect * autoselect, + GList * elem_list); +}; + +GType gst_auto_select_get_type (void); +GST_ELEMENT_REGISTER_DECLARE (autoselect); + +G_END_DECLS +#endif /* __GST_AUTO_SELECT_H__ */ diff --git a/gst/autoselect/gstautovideoconvert2.c b/gst/autoselect/gstautovideoconvert2.c new file mode 100755 index 0000000000..f10669b17d --- /dev/null +++ b/gst/autoselect/gstautovideoconvert2.c @@ -0,0 +1,266 @@ +/* GStreamer + * + * Copyright 2024 NXP + * @author: Elliot Chen + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "gstautovideoconvert2.h" + +GST_DEBUG_CATEGORY (autovideoconvert2_debug); +#define GST_CAT_DEFAULT (autovideoconvert2_debug) + +#define GST_AUTOVIDEOSELECT_LOCK(ac) GST_OBJECT_LOCK (ac) +#define GST_AUTOVIDEOSELECT_UNLOCK(ac) GST_OBJECT_UNLOCK (ac) + +/* element factory information */ +static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GstStaticPadTemplate srctemplate = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +enum +{ + PROP_0, + PROP_PERFERRED_FACTORY_ORDER +}; + +static void gst_auto_video_convert2_dispose (GObject * object); +static GList *gst_auto_video_convert2_create_factory_list (GstAutoVideoConvert2 + * autovideoconvert); +static void gst_auto_video_convert2_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_auto_video_convert2_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); + +G_DEFINE_TYPE (GstAutoVideoConvert2, gst_auto_video_convert2, + GST_TYPE_AUTO_SELECT); +GST_ELEMENT_REGISTER_DEFINE (autovideoconvert2, "autovideoconvert2", + GST_RANK_NONE, GST_TYPE_AUTO_VIDEO_CONVERT2); + +static void +gst_auto_video_convert2_class_init (GstAutoVideoConvert2Class * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstElementClass *gstelement_class = (GstElementClass *) klass; + + GST_DEBUG_CATEGORY_INIT (autovideoconvert2_debug, "autovideoconvert2", 0, + "Auto color space converter 2"); + + gst_element_class_add_static_pad_template (gstelement_class, &srctemplate); + gst_element_class_add_static_pad_template (gstelement_class, &sinktemplate); + + gst_element_class_set_static_metadata (gstelement_class, + "Select or combine color space converters based on caps", "Generic/Bin", + "Select or combine the right color space converters based on the caps", + "Elliot Chen "); + gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_auto_video_convert2_dispose); + gobject_class->get_property = gst_auto_video_convert2_get_property; + gobject_class->set_property = gst_auto_video_convert2_set_property; + + g_object_class_install_property (gobject_class, PROP_PERFERRED_FACTORY_ORDER, + g_param_spec_string ("perferred-factory-order", + "perferred factory order", + "user can configure the perferred factory name order at the start" + " and it can be used to reorder the GList of auto-discovered factories," + " example: perferred-factory-order=videoconvert,videoconvertscale", + NULL, G_PARAM_READWRITE)); +} + +static void +gst_auto_video_convert2_init (GstAutoVideoConvert2 * autovideoconvert) +{ + GList *factories = NULL; + + autovideoconvert->perferred_factory_order = NULL; + factories = gst_auto_video_convert2_create_factory_list (autovideoconvert); + g_object_set (GST_ELEMENT (autovideoconvert), "factories", factories, NULL); + + g_free (factories); + return; +} + +static void +gst_auto_video_convert2_dispose (GObject * object) +{ + GstAutoVideoConvert2 *autovideoconvert = GST_AUTO_VIDEO_CONVERT2 (object); + + g_free (autovideoconvert->perferred_factory_order); + + G_OBJECT_CLASS (gst_auto_video_convert2_parent_class)->dispose (object); +} + +static gboolean +gst_auto_video_convert2_element_filter (GstPluginFeature * feature, + GstAutoVideoConvert2 * autovideoconvert) +{ + const gchar *klass; + + /* we only care about element factories */ + if (G_UNLIKELY (!GST_IS_ELEMENT_FACTORY (feature))) + return FALSE; + + klass = gst_element_factory_get_metadata (GST_ELEMENT_FACTORY_CAST (feature), + GST_ELEMENT_METADATA_KLASS); + + if (strstr (klass, "Filter") && + strstr (klass, "Converter") && strstr (klass, "Video")) { + GST_DEBUG_OBJECT (autovideoconvert, + "gst_auto_video_convert2_element_filter found %s", + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE_CAST (feature))); + return TRUE; + } + + if (strstr (klass, "Filter") && + strstr (klass, "Video") && strstr (klass, "Scaler")) { + GST_DEBUG_OBJECT (autovideoconvert, + "gst_auto_video_convert2_element_filter found %s", + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE_CAST (feature))); + return TRUE; + } + + return FALSE; +} + +static GList * +gst_auto_video_convert2_create_factory_list (GstAutoVideoConvert2 * + autovideoconvert) +{ + GList *result = NULL; + + /* get the feature list using the filter */ + result = gst_registry_feature_filter (gst_registry_get (), + (GstPluginFeatureFilter) gst_auto_video_convert2_element_filter, + FALSE, autovideoconvert); + + /* sort on rank and name */ + result = g_list_sort (result, gst_plugin_feature_rank_compare_func); + + return result; +} + +static gboolean +gst_auto_video_convert2_update_factory_list (GstAutoVideoConvert2 * + autovideoconvert, gchar * perferred_factory_order) +{ + gchar **factories_str = NULL; + gchar **factory_name = NULL; + GList *factories = NULL; + GList *curr = NULL; + GList *result = NULL; + + /* Check the perferred factory name */ + if (!perferred_factory_order) { + GST_DEBUG_OBJECT (autovideoconvert, "The perferred factory name is NULL"); + return FALSE; + } else { + factories_str = g_strsplit (perferred_factory_order, ",", -1); + if (!(*factories_str)) { + GST_DEBUG_OBJECT (autovideoconvert, + "Can't get valid perferred factory name"); + return FALSE; + } + g_free (autovideoconvert->perferred_factory_order); + autovideoconvert->perferred_factory_order = perferred_factory_order; + } + + /* Get the currnet factory list */ + g_object_get (GST_ELEMENT (autovideoconvert), "factories", &factories, NULL); + if (!factories) { + GST_DEBUG_OBJECT (autovideoconvert, "The factory list is NULL"); + return FALSE; + } else { + factories = g_list_copy (factories); + } + + /* reorder the factory list */ + for (factory_name = factories_str; *factory_name; factory_name++) { + for (curr = factories; curr; curr = g_list_next (curr)) { + GstElementFactory *factory = GST_ELEMENT_FACTORY (curr->data); + + if (strcmp (*factory_name, + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE (factory)))) { + continue; + } + factories = g_list_remove_link (factories, curr); + result = g_list_concat (result, curr); + GST_DEBUG_OBJECT (autovideoconvert, "reorder list, factory name %s", + gst_plugin_feature_get_name (GST_PLUGIN_FEATURE (factory))); + break; + } + } + result = g_list_concat (result, factories); + /* Update the factory list */ + g_object_set (GST_ELEMENT (autovideoconvert), "factories", result, NULL); + + if (result) + g_list_free (result); + g_strfreev (factories_str); + return TRUE; +} + +static void +gst_auto_video_convert2_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstState cur_state; + GstAutoVideoConvert2 *autovideoconvert = GST_AUTO_VIDEO_CONVERT2 (object); + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + case PROP_PERFERRED_FACTORY_ORDER: + gst_element_get_state (GST_ELEMENT (autovideoconvert), &cur_state, NULL, + 0); + if (cur_state == GST_STATE_NULL) { + gst_auto_video_convert2_update_factory_list (autovideoconvert, + g_value_dup_string (value)); + } else { + GST_WARNING_OBJECT (object, "Can not set perferred factory order" + "because element is not in the NULL state or initial state"); + } + break; + } +} + +static void +gst_auto_video_convert2_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstAutoVideoConvert2 *autovideoconvert = GST_AUTO_VIDEO_CONVERT2 (object); + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + case PROP_PERFERRED_FACTORY_ORDER: + g_value_set_string (value, autovideoconvert->perferred_factory_order); + break; + } +} diff --git a/gst/autoselect/gstautovideoconvert2.h b/gst/autoselect/gstautovideoconvert2.h new file mode 100755 index 0000000000..931125a1cc --- /dev/null +++ b/gst/autoselect/gstautovideoconvert2.h @@ -0,0 +1,53 @@ +/* GStreamer + * + * Copyright 2024 NXP + * @author: Elliot Chen + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_AUTO_VIDEO_CONVERT2_H__ +#define __GST_AUTO_VIDEO_CONVERT2_H__ + +#include +#include +#include "gstautoselect.h" + +G_BEGIN_DECLS +#define GST_TYPE_AUTO_VIDEO_CONVERT2 (gst_auto_video_convert2_get_type()) +#define GST_AUTO_VIDEO_CONVERT2(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_AUTO_VIDEO_CONVERT2,GstAutoVideoConvert2)) +#define GST_AUTO_VIDEO_CONVERT2_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_AUTO_VIDEO_CONVERT2,GstAutoVideoConvert2Class)) +#define GST_IS_AUTO_VIDEO_CONVERT2(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_AUTO_VIDEO_CONVERT2)) +#define GST_IS_AUTO_VIDEO_CONVERT2_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_AUTO_VIDEO_CONVERT2)) +typedef struct _GstAutoVideoConvert2 GstAutoVideoConvert2; +typedef struct _GstAutoVideoConvert2Class GstAutoVideoConvert2Class; + +struct _GstAutoVideoConvert2 +{ + GstAutoSelect element; + gchar *perferred_factory_order; +}; + +struct _GstAutoVideoConvert2Class +{ + GstAutoSelectClass parent_class; +}; + +GType gst_auto_video_convert2_get_type (void); +GST_ELEMENT_REGISTER_DECLARE (autovideoconvert2); + +G_END_DECLS +#endif /* __GST_AUTO_VIDEO_CONVERT2_H__ */ diff --git a/gst/autoselect/meson.build b/gst/autoselect/meson.build new file mode 100755 index 0000000000..8dc0319d6d --- /dev/null +++ b/gst/autoselect/meson.build @@ -0,0 +1,15 @@ +autocon_sources = [ + 'gstautoselect.c', + 'gstautovideoconvert2.c', + 'plugin.c', +] + +gstautoselect = library('gstautoselect', + autocon_sources, + c_args : gst_plugins_bad_args, + include_directories : [configinc], + dependencies : [gstbase_dep], + install : true, + install_dir : plugins_install_dir, +) +plugins += [gstautoselect] diff --git a/gst/autoselect/plugin.c b/gst/autoselect/plugin.c new file mode 100755 index 0000000000..bb26dd000a --- /dev/null +++ b/gst/autoselect/plugin.c @@ -0,0 +1,43 @@ +/* GStreamer + * Copyright 2024 NXP + * @author: Elliot Chen + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstautoselect.h" +#include "gstautovideoconvert2.h" + +static gboolean +plugin_init (GstPlugin * plugin) +{ + gboolean ret = FALSE; + + ret |= GST_ELEMENT_REGISTER (autoselect, plugin); + ret |= GST_ELEMENT_REGISTER (autovideoconvert2, plugin); + + return ret; +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + autoselect, + "Selects single or multiple elements based on caps", + plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/meson.build b/gst/meson.build index 4232a34aef..9925c20c50 100644 --- a/gst/meson.build +++ b/gst/meson.build @@ -1,6 +1,6 @@ foreach plugin : ['accurip', 'adpcmdec', 'adpcmenc', 'aiff', 'asfmux', 'audiobuffersplit', 'audiofxbad', 'audiomixmatrix', - 'audiolatency', 'audiovisualizers', 'autoconvert', 'bayer', + 'audiolatency', 'audiovisualizers', 'autoconvert', 'autoselect', 'bayer', 'camerabin2', 'codecalpha', 'codectimestamper', 'coloreffects', 'debugutils', 'dvbsubenc', 'dvbsuboverlay', 'dvdspu', 'faceoverlay', 'festival', diff --git a/meson_options.txt b/meson_options.txt index 8ab06d23b3..202539ae17 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -14,6 +14,7 @@ option('audiolatency', type : 'feature', value : 'auto') option('audiomixmatrix', type : 'feature', value : 'auto') option('audiovisualizers', type : 'feature', value : 'auto') option('autoconvert', type : 'feature', value : 'auto') +option('autoselect', type : 'feature', value : 'auto') option('bayer', type : 'feature', value : 'auto') option('camerabin2', type : 'feature', value : 'auto') option('codec2json', type : 'feature', value : 'auto') -- 2.25.1 From c0f2a68cf135a57a55e422ca35794296157ffb7b Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Thu, 29 Mar 2018 11:38:48 +0800 Subject: [PATCH 26/65] MMFMWK-7954 waylandsink: add fps print upstream status: imx specific (cherry picked from commit 95bfb3672b4bce461cff5b82a20722cfd395c8a6) --- ext/wayland/gstwaylandsink.c | 14 ++++++++++++++ ext/wayland/gstwaylandsink.h | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index 1bf537c9f3..09a360b348 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -201,6 +201,8 @@ gst_wayland_sink_init (GstWaylandSink * self) { g_mutex_init (&self->display_lock); g_mutex_init (&self->render_lock); + self->frame_showed = 0; + self->run_time = 0; } static void @@ -437,6 +439,9 @@ gst_wayland_sink_change_state (GstElement * element, GstStateChange transition) return ret; switch (transition) { + case GST_STATE_CHANGE_PLAYING_TO_PAUSED: + self->run_time = gst_element_get_start_time (element); + break; case GST_STATE_CHANGE_PAUSED_TO_READY: gst_buffer_replace (&self->last_buffer, NULL); if (self->window) { @@ -466,6 +471,14 @@ gst_wayland_sink_change_state (GstElement * element, GstStateChange transition) g_mutex_unlock (&self->display_lock); g_clear_object (&self->pool); + + if (self->run_time > 0) { + g_print ("Total showed frames (%lld), playing for (%"GST_TIME_FORMAT"), fps (%.3f).\n", + self->frame_showed, GST_TIME_ARGS (self->run_time), + (gfloat)GST_SECOND * self->frame_showed / self->run_time); + } + self->frame_showed = 0; + self->run_time = 0; break; default: break; @@ -1077,6 +1090,7 @@ dst_map_failed: } done: { + self->frame_showed++; g_mutex_unlock (&self->render_lock); return ret; } diff --git a/ext/wayland/gstwaylandsink.h b/ext/wayland/gstwaylandsink.h index 5bbb10a8e3..294c9ea039 100644 --- a/ext/wayland/gstwaylandsink.h +++ b/ext/wayland/gstwaylandsink.h @@ -69,6 +69,10 @@ struct _GstWaylandSink gchar *drm_device; gboolean skip_dumb_buffer_copy; + + /* fps print support */ + guint64 frame_showed; + GstClockTime run_time; }; struct _GstWaylandSinkClass -- 2.25.1 From f9abdebfe9f54b166e7458f0cd289dbf595c694a Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Fri, 8 Jun 2018 16:52:44 +0800 Subject: [PATCH 27/65] MMFMWK-8030 waylandsink: support video crop meta using viewporter protocol use API wp_viewport_set_source() to support video crop meta handle upstream status: Pending https://bugzilla.gnome.org/show_bug.cgi?id=796541 --- ext/wayland/gstwaylandsink.c | 3 +++ gst-libs/gst/wayland/gstwlwindow.c | 38 +++++++++++++++++++++++++++--- gst-libs/gst/wayland/gstwlwindow.h | 3 +++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index 09a360b348..42671f0304 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -812,6 +812,7 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) alloc = gst_shm_allocator_get (); gst_query_add_allocation_param (query, alloc, NULL); gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + gst_query_add_allocation_meta (query, GST_VIDEO_CROP_META_API_TYPE, NULL); g_object_unref (alloc); return TRUE; @@ -880,6 +881,8 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) if (G_UNLIKELY (gst_wl_window_get_render_rectangle (self->window)->w == 0)) goto no_window_size; + gst_wl_window_set_source_crop (self->window, buffer); + wlbuffer = gst_buffer_get_wl_buffer (self->display, buffer); if (G_LIKELY (wlbuffer && diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index 27084c0d1a..12bdd72551 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -80,6 +80,9 @@ typedef struct _GstWlWindowPrivate gboolean clear_window; struct wl_callback *frame_callback; struct wl_callback *commit_callback; + + /* the coordinate of video crop */ + gint src_x, src_y, src_width, src_height; } GstWlWindowPrivate; G_DEFINE_TYPE_WITH_CODE (GstWlWindow, gst_wl_window, G_TYPE_OBJECT, @@ -185,6 +188,11 @@ gst_wl_window_init (GstWlWindow * self) g_cond_init (&priv->configure_cond); g_mutex_init (&priv->configure_mutex); g_mutex_init (&priv->window_lock); + + priv->src_x = 0; + priv->src_y = 0; + priv->src_width = -1; + priv->src_height = 0; } static void @@ -459,6 +467,11 @@ gst_wl_window_resize_video_surface (GstWlWindow * self, gboolean commit) int wp_src_width; int wp_src_height; + wl_fixed_t src_x = wl_fixed_from_int (priv->src_x); + wl_fixed_t src_y = wl_fixed_from_int (priv->src_y); + wl_fixed_t src_width = wl_fixed_from_int (priv->src_width); + wl_fixed_t src_height = wl_fixed_from_int (priv->src_height); + switch (priv->buffer_transform) { case WL_OUTPUT_TRANSFORM_NORMAL: case WL_OUTPUT_TRANSFORM_180: @@ -488,10 +501,10 @@ gst_wl_window_resize_video_surface (GstWlWindow * self, gboolean commit) /* center the video_subsurface inside area_subsurface */ if (priv->video_viewport) { gst_video_center_rect (&src, &dst, &res, TRUE); - wp_viewport_set_source (priv->video_viewport, wl_fixed_from_int (0), - wl_fixed_from_int (0), wl_fixed_from_int (wp_src_width), - wl_fixed_from_int (wp_src_height)); wp_viewport_set_destination (priv->video_viewport, res.w, res.h); + if (src_width != wl_fixed_from_int(-1)) + wp_viewport_set_source (priv->video_viewport, + src_x, src_y, src_width, src_height); } else { gst_video_center_rect (&src, &dst, &res, FALSE); } @@ -786,6 +799,25 @@ gst_wl_window_set_render_rectangle (GstWlWindow * self, gint x, gint y, gst_wl_window_update_geometry (self); } +void +gst_wl_window_set_source_crop (GstWlWindow * self, GstBuffer * buffer) +{ + GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); + GstVideoCropMeta *crop = NULL; + crop = gst_buffer_get_video_crop_meta(buffer); + + if (crop) { + GST_DEBUG ("buffer crop x=%d y=%d width=%d height=%d\n", + crop->x, crop->y, crop->width, crop->height); + priv->src_x = crop->x; + priv->src_y = crop->y; + priv->src_width = crop->width; + priv->src_height = crop->height; + } else { + priv->src_width = -1; + } +} + const GstVideoRectangle * gst_wl_window_get_render_rectangle (GstWlWindow * self) { diff --git a/gst-libs/gst/wayland/gstwlwindow.h b/gst-libs/gst/wayland/gstwlwindow.h index 4cd85ac36a..f048230c83 100644 --- a/gst-libs/gst/wayland/gstwlwindow.h +++ b/gst-libs/gst/wayland/gstwlwindow.h @@ -67,6 +67,9 @@ GST_WL_API void gst_wl_window_set_render_rectangle (GstWlWindow * self, gint x, gint y, gint w, gint h); +GST_WL_API +void gst_wl_window_set_source_crop (GstWlWindow * self, GstBuffer * buffer); + GST_WL_API const GstVideoRectangle *gst_wl_window_get_render_rectangle (GstWlWindow * self); -- 2.25.1 From 0ea97beffa716fb5807eaa878ae56b6b895d12fd Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Thu, 21 Jun 2018 13:26:37 +0800 Subject: [PATCH 28/65] MMFMWK-7954 waylandsink: implement alpha blending for surface 1. implement alpha blending procotol 2. don't set area_surface to opaque 3. reset alpha when exit upstream status: imx specific --- ext/wayland/gstwaylandsink.c | 31 ++++++++++-- ext/wayland/gstwaylandsink.h | 3 ++ ext/wayland/meson.build | 5 +- gst-libs/gst/wayland/gstwldisplay.c | 16 ++++++ gst-libs/gst/wayland/gstwldisplay.h | 3 ++ gst-libs/gst/wayland/gstwlwindow.c | 78 ++++++++++++++++++++++++++--- gst-libs/gst/wayland/gstwlwindow.h | 12 +++++ gst-libs/gst/wayland/meson.build | 1 + 8 files changed, 138 insertions(+), 11 deletions(-) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index 42671f0304..27fac5e374 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -61,6 +61,7 @@ enum PROP_0, PROP_DISPLAY, PROP_FULLSCREEN, + PROP_ALPHA, PROP_ROTATE_METHOD, PROP_DRM_DEVICE, PROP_LAST @@ -161,6 +162,11 @@ gst_wayland_sink_class_init (GstWaylandSinkClass * klass) "Whether the surface should be made fullscreen ", FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_ALPHA, + g_param_spec_float ("alpha", "Wayland surface alpha", "Wayland " + "surface alpha value, apply custom alpha value to wayland surface", + 0.0f, 1.0f, 0.0f, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /** * waylandsink:rotate-method: * @@ -199,6 +205,7 @@ gst_wayland_sink_class_init (GstWaylandSinkClass * klass) static void gst_wayland_sink_init (GstWaylandSink * self) { + self->alpha = 0.0f; g_mutex_init (&self->display_lock); g_mutex_init (&self->render_lock); self->frame_showed = 0; @@ -270,6 +277,8 @@ gst_wayland_sink_get_property (GObject * object, GST_OBJECT_LOCK (self); g_value_set_boolean (value, self->fullscreen); GST_OBJECT_UNLOCK (self); + case PROP_ALPHA: + g_value_set_float (value, self->alpha); break; case PROP_ROTATE_METHOD: GST_OBJECT_LOCK (self); @@ -303,6 +312,8 @@ gst_wayland_sink_set_property (GObject * object, GST_OBJECT_LOCK (self); gst_wayland_sink_set_fullscreen (self, g_value_get_boolean (value)); GST_OBJECT_UNLOCK (self); + case PROP_ALPHA: + self->alpha = g_value_get_float (value); break; case PROP_ROTATE_METHOD: gst_wayland_sink_set_rotate_method (self, g_value_get_enum (value), @@ -424,6 +435,9 @@ gst_wayland_sink_change_state (GstElement * element, GstStateChange transition) { GstWaylandSink *self = GST_WAYLAND_SINK (element); GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + struct wl_surface *area_surface; + struct wl_display *display_display; + gint render_rectangle_w, render_rectangle_h; switch (transition) { case GST_STATE_CHANGE_NULL_TO_READY: @@ -445,6 +459,15 @@ gst_wayland_sink_change_state (GstElement * element, GstStateChange transition) case GST_STATE_CHANGE_PAUSED_TO_READY: gst_buffer_replace (&self->last_buffer, NULL); if (self->window) { + area_surface = gst_wl_window_get_area_surface (self->window); + render_rectangle_w = gst_wl_window_get_rectangle_w (self->window); + render_rectangle_h = gst_wl_window_get_rectangle_h (self->window); + display_display = gst_wl_display_get_display (self->display); + gst_wl_window_set_alpha (self->window, 1.0); + wl_surface_damage (area_surface, 0, 0, + render_rectangle_w, render_rectangle_h); + wl_surface_commit (area_surface); + wl_display_roundtrip (display_display); if (gst_wl_window_is_toplevel (self->window)) { g_clear_object (&self->window); } else { @@ -473,9 +496,10 @@ gst_wayland_sink_change_state (GstElement * element, GstStateChange transition) g_clear_object (&self->pool); if (self->run_time > 0) { - g_print ("Total showed frames (%lld), playing for (%"GST_TIME_FORMAT"), fps (%.3f).\n", - self->frame_showed, GST_TIME_ARGS (self->run_time), - (gfloat)GST_SECOND * self->frame_showed / self->run_time); + g_print ("Total showed frames (%"G_GUINT64_FORMAT"), playing for (%" GST_TIME_FORMAT + "), fps (%.3f).\n", self->frame_showed, + GST_TIME_ARGS (self->run_time), + (gfloat) GST_SECOND * self->frame_showed / self->run_time); } self->frame_showed = 0; self->run_time = 0; @@ -875,6 +899,7 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) gst_wl_window_set_rotate_method (self->window, self->current_rotate_method); } + gst_wl_window_set_alpha (self->window, self->alpha); } /* make sure that the application has called set_render_rectangle() */ diff --git a/ext/wayland/gstwaylandsink.h b/ext/wayland/gstwaylandsink.h index 294c9ea039..47a69410e2 100644 --- a/ext/wayland/gstwaylandsink.h +++ b/ext/wayland/gstwaylandsink.h @@ -52,6 +52,9 @@ struct _GstWaylandSink GstWlWindow *window; GstBufferPool *pool; + /* alpha compositing */ + gfloat alpha; + gboolean video_info_changed; GstVideoInfo video_info; GstVideoInfoDmaDrm drm_info; diff --git a/ext/wayland/meson.build b/ext/wayland/meson.build index 29c7e2febf..f0a501a2c0 100644 --- a/ext/wayland/meson.build +++ b/ext/wayland/meson.build @@ -5,10 +5,11 @@ wl_sources = [ if use_wayland gstwaylandsink = library('gstwaylandsink', - wl_sources, + wl_sources + protocols_files, c_args : gst_plugins_bad_args + ['-DGST_USE_UNSTABLE_API'], include_directories : [configinc], - dependencies : [gst_dep, gstvideo_dep, gstwayland_dep], + dependencies : [gst_dep, gstvideo_dep, gstwayland_dep, + wl_client_dep, wl_protocol_dep, libdrm_dep], install : true, install_dir : plugins_install_dir, ) diff --git a/gst-libs/gst/wayland/gstwldisplay.c b/gst-libs/gst/wayland/gstwldisplay.c index 9119f1df77..9e4b2b4a7d 100644 --- a/gst-libs/gst/wayland/gstwldisplay.c +++ b/gst-libs/gst/wayland/gstwldisplay.c @@ -29,6 +29,7 @@ #include "single-pixel-buffer-v1-client-protocol.h" #include "viewporter-client-protocol.h" #include "xdg-shell-client-protocol.h" +#include "alpha-compositing-unstable-v1-client-protocol.h" #include #include @@ -53,6 +54,7 @@ typedef struct _GstWlDisplayPrivate struct wl_shm *shm; struct wp_viewporter *viewporter; struct zwp_linux_dmabuf_v1 *dmabuf; + struct zwp_alpha_compositing_v1 *alpha_compositing; GArray *shm_formats; GArray *dmabuf_formats; GArray *dmabuf_modifiers; @@ -155,6 +157,9 @@ gst_wl_display_finalize (GObject * gobject) if (priv->single_pixel_buffer) wp_single_pixel_buffer_manager_v1_destroy (priv->single_pixel_buffer); + if (priv->alpha_compositing) + zwp_alpha_compositing_v1_destroy (priv->alpha_compositing); + if (priv->compositor) wl_compositor_destroy (priv->compositor); @@ -336,6 +341,9 @@ registry_handle_global (void *data, struct wl_registry *registry, priv->single_pixel_buffer = wl_registry_bind (registry, id, &wp_single_pixel_buffer_manager_v1_interface, 1); + } else if (g_strcmp0 (interface, "zwp_alpha_compositing_v1") == 0) { + priv->alpha_compositing = + wl_registry_bind (registry, id, &zwp_alpha_compositing_v1_interface, 1); } } @@ -638,6 +646,14 @@ gst_wl_display_get_viewporter (GstWlDisplay * self) return priv->viewporter; } +struct zwp_alpha_compositing_v1 * +gst_wl_display_get_alpha_compositing (GstWlDisplay * self) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + return priv->alpha_compositing; +} + struct wl_shm * gst_wl_display_get_shm (GstWlDisplay * self) { diff --git a/gst-libs/gst/wayland/gstwldisplay.h b/gst-libs/gst/wayland/gstwldisplay.h index 3d29b4ea46..8d74c6251e 100644 --- a/gst-libs/gst/wayland/gstwldisplay.h +++ b/gst-libs/gst/wayland/gstwldisplay.h @@ -91,6 +91,9 @@ struct zwp_fullscreen_shell_v1 *gst_wl_display_get_fullscreen_shell_v1 (GstWlDis GST_WL_API struct wp_viewporter *gst_wl_display_get_viewporter (GstWlDisplay * self); +GST_WL_API +struct zwp_alpha_compositing_v1 *gst_wl_display_get_alpha_compositing (GstWlDisplay * self); + GST_WL_API struct wl_shm *gst_wl_display_get_shm (GstWlDisplay * self); diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index 12bdd72551..4a0a9c8c13 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -25,11 +25,13 @@ #endif #include "gstwlwindow.h" +#include "gstimxcommon.h" #include "fullscreen-shell-unstable-v1-client-protocol.h" #include "single-pixel-buffer-v1-client-protocol.h" #include "viewporter-client-protocol.h" #include "xdg-shell-client-protocol.h" +#include "alpha-compositing-unstable-v1-client-protocol.h" #define GST_CAT_DEFAULT gst_wl_window_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); @@ -55,6 +57,9 @@ typedef struct _GstWlWindowPrivate GCond configure_cond; GMutex configure_mutex; + struct wl_shell_surface *shell_surface; + struct zwp_blending_v1 *blend_func; + /* the size and position of the area_(sub)surface */ GstVideoRectangle render_rectangle; @@ -220,6 +225,10 @@ gst_wl_window_finalize (GObject * gobject) wp_viewport_destroy (priv->video_viewport); wl_proxy_wrapper_destroy (priv->video_surface_wrapper); + + if (priv->blend_func) + zwp_blending_v1_destroy (priv->blend_func); + wl_subsurface_destroy (priv->video_subsurface); wl_surface_destroy (priv->video_surface); @@ -246,6 +255,7 @@ gst_wl_window_new_internal (GstWlDisplay * display, GMutex * render_lock) struct wl_event_queue *event_queue; struct wl_region *region; struct wp_viewporter *viewporter; + struct zwp_alpha_compositing_v1 *alpha_compositing; self = g_object_new (GST_TYPE_WL_WINDOW, NULL); priv = gst_wl_window_get_instance_private (self); @@ -280,6 +290,12 @@ gst_wl_window_new_internal (GstWlDisplay * display, GMutex * render_lock) priv->video_surface); } + alpha_compositing = gst_wl_display_get_alpha_compositing (display); + if (alpha_compositing) + priv->blend_func = + zwp_alpha_compositing_v1_get_blending (alpha_compositing, + priv->area_surface); + /* never accept input events on the video surface */ region = wl_compositor_create_region (compositor); wl_surface_set_input_region (priv->video_surface, region); @@ -446,6 +462,39 @@ gst_wl_window_get_subsurface (GstWlWindow * self) return priv->area_subsurface; } +struct wl_surface * +gst_wl_window_get_area_surface (GstWlWindow * self) +{ + GstWlWindowPrivate *priv; + + g_return_val_if_fail (self != NULL, NULL); + + priv = gst_wl_window_get_instance_private (self); + return priv->area_surface; +} + +gint +gst_wl_window_get_rectangle_w (GstWlWindow * self) +{ + GstWlWindowPrivate *priv; + + g_return_val_if_fail (self != NULL, -1); + + priv = gst_wl_window_get_instance_private (self); + return priv->render_rectangle.w; +} + +gint +gst_wl_window_get_rectangle_h (GstWlWindow * self) +{ + GstWlWindowPrivate *priv; + + g_return_val_if_fail (self != NULL, -1); + + priv = gst_wl_window_get_instance_private (self); + return priv->render_rectangle.h; +} + gboolean gst_wl_window_is_toplevel (GstWlWindow * self) { @@ -502,7 +551,7 @@ gst_wl_window_resize_video_surface (GstWlWindow * self, gboolean commit) if (priv->video_viewport) { gst_video_center_rect (&src, &dst, &res, TRUE); wp_viewport_set_destination (priv->video_viewport, res.w, res.h); - if (src_width != wl_fixed_from_int(-1)) + if (src_width != wl_fixed_from_int (-1)) wp_viewport_set_source (priv->video_viewport, src_x, src_y, src_width, src_height); } else { @@ -528,12 +577,12 @@ gst_wl_window_set_opaque (GstWlWindow * self, const GstVideoInfo * info) /* Set area opaque */ compositor = gst_wl_display_get_compositor (priv->display); - region = wl_compositor_create_region (compositor); - wl_region_add (region, 0, 0, G_MAXINT32, G_MAXINT32); - wl_surface_set_opaque_region (priv->area_surface, region); - wl_region_destroy (region); if (!GST_VIDEO_INFO_HAS_ALPHA (info)) { + /* for platform support overlay, video should not overlap graphic */ + if (HAS_DCSS () || HAS_DPU ()) + return; + /* Set video opaque */ region = wl_compositor_create_region (compositor); wl_region_add (region, 0, 0, G_MAXINT32, G_MAXINT32); @@ -804,7 +853,7 @@ gst_wl_window_set_source_crop (GstWlWindow * self, GstBuffer * buffer) { GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); GstVideoCropMeta *crop = NULL; - crop = gst_buffer_get_video_crop_meta(buffer); + crop = gst_buffer_get_video_crop_meta (buffer); if (crop) { GST_DEBUG ("buffer crop x=%d y=%d width=%d height=%d\n", @@ -818,6 +867,23 @@ gst_wl_window_set_source_crop (GstWlWindow * self, GstBuffer * buffer) } } +void +gst_wl_window_set_alpha (GstWlWindow * self, gfloat alpha) +{ + GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); + + if (priv && priv->blend_func) { + zwp_blending_v1_set_alpha (priv->blend_func, + wl_fixed_from_double (alpha)); + if (alpha < 1.0) + zwp_blending_v1_set_blending (priv->blend_func, + ZWP_BLENDING_V1_BLENDING_EQUATION_FROMSOURCE); + else + zwp_blending_v1_set_blending (priv->blend_func, + ZWP_BLENDING_V1_BLENDING_EQUATION_PREMULTIPLIED); + } +} + const GstVideoRectangle * gst_wl_window_get_render_rectangle (GstWlWindow * self) { diff --git a/gst-libs/gst/wayland/gstwlwindow.h b/gst-libs/gst/wayland/gstwlwindow.h index f048230c83..611853bc1c 100644 --- a/gst-libs/gst/wayland/gstwlwindow.h +++ b/gst-libs/gst/wayland/gstwlwindow.h @@ -56,6 +56,15 @@ struct wl_surface *gst_wl_window_get_wl_surface (GstWlWindow * self); GST_WL_API struct wl_subsurface *gst_wl_window_get_subsurface (GstWlWindow * self); +GST_WL_API +struct wl_surface *gst_wl_window_get_area_surface (GstWlWindow * self); + +GST_WL_API +gint gst_wl_window_get_rectangle_w (GstWlWindow * self); + +GST_WL_API +gint gst_wl_window_get_rectangle_h (GstWlWindow * self); + GST_WL_API gboolean gst_wl_window_is_toplevel (GstWlWindow * self); @@ -70,6 +79,9 @@ void gst_wl_window_set_render_rectangle (GstWlWindow * self, gint x, gint y, GST_WL_API void gst_wl_window_set_source_crop (GstWlWindow * self, GstBuffer * buffer); +GST_WL_API +void gst_wl_window_set_alpha (GstWlWindow * self, gfloat alpha); + GST_WL_API const GstVideoRectangle *gst_wl_window_get_render_rectangle (GstWlWindow * self); diff --git a/gst-libs/gst/wayland/meson.build b/gst-libs/gst/wayland/meson.build index 75251dbeac..cf6a0e2d19 100644 --- a/gst-libs/gst/wayland/meson.build +++ b/gst-libs/gst/wayland/meson.build @@ -41,6 +41,7 @@ if use_wayland ['fullscreen-shell', 'unstable', 'v1', ], ['xdg-shell', 'stable', ], ['single-pixel-buffer', 'staging', 'v1' ], + ['alpha-compositing', 'unstable', 'v1', ], ] protocols_files = [] -- 2.25.1 From 88fee82e9b0ccc0e3496c0fde4b13cfe0a6f38c0 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Thu, 5 Jul 2018 19:03:48 +0800 Subject: [PATCH 29/65] MMFMWK-7954 waylandsink: apply surface buffer scale for 850D get real display solution using wl_output interface. Compare it with desktop size to decide which scale to use. defaultly, make scale = 1 for compitable purpose. upstream status: imx specific --- gst-libs/gst/wayland/gstwldisplay.c | 69 +++++++++++++ gst-libs/gst/wayland/gstwldisplay.h | 6 ++ gst-libs/gst/wayland/gstwlutils.c | 144 ++++++++++++++++++++++++++++ gst-libs/gst/wayland/gstwlutils.h | 31 ++++++ gst-libs/gst/wayland/gstwlwindow.c | 28 ++++-- gst-libs/gst/wayland/meson.build | 2 + 6 files changed, 274 insertions(+), 6 deletions(-) create mode 100644 gst-libs/gst/wayland/gstwlutils.c create mode 100644 gst-libs/gst/wayland/gstwlutils.h diff --git a/gst-libs/gst/wayland/gstwldisplay.c b/gst-libs/gst/wayland/gstwldisplay.c index 9e4b2b4a7d..3a453ac35f 100644 --- a/gst-libs/gst/wayland/gstwldisplay.c +++ b/gst-libs/gst/wayland/gstwldisplay.c @@ -52,6 +52,7 @@ typedef struct _GstWlDisplayPrivate struct zwp_fullscreen_shell_v1 *fullscreen_shell; struct wp_single_pixel_buffer_manager_v1 *single_pixel_buffer; struct wl_shm *shm; + struct wl_output *output; struct wp_viewporter *viewporter; struct zwp_linux_dmabuf_v1 *dmabuf; struct zwp_alpha_compositing_v1 *alpha_compositing; @@ -59,6 +60,9 @@ typedef struct _GstWlDisplayPrivate GArray *dmabuf_formats; GArray *dmabuf_modifiers; + /* real display resolution */ + gint width, height; + /* private */ gboolean own_display; GThread *thread; @@ -96,6 +100,8 @@ gst_wl_display_init (GstWlDisplay * self) priv->dmabuf_modifiers = g_array_new (FALSE, FALSE, sizeof (guint64)); priv->wl_fd_poll = gst_poll_new (TRUE); priv->buffers = g_hash_table_new (g_direct_hash, g_direct_equal); + priv->width = -1; + priv->height = -1; g_mutex_init (&priv->buffers_mutex); g_rec_mutex_init (&priv->sync_mutex); @@ -307,6 +313,50 @@ static const struct xdg_wm_base_listener xdg_wm_base_listener = { handle_xdg_wm_base_ping }; +output_handle_geometry (void *data, struct wl_output *wl_output, + int32_t x, int32_t y, + int32_t physical_width, int32_t physical_height, + int32_t subpixel, + const char *make, const char *model, int32_t output_transform) +{ + /* Nothing to do now */ +} + +static void +output_handle_mode (void *data, struct wl_output *wl_output, + uint32_t flags, int32_t width, int32_t height, int32_t refresh) +{ + GstWlDisplay *self = data; + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + /* we only care about the current mode */ + if (flags & WL_OUTPUT_MODE_CURRENT) { + priv->width = width; + priv->height = height; + } +} + +static void +output_handle_done (void *data, struct wl_output *wl_output) +{ + /* don't bother waiting for this; there's no good reason a + * compositor will wait more than one roundtrip before sending + * these initial events. */ +} + +static void +output_handle_scale (void *data, struct wl_output *wl_output, int32_t scale) +{ + /* Nothing to do now */ +} + +static const struct wl_output_listener output_listener = { + output_handle_geometry, + output_handle_mode, + output_handle_done, + output_handle_scale, +}; + static void registry_handle_global (void *data, struct wl_registry *registry, uint32_t id, const char *interface, uint32_t version) @@ -344,6 +394,10 @@ registry_handle_global (void *data, struct wl_registry *registry, } else if (g_strcmp0 (interface, "zwp_alpha_compositing_v1") == 0) { priv->alpha_compositing = wl_registry_bind (registry, id, &zwp_alpha_compositing_v1_interface, 1); + } else if (g_strcmp0 (interface, "wl_output") == 0) { + priv->output = + wl_registry_bind (registry, id, &wl_output_interface, MIN (version, 2)); + wl_output_add_listener (priv->output, &output_listener, self); } } @@ -654,6 +708,21 @@ gst_wl_display_get_alpha_compositing (GstWlDisplay * self) return priv->alpha_compositing; } +gint +gst_wl_display_get_width (GstWlDisplay * self) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + return priv->width; +} + +gint +gst_wl_display_get_height (GstWlDisplay * self) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + return priv->height; +} struct wl_shm * gst_wl_display_get_shm (GstWlDisplay * self) { diff --git a/gst-libs/gst/wayland/gstwldisplay.h b/gst-libs/gst/wayland/gstwldisplay.h index 8d74c6251e..6c5eda7e25 100644 --- a/gst-libs/gst/wayland/gstwldisplay.h +++ b/gst-libs/gst/wayland/gstwldisplay.h @@ -94,6 +94,12 @@ struct wp_viewporter *gst_wl_display_get_viewporter (GstWlDisplay * self); GST_WL_API struct zwp_alpha_compositing_v1 *gst_wl_display_get_alpha_compositing (GstWlDisplay * self); +GST_WL_API +gint gst_wl_display_get_width (GstWlDisplay * self); + +GST_WL_API +gint gst_wl_display_get_height (GstWlDisplay * self); + GST_WL_API struct wl_shm *gst_wl_display_get_shm (GstWlDisplay * self); diff --git a/gst-libs/gst/wayland/gstwlutils.c b/gst-libs/gst/wayland/gstwlutils.c new file mode 100644 index 0000000000..7c556a8a8a --- /dev/null +++ b/gst-libs/gst/wayland/gstwlutils.c @@ -0,0 +1,144 @@ +/* GStreamer Wayland video sink + * + * Copyright 2018 NXP + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gstwlutils.h" + +#define WESTON_INI "/etc/xdg/weston/weston.ini" + +gboolean +gst_wl_init_buffer_scale (gint display_width, gint display_height, + guint * scale) +{ + gchar path[] = WESTON_INI; + gchar line[512], *p, *section = NULL, *size = NULL; + gint fd, n, i; + gint desktop_width, desktop_height; + gboolean found_config = FALSE; + gboolean ret = TRUE; + struct stat filestat; + FILE *fp; + + if ((fd = open (path, O_RDONLY)) == -1) { + return FALSE; + } + + if (fstat (fd, &filestat) < 0 || !S_ISREG (filestat.st_mode)) { + close (fd); + return FALSE; + } + + fp = fdopen (fd, "r"); + if (fp == NULL) { + close (fd); + return FALSE; + } + + while (fgets (line, sizeof line, fp)) { + if (found_config) + break; + + switch (line[0]) { + case '#': + case '\n': + continue; + case '[': + p = strchr (&line[1], ']'); + if (!p || p[1] != '\n') { + continue; + } + p[0] = '\0'; + if (section) + g_free (section); + section = g_strdup (&line[1]); + continue; + default: + if (section && strcmp (section, "shell") == 0) { + p = strchr (line, '='); + if (!p || p == line) { + continue; + } + + p[0] = '\0'; + if (strcmp (&line[0], "size") == 0) { + p++; + while (isspace (*p)) + p++; + i = strlen (p); + while (i > 0 && isspace (p[i - 1])) { + p[i - 1] = '\0'; + i--; + } + if (strlen (p) > 0) { + if (size) + g_free (size); + size = g_strdup (p); + found_config = TRUE; + } + } + } + continue; + } + } + + if (found_config && size) { + n = sscanf (size, "%dx%d\n", &desktop_width, &desktop_height); + if (n != 2) { + ret = FALSE; + goto out; + } + } else { + ret = FALSE; + goto out; + } + + /* FIXME: only support buffer scale 2 and 1 */ + if (display_width > 0 && display_height > 0) { + *scale = display_width / desktop_width; + if (*scale != 1 && *scale != 2) { + *scale = 1; + } + } else { + ret = FALSE; + goto out; + } + +out: + if (section) + g_free (section); + if (size) + g_free (size); + fclose (fp); + return ret; +} diff --git a/gst-libs/gst/wayland/gstwlutils.h b/gst-libs/gst/wayland/gstwlutils.h new file mode 100644 index 0000000000..285dcaeca2 --- /dev/null +++ b/gst-libs/gst/wayland/gstwlutils.h @@ -0,0 +1,31 @@ +/* GStreamer Wayland video sink + * + * Copyright 2018 NXP + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301 USA. + */ + +#ifndef __GST_WL_UTILS_H__ +#define __GST_WL_UTILS_H__ + +#include + +G_BEGIN_DECLS + +gboolean gst_wl_init_buffer_scale(gint display_width, gint display_height, guint * scale); +G_END_DECLS + +#endif \ No newline at end of file diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index 4a0a9c8c13..e522ed845e 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -25,6 +25,7 @@ #endif #include "gstwlwindow.h" +#include "gstwlutils.h" #include "gstimxcommon.h" #include "fullscreen-shell-unstable-v1-client-protocol.h" @@ -88,6 +89,9 @@ typedef struct _GstWlWindowPrivate /* the coordinate of video crop */ gint src_x, src_y, src_width, src_height; + + /* video buffer scale */ + guint scale; } GstWlWindowPrivate; G_DEFINE_TYPE_WITH_CODE (GstWlWindow, gst_wl_window, G_TYPE_OBJECT, @@ -198,6 +202,7 @@ gst_wl_window_init (GstWlWindow * self) priv->src_y = 0; priv->src_width = -1; priv->src_height = 0; + priv->scale = 1; } static void @@ -256,6 +261,7 @@ gst_wl_window_new_internal (GstWlDisplay * display, GMutex * render_lock) struct wl_region *region; struct wp_viewporter *viewporter; struct zwp_alpha_compositing_v1 *alpha_compositing; + gint width, height; self = g_object_new (GST_TYPE_WL_WINDOW, NULL); priv = gst_wl_window_get_instance_private (self); @@ -301,6 +307,12 @@ gst_wl_window_new_internal (GstWlDisplay * display, GMutex * render_lock) wl_surface_set_input_region (priv->video_surface, region); wl_region_destroy (region); + width = gst_wl_display_get_width (display); + height = gst_wl_display_get_height (display); + if (!gst_wl_init_buffer_scale (width, height, &priv->scale)) { + GST_WARNING ("init buffer scale fail, fallback to scale=%d", priv->scale); + } + return self; } @@ -391,7 +403,8 @@ gst_wl_window_new_toplevel (GstWlDisplay * display, const GstVideoInfo * info, /* set the initial size to be the same as the reported video size */ gint width = gst_util_uint64_scale_int_round (info->width, info->par_n, info->par_d); - gst_wl_window_set_render_rectangle (self, 0, 0, width, info->height); + gst_wl_window_set_render_rectangle (self, 0, 0, width / priv->scale, + info->height / priv->scale); } return self; @@ -516,10 +529,11 @@ gst_wl_window_resize_video_surface (GstWlWindow * self, gboolean commit) int wp_src_width; int wp_src_height; - wl_fixed_t src_x = wl_fixed_from_int (priv->src_x); - wl_fixed_t src_y = wl_fixed_from_int (priv->src_y); - wl_fixed_t src_width = wl_fixed_from_int (priv->src_width); - wl_fixed_t src_height = wl_fixed_from_int (priv->src_height); + wl_fixed_t src_x = wl_fixed_from_int (priv->src_x / priv->scale); + wl_fixed_t src_y = wl_fixed_from_int (priv->src_y / priv->scale); + wl_fixed_t src_width = wl_fixed_from_int (priv->src_width / priv->scale); + wl_fixed_t src_height = + wl_fixed_from_int (priv->src_height / priv->scale); switch (priv->buffer_transform) { case WL_OUTPUT_TRANSFORM_NORMAL: @@ -551,7 +565,7 @@ gst_wl_window_resize_video_surface (GstWlWindow * self, gboolean commit) if (priv->video_viewport) { gst_video_center_rect (&src, &dst, &res, TRUE); wp_viewport_set_destination (priv->video_viewport, res.w, res.h); - if (src_width != wl_fixed_from_int (-1)) + if (src_width != wl_fixed_from_int (-1 / priv->scale)) wp_viewport_set_source (priv->video_viewport, src_x, src_y, src_width, src_height); } else { @@ -642,6 +656,7 @@ gst_wl_window_commit_buffer (GstWlWindow * self, GstWlBuffer * buffer) priv->frame_callback = callback; wl_callback_add_listener (callback, &frame_callback_listener, self); gst_wl_buffer_attach (buffer, priv->video_surface_wrapper); + wl_surface_set_buffer_scale (priv->video_surface_wrapper, priv->scale); wl_surface_damage_buffer (priv->video_surface_wrapper, 0, 0, G_MAXINT32, G_MAXINT32); wl_surface_commit (priv->video_surface_wrapper); @@ -655,6 +670,7 @@ gst_wl_window_commit_buffer (GstWlWindow * self, GstWlBuffer * buffer) } else { /* clear both video and parent surfaces */ wl_surface_attach (priv->video_surface_wrapper, NULL, 0, 0); + wl_surface_set_buffer_scale (priv->video_surface_wrapper, priv->scale); wl_surface_commit (priv->video_surface_wrapper); wl_surface_attach (priv->area_surface_wrapper, NULL, 0, 0); wl_surface_commit (priv->area_surface_wrapper); diff --git a/gst-libs/gst/wayland/meson.build b/gst-libs/gst/wayland/meson.build index cf6a0e2d19..0e181de564 100644 --- a/gst-libs/gst/wayland/meson.build +++ b/gst-libs/gst/wayland/meson.build @@ -17,6 +17,7 @@ if use_wayland 'gstwlvideobufferpool.c', 'gstwlvideoformat.c', 'gstwlwindow.c', + 'gstwlutils.c', ]) wl_headers = files([ @@ -29,6 +30,7 @@ if use_wayland 'gstwlvideobufferpool.h', 'gstwlvideoformat.h', 'gstwlwindow.h', + 'gstwlutils.h', 'wayland.h', 'wayland-prelude.h', ]) -- 2.25.1 From 383ffecb8e983e7a75a30d7ff3887b05f73772b2 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 23 Sep 2020 20:30:30 +0800 Subject: [PATCH 30/65] MMFMWK-7954 waylandsink: add property to set window resolution add two property for user to set window resolution when using waylandsink in cmdline upstream status: Pending https://bugzilla.gnome.org/show_bug.cgi?id=796932 (cherry picked from commit b86fce8720bc84dffc5bcaded7fc2d600f47a9b4) --- ext/wayland/gstwaylandsink.c | 30 +++++++++++++++++++++++ ext/wayland/gstwaylandsink.h | 3 +++ gst-libs/gst/wayland/gstwldisplay.c | 38 +++++++++++++++++++++++++++++ gst-libs/gst/wayland/gstwldisplay.h | 12 +++++++++ gst-libs/gst/wayland/gstwlwindow.c | 21 ++++++++++++---- 5 files changed, 99 insertions(+), 5 deletions(-) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index 27fac5e374..737ffff08f 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -59,6 +59,8 @@ enum enum { PROP_0, + PROP_WINDOW_WIDTH, + PROP_WINDOW_HEIGHT, PROP_DISPLAY, PROP_FULLSCREEN, PROP_ALPHA, @@ -152,6 +154,16 @@ gst_wayland_sink_class_init (GstWaylandSinkClass * klass) gstvideosink_class->show_frame = GST_DEBUG_FUNCPTR (gst_wayland_sink_show_frame); + g_object_class_install_property (gobject_class, PROP_WINDOW_WIDTH, + g_param_spec_int ("window-width", "Wayland sink window width", "Wayland " + "sink preferred window width in pixel", + -1, G_MAXINT, -1, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_WINDOW_HEIGHT, + g_param_spec_int ("window-height", "Wayland sink window height", + "Wayland " "sink preferred window height in pixel", -1, G_MAXINT, -1, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_DISPLAY, g_param_spec_string ("display", "Wayland Display name", "Wayland " "display name to connect to, if not supplied via the GstContext", @@ -206,6 +218,8 @@ static void gst_wayland_sink_init (GstWaylandSink * self) { self->alpha = 0.0f; + self->preferred_width = -1; + self->preferred_height = -1; g_mutex_init (&self->display_lock); g_mutex_init (&self->render_lock); self->frame_showed = 0; @@ -277,6 +291,13 @@ gst_wayland_sink_get_property (GObject * object, GST_OBJECT_LOCK (self); g_value_set_boolean (value, self->fullscreen); GST_OBJECT_UNLOCK (self); + break; + case PROP_WINDOW_WIDTH: + g_value_set_int (value, self->preferred_width); + break; + case PROP_WINDOW_HEIGHT: + g_value_set_int (value, self->preferred_height); + break; case PROP_ALPHA: g_value_set_float (value, self->alpha); break; @@ -312,6 +333,13 @@ gst_wayland_sink_set_property (GObject * object, GST_OBJECT_LOCK (self); gst_wayland_sink_set_fullscreen (self, g_value_get_boolean (value)); GST_OBJECT_UNLOCK (self); + break; + case PROP_WINDOW_WIDTH: + self->preferred_width = g_value_get_int (value); + break; + case PROP_WINDOW_HEIGHT: + self->preferred_height = g_value_get_int (value); + break; case PROP_ALPHA: self->alpha = g_value_get_float (value); break; @@ -443,6 +471,8 @@ gst_wayland_sink_change_state (GstElement * element, GstStateChange transition) case GST_STATE_CHANGE_NULL_TO_READY: if (!gst_wayland_sink_find_display (self)) return GST_STATE_CHANGE_FAILURE; + gst_wl_display_set_preferred_width (self->display, self->preferred_width); + gst_wl_display_set_preferred_height (self->display, self->preferred_height); break; default: break; diff --git a/ext/wayland/gstwaylandsink.h b/ext/wayland/gstwaylandsink.h index 47a69410e2..5666e13d8f 100644 --- a/ext/wayland/gstwaylandsink.h +++ b/ext/wayland/gstwaylandsink.h @@ -61,6 +61,9 @@ struct _GstWaylandSink gboolean fullscreen; GstCaps *caps; + /* preferred window resolution */ + gint preferred_width, preferred_height; + gchar *display_name; GMutex render_lock; diff --git a/gst-libs/gst/wayland/gstwldisplay.c b/gst-libs/gst/wayland/gstwldisplay.c index 3a453ac35f..9b1e990eac 100644 --- a/gst-libs/gst/wayland/gstwldisplay.c +++ b/gst-libs/gst/wayland/gstwldisplay.c @@ -63,6 +63,9 @@ typedef struct _GstWlDisplayPrivate /* real display resolution */ gint width, height; + /* preferred window resolution */ + gint preferred_width, preferred_height; + /* private */ gboolean own_display; GThread *thread; @@ -102,6 +105,8 @@ gst_wl_display_init (GstWlDisplay * self) priv->buffers = g_hash_table_new (g_direct_hash, g_direct_equal); priv->width = -1; priv->height = -1; + priv->preferred_width = -1; + priv->preferred_height = -1; g_mutex_init (&priv->buffers_mutex); g_rec_mutex_init (&priv->sync_mutex); @@ -723,6 +728,39 @@ gst_wl_display_get_height (GstWlDisplay * self) return priv->height; } + +void +gst_wl_display_set_preferred_width (GstWlDisplay * self, gint preferred_width) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + priv->preferred_width = preferred_width; +} + +void +gst_wl_display_set_preferred_height (GstWlDisplay * self, gint preferred_height) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + priv->preferred_height = preferred_height; +} + +gint +gst_wl_display_get_preferred_width (GstWlDisplay * self) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + return priv->preferred_width; +} + +gint +gst_wl_display_get_preferred_height (GstWlDisplay * self) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + return priv->preferred_height; +} + struct wl_shm * gst_wl_display_get_shm (GstWlDisplay * self) { diff --git a/gst-libs/gst/wayland/gstwldisplay.h b/gst-libs/gst/wayland/gstwldisplay.h index 6c5eda7e25..2d34567fab 100644 --- a/gst-libs/gst/wayland/gstwldisplay.h +++ b/gst-libs/gst/wayland/gstwldisplay.h @@ -100,6 +100,18 @@ gint gst_wl_display_get_width (GstWlDisplay * self); GST_WL_API gint gst_wl_display_get_height (GstWlDisplay * self); +GST_WL_API +void gst_wl_display_set_preferred_width (GstWlDisplay * self, gint preferred_width); + +GST_WL_API +void gst_wl_display_set_preferred_height (GstWlDisplay * self, gint preferred_height); + +GST_WL_API +gint gst_wl_display_get_preferred_width (GstWlDisplay * self); + +GST_WL_API +gint gst_wl_display_get_preferred_height (GstWlDisplay * self); + GST_WL_API struct wl_shm *gst_wl_display_get_shm (GstWlDisplay * self); diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index e522ed845e..8ee8dcf2d7 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -309,6 +309,7 @@ gst_wl_window_new_internal (GstWlDisplay * display, GMutex * render_lock) width = gst_wl_display_get_width (display); height = gst_wl_display_get_height (display); + if (!gst_wl_init_buffer_scale (width, height, &priv->scale)) { GST_WARNING ("init buffer scale fail, fallback to scale=%d", priv->scale); } @@ -400,11 +401,21 @@ gst_wl_window_new_toplevel (GstWlDisplay * display, const GstVideoInfo * info, /* render_rectangle is already set via toplevel_configure in * xdg_shell fullscreen mode */ if (!(xdg_wm_base && fullscreen)) { - /* set the initial size to be the same as the reported video size */ - gint width = - gst_util_uint64_scale_int_round (info->width, info->par_n, info->par_d); - gst_wl_window_set_render_rectangle (self, 0, 0, width / priv->scale, - info->height / priv->scale); + gint width, height; + gint preferred_width = gst_wl_display_get_preferred_width (display); + gint preferred_height = gst_wl_display_get_preferred_height (display); + if (preferred_width > 0 && preferred_height > 0) { + width = preferred_width; + height = preferred_height; + } else { + /* set the initial size to be the same as the reported video size */ + width = + gst_util_uint64_scale_int_round (info->width, info->par_n, + info->par_d) / priv->scale; + height = info->height / priv->scale; + } + + gst_wl_window_set_render_rectangle (self, 0, 0, width, height); } return self; -- 2.25.1 From 423df1b54f32030911098e407242646682734e34 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Thu, 16 Apr 2020 12:04:09 +0800 Subject: [PATCH 31/65] MMFMWK-8759 waylandsink: add listener for wl_point and wl_touch Add listener for wl_point and wl_touch, these handler can make surface respond to mouse and touch event. upstream status:pending https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/1195 (cherry picked from commit 7714b8f0f148acf94f34317e1cadebeae845205c) --- gst-libs/gst/wayland/gstwldisplay.c | 55 ++++++++++++ gst-libs/gst/wayland/gstwldisplay.h | 9 ++ gst-libs/gst/wayland/gstwlwindow.c | 129 +++++++++++++++++++++++++++- 3 files changed, 192 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/wayland/gstwldisplay.c b/gst-libs/gst/wayland/gstwldisplay.c index 9b1e990eac..ceecc37e4a 100644 --- a/gst-libs/gst/wayland/gstwldisplay.c +++ b/gst-libs/gst/wayland/gstwldisplay.c @@ -49,6 +49,9 @@ typedef struct _GstWlDisplayPrivate struct wl_compositor *compositor; struct wl_subcompositor *subcompositor; struct xdg_wm_base *xdg_wm_base; + struct wl_seat *seat; + struct wl_pointer *pointer; + struct wl_touch *touch; struct zwp_fullscreen_shell_v1 *fullscreen_shell; struct wp_single_pixel_buffer_manager_v1 *single_pixel_buffer; struct wl_shm *shm; @@ -307,6 +310,32 @@ gst_wl_display_check_format_for_dmabuf (GstWlDisplay * self, return FALSE; } +static void +seat_handle_capabilities (void *data, struct wl_seat *seat, + enum wl_seat_capability caps) +{ + GstWlDisplay *self = data; + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + if ((caps & WL_SEAT_CAPABILITY_POINTER) && !priv->pointer) { + priv->pointer = wl_seat_get_pointer (seat); + } else if (!(caps & WL_SEAT_CAPABILITY_POINTER) && priv->pointer) { + wl_pointer_destroy (priv->pointer); + priv->pointer = NULL; + } + + if ((caps & WL_SEAT_CAPABILITY_TOUCH) && !priv->touch) { + priv->touch = wl_seat_get_touch (seat); + } else if (!(caps & WL_SEAT_CAPABILITY_TOUCH) && priv->touch) { + wl_touch_destroy (priv->touch); + priv->touch = NULL; + } +} + +static const struct wl_seat_listener seat_listener = { + seat_handle_capabilities, +}; + static void handle_xdg_wm_base_ping (void *user_data, struct xdg_wm_base *xdg_wm_base, uint32_t serial) @@ -379,6 +408,9 @@ registry_handle_global (void *data, struct wl_registry *registry, priv->xdg_wm_base = wl_registry_bind (registry, id, &xdg_wm_base_interface, 1); xdg_wm_base_add_listener (priv->xdg_wm_base, &xdg_wm_base_listener, self); + } else if (strcmp (interface, "wl_seat") == 0) { + priv->seat = wl_registry_bind (registry, id, &wl_seat_interface, 1); + wl_seat_add_listener (priv->seat, &seat_listener, self); } else if (g_strcmp0 (interface, "zwp_fullscreen_shell_v1") == 0) { priv->fullscreen_shell = wl_registry_bind (registry, id, &zwp_fullscreen_shell_v1_interface, 1); @@ -689,6 +721,13 @@ gst_wl_display_get_xdg_wm_base (GstWlDisplay * self) return priv->xdg_wm_base; } +struct wl_seat *gst_wl_display_get_seat (GstWlDisplay * self) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + return priv->seat; +} + struct zwp_fullscreen_shell_v1 * gst_wl_display_get_fullscreen_shell_v1 (GstWlDisplay * self) { @@ -713,6 +752,22 @@ gst_wl_display_get_alpha_compositing (GstWlDisplay * self) return priv->alpha_compositing; } +struct wl_pointer * +gst_wl_display_get_pointer (GstWlDisplay * self) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + return priv->pointer; +} + +struct wl_touch * +gst_wl_display_get_touch (GstWlDisplay * self) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + return priv->touch; +} + gint gst_wl_display_get_width (GstWlDisplay * self) { diff --git a/gst-libs/gst/wayland/gstwldisplay.h b/gst-libs/gst/wayland/gstwldisplay.h index 2d34567fab..9eae9f88de 100644 --- a/gst-libs/gst/wayland/gstwldisplay.h +++ b/gst-libs/gst/wayland/gstwldisplay.h @@ -85,6 +85,9 @@ struct wl_subcompositor *gst_wl_display_get_subcompositor (GstWlDisplay * self); GST_WL_API struct xdg_wm_base *gst_wl_display_get_xdg_wm_base (GstWlDisplay * self); +GST_WL_API +struct wl_seat *gst_wl_display_get_seat (GstWlDisplay * self); + GST_WL_API struct zwp_fullscreen_shell_v1 *gst_wl_display_get_fullscreen_shell_v1 (GstWlDisplay * self); @@ -94,6 +97,12 @@ struct wp_viewporter *gst_wl_display_get_viewporter (GstWlDisplay * self); GST_WL_API struct zwp_alpha_compositing_v1 *gst_wl_display_get_alpha_compositing (GstWlDisplay * self); +GST_WL_API +struct wl_pointer *gst_wl_display_get_pointer (GstWlDisplay * self); + +GST_WL_API +struct wl_touch *gst_wl_display_get_touch (GstWlDisplay * self); + GST_WL_API gint gst_wl_display_get_width (GstWlDisplay * self); diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index 8ee8dcf2d7..6b9f8e14c6 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -24,6 +24,8 @@ #include #endif +#include + #include "gstwlwindow.h" #include "gstwlutils.h" #include "gstimxcommon.h" @@ -92,6 +94,9 @@ typedef struct _GstWlWindowPrivate /* video buffer scale */ guint scale; + + /* mouse location when click */ + gint pointer_x, pointer_y; } GstWlWindowPrivate; G_DEFINE_TYPE_WITH_CODE (GstWlWindow, gst_wl_window, G_TYPE_OBJECT, @@ -100,6 +105,9 @@ G_DEFINE_TYPE_WITH_CODE (GstWlWindow, gst_wl_window, G_TYPE_OBJECT, "wlwindow", 0, "wlwindow library"); ); +/* resize trigger margin in pixel */ +#define RESIZE_MARGIN 20 + enum { CLOSED, @@ -116,6 +124,110 @@ static void gst_wl_window_update_borders (GstWlWindow * self); static void gst_wl_window_commit_buffer (GstWlWindow * self, GstWlBuffer * buffer); +static void +pointer_handle_enter (void *data, struct wl_pointer *pointer, + uint32_t serial, struct wl_surface *surface, wl_fixed_t sx, wl_fixed_t sy) +{ + GstWlWindow *self = data; + GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); + + priv->pointer_x = wl_fixed_to_int (sx); + priv->pointer_y = wl_fixed_to_int (sy); +} + +static void +pointer_handle_leave (void *data, struct wl_pointer *pointer, + uint32_t serial, struct wl_surface *surface) +{ +} + +static void +pointer_handle_motion (void *data, struct wl_pointer *pointer, + uint32_t time, wl_fixed_t sx, wl_fixed_t sy) +{ +} + +static void +pointer_handle_button (void *data, struct wl_pointer *wl_pointer, + uint32_t serial, uint32_t time, uint32_t button, uint32_t state) +{ + GstWlWindow *self = data; + GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); + + if (!priv->xdg_toplevel) + return; + + if (button == BTN_LEFT && state == WL_POINTER_BUTTON_STATE_PRESSED) { + struct wl_seat *seat = gst_wl_display_get_seat (priv->display); + if (priv->render_rectangle.w - priv->pointer_x <= RESIZE_MARGIN + && priv->render_rectangle.h - priv->pointer_y <= RESIZE_MARGIN) + xdg_toplevel_resize (priv->xdg_toplevel, seat, serial, + XDG_TOPLEVEL_RESIZE_EDGE_BOTTOM_RIGHT); + else + xdg_toplevel_move (priv->xdg_toplevel, seat, serial); + } +} + +static void +pointer_handle_axis (void *data, struct wl_pointer *wl_pointer, + uint32_t time, uint32_t axis, wl_fixed_t value) +{ +} + +static const struct wl_pointer_listener pointer_listener = { + pointer_handle_enter, + pointer_handle_leave, + pointer_handle_motion, + pointer_handle_button, + pointer_handle_axis, +}; + +static void +touch_handle_down (void *data, struct wl_touch *wl_touch, + uint32_t serial, uint32_t time, struct wl_surface *surface, + int32_t id, wl_fixed_t x_w, wl_fixed_t y_w) +{ + GstWlWindow *self = data; + GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); + struct wl_seat *seat; + + if (!priv->xdg_toplevel) + return; + + seat = gst_wl_display_get_seat (priv->display); + xdg_toplevel_move (priv->xdg_toplevel, seat, serial); +} + +static void +touch_handle_up (void *data, struct wl_touch *wl_touch, + uint32_t serial, uint32_t time, int32_t id) +{ +} + +static void +touch_handle_motion (void *data, struct wl_touch *wl_touch, + uint32_t time, int32_t id, wl_fixed_t x_w, wl_fixed_t y_w) +{ +} + +static void +touch_handle_frame (void *data, struct wl_touch *wl_touch) +{ +} + +static void +touch_handle_cancel (void *data, struct wl_touch *wl_touch) +{ +} + +static const struct wl_touch_listener touch_listener = { + touch_handle_down, + touch_handle_up, + touch_handle_motion, + touch_handle_frame, + touch_handle_cancel, +}; + static void handle_xdg_toplevel_close (void *data, struct xdg_toplevel *xdg_toplevel) { @@ -130,6 +242,7 @@ handle_xdg_toplevel_configure (void *data, struct xdg_toplevel *xdg_toplevel, int32_t width, int32_t height, struct wl_array *states) { GstWlWindow *self = data; + GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); const uint32_t *state; GST_DEBUG ("XDG toplevel got a \"configure\" event, [ %d, %d ].", @@ -145,10 +258,12 @@ handle_xdg_toplevel_configure (void *data, struct xdg_toplevel *xdg_toplevel, } } - if (width <= 0 || height <= 0) + if (width <= 2 * RESIZE_MARGIN || height <= 2 * RESIZE_MARGIN) return; + g_mutex_lock (priv->render_lock); gst_wl_window_set_render_rectangle (self, 0, 0, width, height); + g_mutex_unlock (priv->render_lock); } static const struct xdg_toplevel_listener xdg_toplevel_listener = { @@ -349,6 +464,8 @@ gst_wl_window_new_toplevel (GstWlDisplay * display, const GstVideoInfo * info, /* Check which protocol we will use (in order of preference) */ if (xdg_wm_base) { gint64 timeout; + struct wl_pointer *pointer; + struct wl_touch *touch; /* First create the XDG surface */ priv->xdg_surface = xdg_wm_base_get_xdg_surface (xdg_wm_base, @@ -373,6 +490,16 @@ gst_wl_window_new_toplevel (GstWlDisplay * display, const GstVideoInfo * info, xdg_toplevel_set_app_id (priv->xdg_toplevel, "org.gstreamer.wayland"); } + pointer = gst_wl_display_get_pointer (display); + touch = gst_wl_display_get_touch (display); + if (pointer) + wl_pointer_add_listener (pointer, &pointer_listener, self); + + if (touch) { + wl_touch_set_user_data (touch, self); + wl_touch_add_listener (touch, &touch_listener, self); + } + gst_wl_window_ensure_fullscreen (self, fullscreen); /* Finally, commit the xdg_surface state as toplevel */ -- 2.25.1 From b0eeaade5f2790cea6643cb2cd59580e865c298f Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Mon, 20 Apr 2020 16:02:55 +0800 Subject: [PATCH 32/65] MMFMWK-8759 waylandsink: chose the first display size when enable multi display, need to chose the first display size as the default full screen size. (cherry picked from commit 34c6f9891066d39748e56b777747358cd64c3db6) --- gst-libs/gst/wayland/gstwldisplay.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/wayland/gstwldisplay.c b/gst-libs/gst/wayland/gstwldisplay.c index ceecc37e4a..447119c08e 100644 --- a/gst-libs/gst/wayland/gstwldisplay.c +++ b/gst-libs/gst/wayland/gstwldisplay.c @@ -347,6 +347,7 @@ static const struct xdg_wm_base_listener xdg_wm_base_listener = { handle_xdg_wm_base_ping }; +static void output_handle_geometry (void *data, struct wl_output *wl_output, int32_t x, int32_t y, int32_t physical_width, int32_t physical_height, @@ -365,8 +366,11 @@ output_handle_mode (void *data, struct wl_output *wl_output, /* we only care about the current mode */ if (flags & WL_OUTPUT_MODE_CURRENT) { - priv->width = width; - priv->height = height; + if (priv->width == -1 && priv->height == -1) { + priv->width = width; + priv->height = height; + } + } } -- 2.25.1 From 1e73f8a57ad17ce1a3723781427a7817e1b4812e Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Mon, 12 Oct 2020 19:00:23 +0800 Subject: [PATCH 33/65] waylandsink: support HDR10 video playback transfor to use HDR metadata in caps upstream status: imx specific --- ext/wayland/gstwaylandsink.c | 100 ++++++++++++++++++++++++ gst-libs/gst/wayland/gstwldisplay.c | 16 ++++ gst-libs/gst/wayland/gstwldisplay.h | 3 + gst-libs/gst/wayland/gstwlvideoformat.c | 7 ++ gst-libs/gst/wayland/meson.build | 1 + 5 files changed, 127 insertions(+) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index 737ffff08f..be485dbb7d 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -47,6 +47,9 @@ #include #include #include +#include + +#include "hdr10-metadata-unstable-v1-client-protocol.h" /* signals */ enum @@ -98,6 +101,8 @@ static gboolean gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query); static GstFlowReturn gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer); +static void +gst_wayland_sink_config_hdr10 (GstWaylandSink * self, const GstCaps * caps); /* VideoOverlay interface */ static void gst_wayland_sink_videooverlay_init (GstVideoOverlayInterface * @@ -488,6 +493,7 @@ gst_wayland_sink_change_state (GstElement * element, GstStateChange transition) break; case GST_STATE_CHANGE_PAUSED_TO_READY: gst_buffer_replace (&self->last_buffer, NULL); + gst_wayland_sink_config_hdr10 (self, NULL); if (self->window) { area_surface = gst_wl_window_get_area_surface (self->window); render_rectangle_w = gst_wl_window_get_rectangle_w (self->window); @@ -626,6 +632,16 @@ gst_wayland_sink_get_caps (GstBaseSink * bsink, GstCaps * filter) } } + /** FIXME:work around for 10bit format not in the none capsfeature list + * need vpu add memory:DMABuf capsfeature when output dmabuf + */ + if (HAS_DCSS ()) { + g_value_init (&value, G_TYPE_STRING); + g_value_set_static_string (&value, + gst_video_format_to_string (GST_VIDEO_FORMAT_NV12_10LE40)); + gst_value_list_append_and_take_value (&shm_list, &value); + } + gst_structure_take_value (gst_caps_get_structure (caps, 0), "format", &shm_list); @@ -799,6 +815,8 @@ gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) /* Will be used to create buffer pools */ gst_caps_replace (&self->caps, caps); + gst_wayland_sink_config_hdr10 (self, caps); + return TRUE; invalid_format: @@ -898,6 +916,88 @@ on_window_closed (GstWlWindow * window, gpointer user_data) ("Output window was closed"), (NULL)); } +static void +gst_wayland_sink_config_hdr10 (GstWaylandSink * self, const GstCaps * caps) +{ + GstWlDisplay *display = self->display; + struct zwp_hdr10_metadata_v1 *hdr10_metadata; + struct wl_display *display_display; + guint32 eotf = 0; + guint32 type = 0; + guint32 display_primaries_red = 0; + guint32 display_primaries_green = 0; + guint32 display_primaries_blue = 0; + guint32 white_point = 0; + guint32 mastering_display_luminance = 0; + guint32 max_cll = 0; + guint32 max_fall = 0; + + /* buf could be NULL when resize */ + + if (caps) { + GstVideoMasteringDisplayInfo minfo; + GstVideoContentLightLevel cll; + if (!gst_video_mastering_display_info_from_caps (&minfo, caps) + || !gst_video_content_light_level_from_caps (&cll, caps)) { + GST_INFO_OBJECT (self, "no HDR metadata present in caps"); + return; + } + + GST_INFO_OBJECT (self, "redPrimary x=%d y=%d", minfo.display_primaries[0].x, + minfo.display_primaries[0].y); + GST_INFO_OBJECT (self, "greenPrimary x=%d y=%d", + minfo.display_primaries[1].x, minfo.display_primaries[1].y); + GST_INFO_OBJECT (self, "bluePrimary x=%d y=%d", + minfo.display_primaries[2].x, minfo.display_primaries[2].y); + GST_INFO_OBJECT (self, "whitePoint x=%d y=%d", minfo.white_point.x, + minfo.white_point.y); + GST_INFO_OBJECT (self, "maxMasteringLuminance %d", + minfo.max_display_mastering_luminance); + GST_INFO_OBJECT (self, "minMasteringLuminance %d", + minfo.min_display_mastering_luminance); + GST_INFO_OBJECT (self, "maxContentLightLevel %d", + cll.max_content_light_level); + GST_INFO_OBJECT (self, "maxFrameAverageLightLevel %d", + cll.max_frame_average_light_level); + + eotf = SMPTE_ST2084; +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + type = 0; +#else + type = 1; +#endif + display_primaries_red = + (guint) (minfo.display_primaries[0].x << 16 | minfo. + display_primaries[0].y); + display_primaries_green = + (guint) (minfo.display_primaries[1].x << 16 | minfo. + display_primaries[1].y); + display_primaries_blue = + (guint) (minfo.display_primaries[2].x << 16 | minfo. + display_primaries[2].y); + white_point = (guint) (minfo.white_point.x << 16 | minfo.white_point.y); + mastering_display_luminance = + (guint) (((minfo.max_display_mastering_luminance / + 10000) & 0xffff) << 16 | (minfo.min_display_mastering_luminance + & 0xffff)); + max_cll = cll.max_content_light_level; + max_fall = cll.max_frame_average_light_level; + } + + hdr10_metadata = gst_wl_display_get_hdr10_metadata (display); + display_display = gst_wl_display_get_display (display); + if (hdr10_metadata) { + zwp_hdr10_metadata_v1_set_metadata (hdr10_metadata, + eotf, + type, + display_primaries_red, + display_primaries_green, + display_primaries_blue, + white_point, mastering_display_luminance, max_cll, max_fall); + wl_display_roundtrip (display_display); + } +} + static GstFlowReturn gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) { diff --git a/gst-libs/gst/wayland/gstwldisplay.c b/gst-libs/gst/wayland/gstwldisplay.c index 447119c08e..62dec69f00 100644 --- a/gst-libs/gst/wayland/gstwldisplay.c +++ b/gst-libs/gst/wayland/gstwldisplay.c @@ -30,6 +30,7 @@ #include "viewporter-client-protocol.h" #include "xdg-shell-client-protocol.h" #include "alpha-compositing-unstable-v1-client-protocol.h" +#include "hdr10-metadata-unstable-v1-client-protocol.h" #include #include @@ -59,6 +60,7 @@ typedef struct _GstWlDisplayPrivate struct wp_viewporter *viewporter; struct zwp_linux_dmabuf_v1 *dmabuf; struct zwp_alpha_compositing_v1 *alpha_compositing; + struct zwp_hdr10_metadata_v1 *hdr10_metadata; GArray *shm_formats; GArray *dmabuf_formats; GArray *dmabuf_modifiers; @@ -180,6 +182,9 @@ gst_wl_display_finalize (GObject * gobject) if (priv->subcompositor) wl_subcompositor_destroy (priv->subcompositor); + if (priv->hdr10_metadata) + zwp_hdr10_metadata_v1_destroy (priv->hdr10_metadata); + if (priv->registry) wl_registry_destroy (priv->registry); @@ -435,6 +440,9 @@ registry_handle_global (void *data, struct wl_registry *registry, } else if (g_strcmp0 (interface, "zwp_alpha_compositing_v1") == 0) { priv->alpha_compositing = wl_registry_bind (registry, id, &zwp_alpha_compositing_v1_interface, 1); + } else if (g_strcmp0 (interface, "zwp_hdr10_metadata_v1") == 0) { + priv->hdr10_metadata = + wl_registry_bind (registry, id, &zwp_hdr10_metadata_v1_interface, 1); } else if (g_strcmp0 (interface, "wl_output") == 0) { priv->output = wl_registry_bind (registry, id, &wl_output_interface, MIN (version, 2)); @@ -772,6 +780,14 @@ gst_wl_display_get_touch (GstWlDisplay * self) return priv->touch; } +struct zwp_hdr10_metadata_v1 * +gst_wl_display_get_hdr10_metadata (GstWlDisplay * self) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + return priv->hdr10_metadata; +} + gint gst_wl_display_get_width (GstWlDisplay * self) { diff --git a/gst-libs/gst/wayland/gstwldisplay.h b/gst-libs/gst/wayland/gstwldisplay.h index 9eae9f88de..6c16405ee6 100644 --- a/gst-libs/gst/wayland/gstwldisplay.h +++ b/gst-libs/gst/wayland/gstwldisplay.h @@ -103,6 +103,9 @@ struct wl_pointer *gst_wl_display_get_pointer (GstWlDisplay * self); GST_WL_API struct wl_touch *gst_wl_display_get_touch (GstWlDisplay * self); +GST_WL_API +struct zwp_hdr10_metadata_v1 *gst_wl_display_get_hdr10_metadata (GstWlDisplay * self); + GST_WL_API gint gst_wl_display_get_width (GstWlDisplay * self); diff --git a/gst-libs/gst/wayland/gstwlvideoformat.c b/gst-libs/gst/wayland/gstwlvideoformat.c index 79b2befbda..aeea4fd071 100644 --- a/gst-libs/gst/wayland/gstwlvideoformat.c +++ b/gst-libs/gst/wayland/gstwlvideoformat.c @@ -28,6 +28,7 @@ #include "gstwlvideoformat.h" #include +#include /* This can be removed once we can bump the required wl_client_dep, * which again is blocked by a CI image update, see @@ -90,6 +91,12 @@ static const wl_VideoFormat wl_formats[] = { {WL_SHM_FORMAT_YVU420, DRM_FORMAT_YVU420, GST_VIDEO_FORMAT_YV12}, {WL_SHM_FORMAT_YUV422, DRM_FORMAT_YUV422, GST_VIDEO_FORMAT_Y42B}, {WL_SHM_FORMAT_YUV444, DRM_FORMAT_YUV444, GST_VIDEO_FORMAT_v308}, + /*FIXME. no suitable format for shm when it is NV12 10bit */ +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0) + {WL_SHM_FORMAT_NV12, DRM_FORMAT_P010, GST_VIDEO_FORMAT_NV12_10LE40}, +#else + {WL_SHM_FORMAT_NV12, DRM_FORMAT_NV15, GST_VIDEO_FORMAT_NV12_10LE40}, +#endif }; enum wl_shm_format diff --git a/gst-libs/gst/wayland/meson.build b/gst-libs/gst/wayland/meson.build index 0e181de564..912d26e4e3 100644 --- a/gst-libs/gst/wayland/meson.build +++ b/gst-libs/gst/wayland/meson.build @@ -44,6 +44,7 @@ if use_wayland ['xdg-shell', 'stable', ], ['single-pixel-buffer', 'staging', 'v1' ], ['alpha-compositing', 'unstable', 'v1', ], + ['hdr10-metadata', 'unstable', 'v1', ], ] protocols_files = [] -- 2.25.1 From d9944c40e70e4efb6a1c33c6a488c16038070363 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 28 Oct 2020 19:10:26 +0800 Subject: [PATCH 34/65] waylandsink: fix critical warning log of unref current_gstbuffer After change to cache GstMemory, if the GstMemory key moves from one GstBuffer to another when this wlbuffer still used by compositor, need to use gst_buffer_replace to change this pointer. Because we didn't increase the new gstbuffer's refcount in attach(), and we will lost the previous gstbuffer. (cherry picked from commit 42fe40332f8bfff83587243119250aa21674fad8) --- gst-libs/gst/wayland/gstwlbuffer.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/wayland/gstwlbuffer.c b/gst-libs/gst/wayland/gstwlbuffer.c index 988c7b628e..684fae9c8f 100644 --- a/gst-libs/gst/wayland/gstwlbuffer.c +++ b/gst-libs/gst/wayland/gstwlbuffer.c @@ -223,7 +223,10 @@ gst_buffer_get_wl_buffer (GstWlDisplay * display, GstBuffer * gstbuffer) if (wlbuf) { GstWlBufferPrivate *priv = gst_wl_buffer_get_instance_private (wlbuf); - priv->current_gstbuffer = gstbuffer; + if (priv->used_by_compositor) + gst_buffer_replace (&priv->current_gstbuffer, gstbuffer); + else + priv->current_gstbuffer = gstbuffer; } return wlbuf; -- 2.25.1 From b812d871fe05d447d43563fd61a5633118c27bba Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Fri, 27 Apr 2018 18:01:06 +0800 Subject: [PATCH 35/65] MMFMWK-7954 waylandsink: set to default video sink on imx8 and mx93 platform upstream status: imx specific --- ext/wayland/gstwaylandsink.c | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index be485dbb7d..9ec5fca88c 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -44,6 +44,8 @@ #include "gstwaylandsink.h" +#include "gstimxcommon.h" + #include #include #include @@ -117,8 +119,6 @@ static void gst_wayland_sink_expose (GstVideoOverlay * overlay); G_DEFINE_TYPE_WITH_CODE (GstWaylandSink, gst_wayland_sink, GST_TYPE_VIDEO_SINK, G_IMPLEMENT_INTERFACE (GST_TYPE_VIDEO_OVERLAY, gst_wayland_sink_videooverlay_init)); -GST_ELEMENT_REGISTER_DEFINE (waylandsink, "waylandsink", GST_RANK_MARGINAL, - GST_TYPE_WAYLAND_SINK); static void gst_wayland_sink_class_init (GstWaylandSinkClass * klass) @@ -1349,9 +1349,23 @@ gst_wayland_sink_expose (GstVideoOverlay * overlay) static gboolean plugin_init (GstPlugin * plugin) { + GstRank rank = GST_RANK_MARGINAL; + GST_DEBUG_CATEGORY_INIT (gstwayland_debug, "waylandsink", 0, " wayland video sink"); + if (HAS_DPU ()) { + if (HAS_VPU ()) + rank = IMX_GST_PLUGIN_RANK + 1; + } else if (IS_IMX8MM () || IS_IMX8MN () || IS_IMX8MP () || IS_IMX8ULP () || IS_IMX93()) { + rank = IMX_GST_PLUGIN_RANK + 1; + } else if (HAS_DCSS ()) { + rank = IMX_GST_PLUGIN_RANK; + } + + GST_ELEMENT_REGISTER_DEFINE (waylandsink, "waylandsink", rank, + GST_TYPE_WAYLAND_SINK); + return GST_ELEMENT_REGISTER (waylandsink, plugin); } -- 2.25.1 From 1f92edad57e77d0e9b9994a14cce69f8df0500ce Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 10 Apr 2019 10:41:04 +0800 Subject: [PATCH 36/65] wlwindow: fix 8qxp and 8qm video playback performance issue Need set video surface opaque to disable below GUI refresh Need check if 8mq still need this change upstream status: imx specific Signed-off-by: Haihua Hu (cherry picked from commit 68840b838c25aa1ab771b4ad884a086f23e3804f) --- gst-libs/gst/wayland/gstwlwindow.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index 6b9f8e14c6..7defa01f97 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -731,8 +731,9 @@ gst_wl_window_set_opaque (GstWlWindow * self, const GstVideoInfo * info) compositor = gst_wl_display_get_compositor (priv->display); if (!GST_VIDEO_INFO_HAS_ALPHA (info)) { - /* for platform support overlay, video should not overlap graphic */ - if (HAS_DCSS () || HAS_DPU ()) + /* for platform support overlay, video should not overlap graphic + FIXME. Not sure whether still need this change */ + if (HAS_DCSS ()) return; /* Set video opaque */ -- 2.25.1 From 25cf2f5f888f2898d992ebd22ed7b9b87ebddaca Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Mon, 29 Aug 2022 14:32:18 +0800 Subject: [PATCH 37/65] LF-6321 wldisplay: don't break main loop when poll wait was interrupted sometimes when do system suspend resume test, poll wait will return with errno == EINTR, which means poll wait was interrupted by system suspend, need continue main loop instead of break the loop and report communicating error with wayland server because we can resume connection after system resume back. upstream status: pending https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/2949 --- gst-libs/gst/wayland/gstwldisplay.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/wayland/gstwldisplay.c b/gst-libs/gst/wayland/gstwldisplay.c index 62dec69f00..0876436c3d 100644 --- a/gst-libs/gst/wayland/gstwldisplay.c +++ b/gst-libs/gst/wayland/gstwldisplay.c @@ -468,6 +468,8 @@ gst_wl_display_thread_run (gpointer data) GstWlDisplay *self = data; GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); GstPollFD pollfd = GST_POLL_FD_INIT; + gint err; + gboolean normal = FALSE; pollfd.fd = wl_display_get_fd (priv->display); gst_poll_add_fd (priv->wl_fd_poll, &pollfd); @@ -482,10 +484,13 @@ gst_wl_display_thread_run (gpointer data) wl_display_flush (priv->display); if (gst_poll_wait (priv->wl_fd_poll, GST_CLOCK_TIME_NONE) < 0) { - gboolean normal = (errno == EBUSY); + err = errno; + normal = (err == EBUSY); wl_display_cancel_read (priv->display); if (normal) break; + else if (err == EINTR) + continue; else goto error; } -- 2.25.1 From 1a78a28a2d09c2bfd731c1b63ba39c0c2e1ad380 Mon Sep 17 00:00:00 2001 From: Chao Guo Date: Wed, 6 Dec 2023 20:59:32 +0900 Subject: [PATCH 38/65] waylandsink: Set source width and height based on rotation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the rotation angle is 90 or 270°, the source width and height values need to be exchanged, otherwise the weston source validity check will fail. upstream status: imx specific Signed-off-by: Chao Guo --- gst-libs/gst/wayland/gstwlwindow.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index 7defa01f97..b2815498cc 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -664,14 +664,10 @@ gst_wl_window_resize_video_surface (GstWlWindow * self, gboolean commit) GstVideoRectangle src = { 0, }; GstVideoRectangle dst = { 0, }; GstVideoRectangle res; - int wp_src_width; - int wp_src_height; - - wl_fixed_t src_x = wl_fixed_from_int (priv->src_x / priv->scale); - wl_fixed_t src_y = wl_fixed_from_int (priv->src_y / priv->scale); - wl_fixed_t src_width = wl_fixed_from_int (priv->src_width / priv->scale); - wl_fixed_t src_height = - wl_fixed_from_int (priv->src_height / priv->scale); + int wp_src_width = -1; + int wp_src_height = -1; + int wp_src_x = priv->src_x; + int wp_src_y = priv->src_y; switch (priv->buffer_transform) { case WL_OUTPUT_TRANSFORM_NORMAL: @@ -680,8 +676,8 @@ gst_wl_window_resize_video_surface (GstWlWindow * self, gboolean commit) case WL_OUTPUT_TRANSFORM_FLIPPED_180: src.w = priv->scaled_width; src.h = priv->video_height; - wp_src_width = priv->video_width; - wp_src_height = priv->video_height; + wp_src_width = priv->src_width; + wp_src_height = priv->src_height; break; case WL_OUTPUT_TRANSFORM_90: case WL_OUTPUT_TRANSFORM_270: @@ -689,8 +685,8 @@ gst_wl_window_resize_video_surface (GstWlWindow * self, gboolean commit) case WL_OUTPUT_TRANSFORM_FLIPPED_270: src.w = priv->video_height; src.h = priv->scaled_width; - wp_src_width = priv->video_height; - wp_src_height = priv->video_width; + wp_src_width = priv->src_height; + wp_src_height = priv->src_width; break; default: g_assert_not_reached (); @@ -703,9 +699,12 @@ gst_wl_window_resize_video_surface (GstWlWindow * self, gboolean commit) if (priv->video_viewport) { gst_video_center_rect (&src, &dst, &res, TRUE); wp_viewport_set_destination (priv->video_viewport, res.w, res.h); - if (src_width != wl_fixed_from_int (-1 / priv->scale)) + if (wp_src_width != -1 && wp_src_height != -1) wp_viewport_set_source (priv->video_viewport, - src_x, src_y, src_width, src_height); + wl_fixed_from_int (wp_src_x / priv->scale), + wl_fixed_from_int (wp_src_y / priv->scale), + wl_fixed_from_int (wp_src_width / priv->scale), + wl_fixed_from_int (wp_src_height / priv->scale)); } else { gst_video_center_rect (&src, &dst, &res, FALSE); } -- 2.25.1 From 8154360310488227e8add3831263e88c335cef77 Mon Sep 17 00:00:00 2001 From: Hou Qi Date: Wed, 24 Jan 2024 10:28:38 +0800 Subject: [PATCH 39/65] LF-11398 wllinuxdmabuf: support cropping with left and top padding When adding one dmabuf to the set of zwp_linux_buffer_params_v1, the offset should not take into account left and top padding. But should use padded width and height to create wl buffer. Signed-off-by: Hou Qi --- gst-libs/gst/wayland/gstwllinuxdmabuf.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/gst-libs/gst/wayland/gstwllinuxdmabuf.c b/gst-libs/gst/wayland/gstwllinuxdmabuf.c index ac53e06816..ffa2605043 100644 --- a/gst-libs/gst/wayland/gstwllinuxdmabuf.c +++ b/gst-libs/gst/wayland/gstwllinuxdmabuf.c @@ -137,12 +137,29 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, params = zwp_linux_dmabuf_v1_create_params (gst_wl_display_get_dmabuf_v1 (display)); + if (vmeta) { + width += vmeta->alignment.padding_left + vmeta->alignment.padding_right; + height += vmeta->alignment.padding_top + vmeta->alignment.padding_bottom; + GST_DEBUG_OBJECT (display, "padding left: %d, right: %d, top: %d, bottom: %d", + vmeta->alignment.padding_left, vmeta->alignment.padding_right, + vmeta->alignment.padding_top, vmeta->alignment.padding_bottom); + GST_DEBUG_OBJECT (display, "padding size: %dx%d", width, height); + } + for (i = 0; i < nplanes; i++) { guint offset, stride, mem_idx, length; gsize skip; offset = offsets[i]; stride = strides[i]; + /* adjust the offset to take into account left and top */ + if (vmeta && (vmeta->alignment.padding_left || vmeta->alignment.padding_top)) { + gint vedge, hedge; + hedge = GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (drm_info->vinfo.finfo, i, vmeta->alignment.padding_left); + vedge = GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (drm_info->vinfo.finfo, i, vmeta->alignment.padding_top); + offset -= ((vedge * stride) + (hedge * GST_VIDEO_INFO_COMP_PSTRIDE (&drm_info->vinfo, i))); + } + if (gst_buffer_find_memory (buf, offset, 1, &mem_idx, &length, &skip)) { GstMemory *m = gst_buffer_peek_memory (buf, mem_idx); gint fd = gst_dmabuf_memory_get_fd (m); -- 2.25.1 From 51ec7c45b6caf80fb5e62d4037805c4ebed5b96e Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Tue, 9 Jul 2019 15:31:36 +0800 Subject: [PATCH 40/65] MMFMWK-8468 waylandsink: implement wayland linux-explicit-synchronization protocol add implementation of linux-explicit-synchronization protocol in waylandsink upstream status: pending https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/merge_requests/440 (cherry picked from commit f8489b517e57cd85a7546b596bda784f6da183a7) This commit also includes commit 7a6909acc8634cddd2dee78ddfe38befec1f1e37 MMFMWK-9268 wlbuffer: destroy buffer_release object if needed --- gst-libs/gst/wayland/gstwlbuffer.c | 56 ++++++++++++++- gst-libs/gst/wayland/gstwlbuffer.h | 15 +++++ gst-libs/gst/wayland/gstwldisplay.c | 19 ++++++ gst-libs/gst/wayland/gstwldisplay.h | 3 + gst-libs/gst/wayland/gstwlwindow.c | 101 ++++++++++++++++++++++++++++ gst-libs/gst/wayland/meson.build | 1 + 6 files changed, 194 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/wayland/gstwlbuffer.c b/gst-libs/gst/wayland/gstwlbuffer.c index 684fae9c8f..4caa991a5e 100644 --- a/gst-libs/gst/wayland/gstwlbuffer.c +++ b/gst-libs/gst/wayland/gstwlbuffer.c @@ -82,6 +82,8 @@ #include "gstwlbuffer.h" +#include "linux-explicit-synchronization-unstable-v1-client-protocol.h" + #define GST_CAT_DEFAULT gst_wl_buffer_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); @@ -93,6 +95,8 @@ typedef struct _GstWlBufferPrivate GstWlDisplay *display; + struct zwp_linux_buffer_release_v1 *buffer_release; + gboolean used_by_compositor; } GstWlBufferPrivate; @@ -189,6 +193,7 @@ gst_buffer_add_wl_buffer (GstBuffer * gstbuffer, struct wl_buffer *wlbuffer, { GstWlBuffer *self; GstWlBufferPrivate *priv; + struct zwp_linux_explicit_synchronization_v1 *explicit_sync; self = g_object_new (GST_TYPE_WL_BUFFER, NULL); priv = gst_wl_buffer_get_instance_private (self); @@ -199,7 +204,10 @@ gst_buffer_add_wl_buffer (GstBuffer * gstbuffer, struct wl_buffer *wlbuffer, gst_wl_display_register_buffer (priv->display, priv->gstmem, self); - wl_buffer_add_listener (priv->wlbuffer, &buffer_listener, self); + explicit_sync = gst_wl_display_get_explicit_sync (display); + if (!explicit_sync) { + wl_buffer_add_listener (priv->wlbuffer, &buffer_listener, self); + } gst_mini_object_weak_ref (GST_MINI_OBJECT (priv->gstmem), (GstMiniObjectNotify) gstmemory_disposed, self); @@ -261,6 +269,11 @@ gst_wl_buffer_force_release_and_unref (GstBuffer * buf, GstWlBuffer * self) priv->display = NULL; priv->current_gstbuffer = NULL; + /* destroy buffer_release object attacted before wl queue release */ + if (priv->buffer_release) { + zwp_linux_buffer_release_v1_destroy (priv->buffer_release); + } + /* remove the reference that the caller (GstWlDisplay) owns */ g_object_unref (self); } @@ -306,3 +319,44 @@ gst_wl_buffer_get_display (GstWlBuffer * self) return priv->display; } + +GstBuffer * +gst_wl_buffer_get_current_gstbuffer (GstWlBuffer * self) +{ + GstWlBufferPrivate *priv = gst_wl_buffer_get_instance_private (self); + + return priv->current_gstbuffer; +} + +struct zwp_linux_buffer_release_v1 * +gst_wl_buffer_get_buffer_release (GstWlBuffer * self) +{ + GstWlBufferPrivate *priv = gst_wl_buffer_get_instance_private (self); + + return priv->buffer_release; +} + +gboolean +gst_wl_buffer_get_used_by_compositor (GstWlBuffer * self) +{ + GstWlBufferPrivate *priv = gst_wl_buffer_get_instance_private (self); + + return priv->used_by_compositor; +} + +void +gst_wl_buffer_set_buffer_release (GstWlBuffer * self, + struct zwp_linux_buffer_release_v1 * buffer_release) +{ + GstWlBufferPrivate *priv = gst_wl_buffer_get_instance_private (self); + + priv->buffer_release = buffer_release; +} + +void +gst_wl_buffer_set_used_by_compositor (GstWlBuffer * self, gboolean used_by_compositor) +{ + GstWlBufferPrivate *priv = gst_wl_buffer_get_instance_private (self); + + priv->used_by_compositor = used_by_compositor; +} diff --git a/gst-libs/gst/wayland/gstwlbuffer.h b/gst-libs/gst/wayland/gstwlbuffer.h index 5a0a6e6f13..581923b1a8 100644 --- a/gst-libs/gst/wayland/gstwlbuffer.h +++ b/gst-libs/gst/wayland/gstwlbuffer.h @@ -56,4 +56,19 @@ void gst_wl_buffer_unref_buffer(GstWlBuffer * self); GST_WL_API GstWlDisplay *gst_wl_buffer_get_display (GstWlBuffer * self); +GST_WL_API +GstBuffer *gst_wl_buffer_get_current_gstbuffer (GstWlBuffer * self); + +GST_WL_API +struct zwp_linux_buffer_release_v1 *gst_wl_buffer_get_buffer_release (GstWlBuffer * self); + +GST_WL_API +gboolean gst_wl_buffer_get_used_by_compositor (GstWlBuffer * self); + +GST_WL_API +void gst_wl_buffer_set_buffer_release (GstWlBuffer * self, struct zwp_linux_buffer_release_v1 * buffer_release); + +GST_WL_API +void gst_wl_buffer_set_used_by_compositor (GstWlBuffer * self, gboolean used_by_compositor); + G_END_DECLS diff --git a/gst-libs/gst/wayland/gstwldisplay.c b/gst-libs/gst/wayland/gstwldisplay.c index 0876436c3d..141cc323bc 100644 --- a/gst-libs/gst/wayland/gstwldisplay.c +++ b/gst-libs/gst/wayland/gstwldisplay.c @@ -31,6 +31,7 @@ #include "xdg-shell-client-protocol.h" #include "alpha-compositing-unstable-v1-client-protocol.h" #include "hdr10-metadata-unstable-v1-client-protocol.h" +#include "linux-explicit-synchronization-unstable-v1-client-protocol.h" #include #include @@ -61,6 +62,8 @@ typedef struct _GstWlDisplayPrivate struct zwp_linux_dmabuf_v1 *dmabuf; struct zwp_alpha_compositing_v1 *alpha_compositing; struct zwp_hdr10_metadata_v1 *hdr10_metadata; + struct zwp_linux_explicit_synchronization_v1 *explicit_sync; + GArray *shm_formats; GArray *dmabuf_formats; GArray *dmabuf_modifiers; @@ -185,6 +188,9 @@ gst_wl_display_finalize (GObject * gobject) if (priv->hdr10_metadata) zwp_hdr10_metadata_v1_destroy (priv->hdr10_metadata); + if (priv->explicit_sync) + zwp_linux_explicit_synchronization_v1_destroy (priv->explicit_sync); + if (priv->registry) wl_registry_destroy (priv->registry); @@ -426,6 +432,11 @@ registry_handle_global (void *data, struct wl_registry *registry, } else if (g_strcmp0 (interface, "wl_shm") == 0) { priv->shm = wl_registry_bind (registry, id, &wl_shm_interface, 1); wl_shm_add_listener (priv->shm, &shm_listener, self); + } else if (g_strcmp0 (interface, + "zwp_linux_explicit_synchronization_v1") == 0) { + priv->explicit_sync = + wl_registry_bind (registry, id, + &zwp_linux_explicit_synchronization_v1_interface, 1); } else if (g_strcmp0 (interface, "wp_viewporter") == 0) { priv->viewporter = wl_registry_bind (registry, id, &wp_viewporter_interface, 1); @@ -793,6 +804,14 @@ gst_wl_display_get_hdr10_metadata (GstWlDisplay * self) return priv->hdr10_metadata; } +struct zwp_linux_explicit_synchronization_v1 * +gst_wl_display_get_explicit_sync (GstWlDisplay * self) +{ + GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); + + return priv->explicit_sync; +} + gint gst_wl_display_get_width (GstWlDisplay * self) { diff --git a/gst-libs/gst/wayland/gstwldisplay.h b/gst-libs/gst/wayland/gstwldisplay.h index 6c16405ee6..1a0592cf3b 100644 --- a/gst-libs/gst/wayland/gstwldisplay.h +++ b/gst-libs/gst/wayland/gstwldisplay.h @@ -106,6 +106,9 @@ struct wl_touch *gst_wl_display_get_touch (GstWlDisplay * self); GST_WL_API struct zwp_hdr10_metadata_v1 *gst_wl_display_get_hdr10_metadata (GstWlDisplay * self); +GST_WL_API +struct zwp_linux_explicit_synchronization_v1 *gst_wl_display_get_explicit_sync (GstWlDisplay * self); + GST_WL_API gint gst_wl_display_get_width (GstWlDisplay * self); diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index b2815498cc..2ef7598391 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -25,6 +25,7 @@ #endif #include +#include #include "gstwlwindow.h" #include "gstwlutils.h" @@ -35,6 +36,7 @@ #include "viewporter-client-protocol.h" #include "xdg-shell-client-protocol.h" #include "alpha-compositing-unstable-v1-client-protocol.h" +#include "linux-explicit-synchronization-unstable-v1-client-protocol.h" #define GST_CAT_DEFAULT gst_wl_window_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); @@ -56,6 +58,7 @@ typedef struct _GstWlWindowPrivate struct wp_viewport *video_viewport; struct xdg_surface *xdg_surface; struct xdg_toplevel *xdg_toplevel; + struct zwp_linux_surface_synchronization_v1 *surface_sync; gboolean configured; GCond configure_cond; GMutex configure_mutex; @@ -290,6 +293,79 @@ static const struct xdg_surface_listener xdg_surface_listener = { handle_xdg_surface_configure, }; +static gboolean +gst_wl_poll_wait_fence (int32_t fence) +{ + GstPollFD pollfd = GST_POLL_FD_INIT; + GstPoll *fence_poll = gst_poll_new (TRUE); + + pollfd.fd = fence; + gst_poll_add_fd (fence_poll, &pollfd); + gst_poll_fd_ctl_read (fence_poll, &pollfd, TRUE); + gst_poll_fd_ctl_write (fence_poll, &pollfd, TRUE); + + if (gst_poll_wait (fence_poll, GST_CLOCK_TIME_NONE) < 0) { + GST_ERROR ("wait on fence failed, errno %d", errno); + gst_poll_free (fence_poll); + return FALSE; + } + + GST_DEBUG ("wait on fence %d done", fence); + gst_poll_free (fence_poll); + + return TRUE; +} + +static void +buffer_fenced_release (void *data, + struct zwp_linux_buffer_release_v1 *release, int32_t fence) +{ + GstWlBuffer *buffer = data; + GstBuffer *current_gstbuffer = gst_wl_buffer_get_current_gstbuffer (buffer); + + g_assert (release == gst_wl_buffer_get_buffer_release (buffer)); + + gst_wl_buffer_set_used_by_compositor (buffer, FALSE); + zwp_linux_buffer_release_v1_destroy (gst_wl_buffer_get_buffer_release (buffer)); + gst_wl_buffer_set_buffer_release (buffer, NULL); + GST_LOG ("wl_buffer::fenced_release %d (GstBuffer: %p)", + fence, current_gstbuffer); + + if (fence > 0) { + gst_wl_poll_wait_fence (fence); + close (fence); + if (buffer) + gst_buffer_unref (current_gstbuffer); + } + +} + +static void +buffer_immediate_release (void *data, + struct zwp_linux_buffer_release_v1 *release) +{ + GstWlBuffer *buffer = data; + GstBuffer *current_gstbuffer = gst_wl_buffer_get_current_gstbuffer (buffer); + + g_assert (release == gst_wl_buffer_get_buffer_release (buffer)); + + gst_wl_buffer_set_used_by_compositor (buffer, FALSE); + zwp_linux_buffer_release_v1_destroy (gst_wl_buffer_get_buffer_release (buffer)); + gst_wl_buffer_set_buffer_release (buffer, NULL); + GST_LOG ("wl_buffer::immediate_release (GstBuffer: %p)", + current_gstbuffer); + + /* unref should be last, because it may end up destroying the GstWlBuffer */ + gst_buffer_unref (current_gstbuffer); +} + +static const struct zwp_linux_buffer_release_v1_listener buffer_release_listener + = { + buffer_fenced_release, + buffer_immediate_release, +}; + + static void gst_wl_window_class_init (GstWlWindowClass * klass) { @@ -344,6 +420,9 @@ gst_wl_window_finalize (GObject * gobject) if (priv->video_viewport) wp_viewport_destroy (priv->video_viewport); + if (priv->surface_sync) + zwp_linux_surface_synchronization_v1_destroy (priv->surface_sync); + wl_proxy_wrapper_destroy (priv->video_surface_wrapper); if (priv->blend_func) @@ -376,6 +455,7 @@ gst_wl_window_new_internal (GstWlDisplay * display, GMutex * render_lock) struct wl_region *region; struct wp_viewporter *viewporter; struct zwp_alpha_compositing_v1 *alpha_compositing; + struct zwp_linux_explicit_synchronization_v1 *explicit_sync; gint width, height; self = g_object_new (GST_TYPE_WL_WINDOW, NULL); @@ -417,6 +497,12 @@ gst_wl_window_new_internal (GstWlDisplay * display, GMutex * render_lock) zwp_alpha_compositing_v1_get_blending (alpha_compositing, priv->area_surface); + explicit_sync = gst_wl_display_get_explicit_sync (display); + if (explicit_sync) + priv->surface_sync = + zwp_linux_explicit_synchronization_v1_get_synchronization(explicit_sync, + priv->video_surface_wrapper); + /* never accept input events on the video surface */ region = wl_compositor_create_region (compositor); wl_surface_set_input_region (priv->video_surface, region); @@ -777,6 +863,9 @@ gst_wl_window_commit_buffer (GstWlWindow * self, GstWlBuffer * buffer) GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); GstVideoInfo *info = priv->next_video_info; struct wl_callback *callback; + GstBuffer *current_gstbuffer; + struct zwp_linux_buffer_release_v1 *buffer_release; + gboolean used_by_compositor; if (G_UNLIKELY (info)) { priv->scaled_width = @@ -789,6 +878,18 @@ gst_wl_window_commit_buffer (GstWlWindow * self, GstWlBuffer * buffer) gst_wl_window_set_opaque (self, info); } + current_gstbuffer = gst_wl_buffer_get_current_gstbuffer (buffer); + used_by_compositor = gst_wl_buffer_get_used_by_compositor (buffer); + if (buffer && !used_by_compositor && priv->surface_sync) { + GST_DEBUG ("use explicit sync create buffer release (GstBuffer: %p)", + current_gstbuffer); + gst_wl_buffer_set_buffer_release (buffer, + zwp_linux_surface_synchronization_v1_get_release (priv->surface_sync)); + buffer_release = gst_wl_buffer_get_buffer_release (buffer); + zwp_linux_buffer_release_v1_add_listener (buffer_release, + &buffer_release_listener, buffer); + } + if (G_LIKELY (buffer)) { callback = wl_surface_frame (priv->video_surface_wrapper); priv->frame_callback = callback; diff --git a/gst-libs/gst/wayland/meson.build b/gst-libs/gst/wayland/meson.build index 912d26e4e3..b0f12eed61 100644 --- a/gst-libs/gst/wayland/meson.build +++ b/gst-libs/gst/wayland/meson.build @@ -45,6 +45,7 @@ if use_wayland ['single-pixel-buffer', 'staging', 'v1' ], ['alpha-compositing', 'unstable', 'v1', ], ['hdr10-metadata', 'unstable', 'v1', ], + ['linux-explicit-synchronization', 'unstable', 'v1'], ] protocols_files = [] -- 2.25.1 From 23e3533a2c0312db30fd55125b905a0b32bfd4ae Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Tue, 25 Oct 2022 15:25:09 +0800 Subject: [PATCH 41/65] wldisplay: fix memory leak of wayland objects need release the memory of wl pointer, seat, touch & outputs, also need release the hash table of dmabuf_modifiers upstream status: imx specific --- gst-libs/gst/wayland/gstwldisplay.c | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/wayland/gstwldisplay.c b/gst-libs/gst/wayland/gstwldisplay.c index 141cc323bc..28be22fa02 100644 --- a/gst-libs/gst/wayland/gstwldisplay.c +++ b/gst-libs/gst/wayland/gstwldisplay.c @@ -57,7 +57,6 @@ typedef struct _GstWlDisplayPrivate struct zwp_fullscreen_shell_v1 *fullscreen_shell; struct wp_single_pixel_buffer_manager_v1 *single_pixel_buffer; struct wl_shm *shm; - struct wl_output *output; struct wp_viewporter *viewporter; struct zwp_linux_dmabuf_v1 *dmabuf; struct zwp_alpha_compositing_v1 *alpha_compositing; @@ -67,6 +66,7 @@ typedef struct _GstWlDisplayPrivate GArray *shm_formats; GArray *dmabuf_formats; GArray *dmabuf_modifiers; + GArray *outputs; /* real display resolution */ gint width, height; @@ -109,6 +109,7 @@ gst_wl_display_init (GstWlDisplay * self) priv->shm_formats = g_array_new (FALSE, FALSE, sizeof (uint32_t)); priv->dmabuf_formats = g_array_new (FALSE, FALSE, sizeof (uint32_t)); priv->dmabuf_modifiers = g_array_new (FALSE, FALSE, sizeof (guint64)); + priv->outputs = g_array_new (FALSE, FALSE, sizeof (struct wl_output *)); priv->wl_fd_poll = gst_poll_new (TRUE); priv->buffers = g_hash_table_new (g_direct_hash, g_direct_equal); priv->width = -1; @@ -132,6 +133,8 @@ gst_wl_ref_wl_buffer (gpointer key, gpointer value, gpointer user_data) static void gst_wl_display_finalize (GObject * gobject) { + guint i; + struct wl_output *output = NULL; GstWlDisplay *self = GST_WL_DISPLAY (gobject); GstWlDisplayPrivate *priv = gst_wl_display_get_instance_private (self); @@ -170,6 +173,15 @@ gst_wl_display_finalize (GObject * gobject) if (priv->xdg_wm_base) xdg_wm_base_destroy (priv->xdg_wm_base); + if (priv->seat) + wl_seat_destroy (priv->seat); + + if (priv->pointer) + wl_pointer_destroy (priv->pointer); + + if (priv->touch) + wl_touch_destroy (priv->touch); + if (priv->fullscreen_shell) zwp_fullscreen_shell_v1_release (priv->fullscreen_shell); @@ -191,6 +203,15 @@ gst_wl_display_finalize (GObject * gobject) if (priv->explicit_sync) zwp_linux_explicit_synchronization_v1_destroy (priv->explicit_sync); + if (priv->outputs) { + for (i = 0; i < priv->outputs->len; i++) { + output = g_array_index (priv->outputs, struct wl_output *, i); + if (output) + wl_output_destroy (output); + } + g_array_unref (priv->outputs); + } + if (priv->registry) wl_registry_destroy (priv->registry); @@ -455,9 +476,11 @@ registry_handle_global (void *data, struct wl_registry *registry, priv->hdr10_metadata = wl_registry_bind (registry, id, &zwp_hdr10_metadata_v1_interface, 1); } else if (g_strcmp0 (interface, "wl_output") == 0) { - priv->output = + struct wl_output *output; + output = wl_registry_bind (registry, id, &wl_output_interface, MIN (version, 2)); - wl_output_add_listener (priv->output, &output_listener, self); + wl_output_add_listener (output, &output_listener, self); + g_array_append_val (priv->outputs, output); } } -- 2.25.1 From 61dc5775da984e0af3515abfa8dff21a1dbc2ca2 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Tue, 12 Dec 2023 00:21:28 +0900 Subject: [PATCH 42/65] gstwlwindow: add protection when call gstwlbuffer api with null wlbuffer upstream status: imx specific --- gst-libs/gst/wayland/gstwlwindow.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index 2ef7598391..68afdca28b 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -878,19 +878,19 @@ gst_wl_window_commit_buffer (GstWlWindow * self, GstWlBuffer * buffer) gst_wl_window_set_opaque (self, info); } - current_gstbuffer = gst_wl_buffer_get_current_gstbuffer (buffer); - used_by_compositor = gst_wl_buffer_get_used_by_compositor (buffer); - if (buffer && !used_by_compositor && priv->surface_sync) { - GST_DEBUG ("use explicit sync create buffer release (GstBuffer: %p)", + if (G_LIKELY (buffer)) { + current_gstbuffer = gst_wl_buffer_get_current_gstbuffer (buffer); + used_by_compositor = gst_wl_buffer_get_used_by_compositor (buffer); + if (!used_by_compositor && priv->surface_sync) { + GST_DEBUG ("use explicit sync create buffer release (GstBuffer: %p)", current_gstbuffer); - gst_wl_buffer_set_buffer_release (buffer, - zwp_linux_surface_synchronization_v1_get_release (priv->surface_sync)); - buffer_release = gst_wl_buffer_get_buffer_release (buffer); - zwp_linux_buffer_release_v1_add_listener (buffer_release, - &buffer_release_listener, buffer); - } + gst_wl_buffer_set_buffer_release (buffer, + zwp_linux_surface_synchronization_v1_get_release (priv->surface_sync)); + buffer_release = gst_wl_buffer_get_buffer_release (buffer); + zwp_linux_buffer_release_v1_add_listener (buffer_release, + &buffer_release_listener, buffer); + } - if (G_LIKELY (buffer)) { callback = wl_surface_frame (priv->video_surface_wrapper); priv->frame_callback = callback; wl_callback_add_listener (callback, &frame_callback_listener, self); -- 2.25.1 From e7df9699b486148146a98c4902e867f92e44d4fa Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Mon, 23 Apr 2018 12:38:38 +0800 Subject: [PATCH 43/65] MMFMWK-7954 waylandsink: pass dmabuf modifier to weston upstream status: imx specific (cherry picked from commit 78d9555078c83b1b09bc603b9bbd05c9313d2611) --- ext/wayland/gstwaylandsink.c | 7 +++++++ gst-libs/gst/wayland/gstwllinuxdmabuf.c | 12 ++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index 9ec5fca88c..892ffa4947 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -51,6 +51,9 @@ #include #include +#include +#include + #include "hdr10-metadata-unstable-v1-client-protocol.h" /* signals */ @@ -847,6 +850,7 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) GstBufferPool *pool = NULL; gboolean need_pool; GstAllocator *alloc; + guint64 drm_modifier; GstVideoInfoDmaDrm drm_info; GstVideoInfo vinfo; guint size; @@ -867,6 +871,9 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) size = vinfo.size; } + drm_modifier = DRM_FORMAT_MOD_AMPHION_TILED; + gst_query_add_allocation_dmabuf_meta (query, drm_modifier); + if (need_pool) { GstStructure *config; pool = gst_wl_video_buffer_pool_new (); diff --git a/gst-libs/gst/wayland/gstwllinuxdmabuf.c b/gst-libs/gst/wayland/gstwllinuxdmabuf.c index ffa2605043..dc9af10b40 100644 --- a/gst-libs/gst/wayland/gstwllinuxdmabuf.c +++ b/gst-libs/gst/wayland/gstwllinuxdmabuf.c @@ -24,6 +24,8 @@ #include #endif +#include + #include "gstwllinuxdmabuf.h" #include "linux-dmabuf-unstable-v1-client-protocol.h" @@ -87,7 +89,7 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, { GstMemory *mem; guint fourcc; - guint64 modifier; + guint64 modifier = 0; guint i, width = 0, height = 0; GstVideoInfo info; const gsize *offsets = NULL; @@ -97,13 +99,19 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, struct zwp_linux_buffer_params_v1 *params; gint64 timeout; ConstructBufferData data; + GstDmabufMeta *dmabuf_meta; g_return_val_if_fail (gst_wl_display_check_format_for_dmabuf (display, drm_info), NULL); mem = gst_buffer_peek_memory (buf, 0); fourcc = drm_info->drm_fourcc; - modifier = drm_info->drm_modifier; + + dmabuf_meta = gst_buffer_get_dmabuf_meta (buf); + if (dmabuf_meta) + modifier = dmabuf_meta->drm_modifier; + else + modifier = drm_info->drm_modifier; g_cond_init (&data.cond); g_mutex_init (&data.lock); -- 2.25.1 From 1bedf78be918f84f06cc8c038535cfe42a6c8b56 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 5 Sep 2018 09:36:45 +0800 Subject: [PATCH 44/65] MMFMWK-7954 waylandsink: propose vsi modifier to vpu propose vsi modifiers to vpu to support DTRC in dcss, default will disable this feature. Can enable by set property enable-tile to true. (cherry picked from commit 71e6c63fd328a4c11ebafd6e7c7853cb11346dc3) --- ext/wayland/gstwaylandsink.c | 23 +++++++++++++++++++++++ ext/wayland/gstwaylandsink.h | 2 ++ 2 files changed, 25 insertions(+) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index 892ffa4947..ac5755e999 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -72,6 +72,7 @@ enum PROP_DISPLAY, PROP_FULLSCREEN, PROP_ALPHA, + PROP_ENABLE_TILE, PROP_ROTATE_METHOD, PROP_DRM_DEVICE, PROP_LAST @@ -187,6 +188,11 @@ gst_wayland_sink_class_init (GstWaylandSinkClass * klass) "surface alpha value, apply custom alpha value to wayland surface", 0.0f, 1.0f, 0.0f, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_ENABLE_TILE, + g_param_spec_boolean ("enable-tile", "enable hantro tile", + "When enabled, the sink propose VSI tile modifier to VPU", FALSE, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT)); + /** * waylandsink:rotate-method: * @@ -232,6 +238,7 @@ gst_wayland_sink_init (GstWaylandSink * self) g_mutex_init (&self->render_lock); self->frame_showed = 0; self->run_time = 0; + self->enable_tile = FALSE; } static void @@ -319,6 +326,9 @@ gst_wayland_sink_get_property (GObject * object, g_value_set_string (value, self->drm_device); GST_OBJECT_UNLOCK (self); break; + case PROP_ENABLE_TILE: + g_value_set_boolean (value, self->enable_tile); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -360,6 +370,9 @@ gst_wayland_sink_set_property (GObject * object, self->drm_device = g_value_dup_string (value); GST_OBJECT_UNLOCK (self); break; + case PROP_ENABLE_TILE: + self->enable_tile = g_value_get_boolean (value); + break; default: if (!gst_video_overlay_set_property (object, PROP_LAST, prop_id, value)) G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -846,6 +859,7 @@ unsupported_format: static gboolean gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) { + GstWaylandSink *self = GST_WAYLAND_SINK (bsink); GstCaps *caps; GstBufferPool *pool = NULL; gboolean need_pool; @@ -874,6 +888,15 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) drm_modifier = DRM_FORMAT_MOD_AMPHION_TILED; gst_query_add_allocation_dmabuf_meta (query, drm_modifier); + if (self->enable_tile && HAS_DCSS ()) { + drm_modifier = DRM_FORMAT_MOD_VSI_G1_TILED; + gst_query_add_allocation_dmabuf_meta (query, drm_modifier); + drm_modifier = DRM_FORMAT_MOD_VSI_G2_TILED; + gst_query_add_allocation_dmabuf_meta (query, drm_modifier); + drm_modifier = DRM_FORMAT_MOD_VSI_G2_TILED_COMPRESSED; + gst_query_add_allocation_dmabuf_meta (query, drm_modifier); + } + if (need_pool) { GstStructure *config; pool = gst_wl_video_buffer_pool_new (); diff --git a/ext/wayland/gstwaylandsink.h b/ext/wayland/gstwaylandsink.h index 5666e13d8f..c8936186fa 100644 --- a/ext/wayland/gstwaylandsink.h +++ b/ext/wayland/gstwaylandsink.h @@ -79,6 +79,8 @@ struct _GstWaylandSink /* fps print support */ guint64 frame_showed; GstClockTime run_time; + + gboolean enable_tile; }; struct _GstWaylandSinkClass -- 2.25.1 From 98977ee224a9beee046961cf195d40cc8d3ed3de Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 5 Sep 2018 15:53:48 +0800 Subject: [PATCH 45/65] MMFMWK-7954 waylandsink: pass dtrc table offset to compositor we pass dtrc table offset to compositor when modifier is not LINEAR and phymem meta is valid upstream status: imx specific (cherry picked from commit 2d83c233451cec96aa2f013e700fb95bde3a9195) --- gst-libs/gst/wayland/gstwllinuxdmabuf.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gst-libs/gst/wayland/gstwllinuxdmabuf.c b/gst-libs/gst/wayland/gstwllinuxdmabuf.c index dc9af10b40..1e0dc02548 100644 --- a/gst-libs/gst/wayland/gstwllinuxdmabuf.c +++ b/gst-libs/gst/wayland/gstwllinuxdmabuf.c @@ -25,6 +25,7 @@ #endif #include +#include #include "gstwllinuxdmabuf.h" @@ -100,6 +101,7 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, gint64 timeout; ConstructBufferData data; GstDmabufMeta *dmabuf_meta; + GstPhyMemMeta *phymemmeta = NULL; g_return_val_if_fail (gst_wl_display_check_format_for_dmabuf (display, drm_info), NULL); @@ -192,6 +194,16 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, } } + phymemmeta = GST_PHY_MEM_META_GET (buf); + if (modifier != 0 && phymemmeta) { + GST_DEBUG_OBJECT (display, "physical memory meta x_padding: %d y_padding: %d \ + RFC luma offset: %d chroma offset: %d", + phymemmeta->x_padding, phymemmeta->y_padding, phymemmeta->rfc_luma_offset, phymemmeta->rfc_chroma_offset); + zwp_linux_buffer_params_v1_add_dtrc_meta (params, phymemmeta->rfc_chroma_offset, phymemmeta->rfc_luma_offset); + } else { + zwp_linux_buffer_params_v1_add_dtrc_meta (params, 0, 0); + } + /* Request buffer creation */ zwp_linux_buffer_params_v1_add_listener (params, ¶ms_listener, &data); zwp_linux_buffer_params_v1_create (params, width, height, fourcc, flags); -- 2.25.1 From dd6db8497ee1ee6707ee689c85360c8823f201e2 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 20 Mar 2024 19:14:51 +0900 Subject: [PATCH 46/65] MMFMWK-8195 waylandsink: add internal dmabuf pool to support sw decode Add internal dmabuf pool to support SW decode. Propose to upstream if available. upstreams status: imx specific --- ext/wayland/gstwaylandsink.c | 111 +++++++++++++++++++++++++++++++---- ext/wayland/gstwaylandsink.h | 2 +- ext/wayland/meson.build | 8 ++- 3 files changed, 109 insertions(+), 12 deletions(-) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index ac5755e999..88766b99cd 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -51,6 +51,10 @@ #include #include +#ifdef HAVE_DMABUFHEAPS_ALLOCATOR +#include +#endif + #include #include @@ -78,6 +82,8 @@ enum PROP_LAST }; +#define ISALIGNED(a, b) (!(a & (b-1))) + GST_DEBUG_CATEGORY (gstwayland_debug); #define GST_CAT_DEFAULT gstwayland_debug @@ -699,6 +705,8 @@ static gboolean gst_wayland_update_pool (GstWaylandSink * self, GstAllocator * allocator) { gsize size = self->video_info.size; + gint w = GST_VIDEO_INFO_WIDTH (&self->video_info); + gint h = GST_VIDEO_INFO_HEIGHT (&self->video_info); GstStructure *config; /* Pools with outstanding buffer cannot be reconfigured, so we must use @@ -714,6 +722,24 @@ gst_wayland_update_pool (GstWaylandSink * self, GstAllocator * allocator) gst_buffer_pool_config_set_params (config, self->caps, size, 2, 0); gst_buffer_pool_config_set_allocator (config, allocator, NULL); + if (!ISALIGNED (w, 16) || !ISALIGNED (h, 16)) { + GstVideoAlignment alignment; + + memset (&alignment, 0, sizeof (GstVideoAlignment)); + alignment.padding_right = GST_ROUND_UP_N (w, 16) - w; + alignment.padding_bottom = GST_ROUND_UP_N (h, 16) - h; + + GST_DEBUG + ("align buffer pool, w(%d) h(%d), padding_right (%d), padding_bottom (%d)", + w, h, alignment.padding_right, alignment.padding_bottom); + + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_META); + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT); + gst_buffer_pool_config_set_video_alignment (config, &alignment); + } + if (!gst_buffer_pool_set_config (self->pool, config)) return FALSE; @@ -778,6 +804,35 @@ gst_wayland_activate_drm_dumb_pool (GstWaylandSink * self) return TRUE; } +static gboolean +gst_wayland_activate_dmaheaps_pool (GstWaylandSink * self) +{ +#ifdef HAVE_DMABUFHEAPS_ALLOCATOR + GstAllocator *alloc = NULL; + + if (self->pool && gst_buffer_pool_is_active (self->pool)) { + GstStructure *config = gst_buffer_pool_get_config (self->pool); + gboolean is_dmaheaps = FALSE; + + if (gst_buffer_pool_config_get_allocator (config, &alloc, NULL) && alloc) + is_dmaheaps = GST_IS_DMABUFHEAPS_ALLOCATOR (alloc); + + gst_structure_free (config); + + if (is_dmaheaps) + return TRUE; + } + + alloc = gst_dmabufheaps_allocator_obtain (); + gst_wayland_update_pool (self, alloc); + gst_object_unref (alloc); + + return TRUE; +#else + return FALSE; +#endif +} + static gboolean gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) { @@ -804,7 +859,7 @@ gst_wayland_sink_set_caps (GstBaseSink * bsink, GstCaps * caps) } self->video_info_changed = TRUE; - self->skip_dumb_buffer_copy = FALSE; + self->skip_dma_buffer_copy = FALSE; /* free pooled buffer used with previous caps */ if (self->pool) { @@ -863,7 +918,7 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) GstCaps *caps; GstBufferPool *pool = NULL; gboolean need_pool; - GstAllocator *alloc; + GstAllocator *alloc = NULL; guint64 drm_modifier; GstVideoInfoDmaDrm drm_info; GstVideoInfo vinfo; @@ -901,9 +956,39 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) GstStructure *config; pool = gst_wl_video_buffer_pool_new (); config = gst_buffer_pool_get_config (pool); + + gint w = GST_VIDEO_INFO_WIDTH (&self->video_info); + gint h = GST_VIDEO_INFO_HEIGHT (&self->video_info); + + if (gst_wl_display_check_format_for_dmabuf (self->display, &self->drm_info)) { +#ifdef HAVE_DMABUFHEAPS_ALLOCATOR + alloc = gst_dmabufheaps_allocator_obtain (); +#endif + } else { + alloc = gst_shm_allocator_get (); + } + + if (!ISALIGNED (w, 16) || !ISALIGNED (h, 16)) { + GstVideoAlignment alignment; + + memset (&alignment, 0, sizeof (GstVideoAlignment)); + alignment.padding_right = GST_ROUND_UP_N (w, 16) - w; + alignment.padding_bottom = GST_ROUND_UP_N (h, 16) - h; + + GST_DEBUG_OBJECT + (self, "align buffer pool, w(%d) h(%d), padding_right (%d), padding_bottom (%d)", + w, h, alignment.padding_right, alignment.padding_bottom); + + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT); + gst_buffer_pool_config_set_video_alignment (config, &alignment); + } + gst_buffer_pool_config_set_params (config, caps, size, 2, 0); gst_buffer_pool_config_set_allocator (config, - gst_shm_allocator_get (), NULL); + alloc, NULL); + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_META); gst_buffer_pool_set_config (pool, config); } @@ -911,11 +996,13 @@ gst_wayland_sink_propose_allocation (GstBaseSink * bsink, GstQuery * query) if (pool) g_object_unref (pool); - alloc = gst_shm_allocator_get (); - gst_query_add_allocation_param (query, alloc, NULL); + if (alloc) { + gst_object_ref_sink (alloc); + gst_query_add_allocation_param (query, alloc, NULL); + g_object_unref (alloc); + } gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); gst_query_add_allocation_meta (query, GST_VIDEO_CROP_META_API_TYPE, NULL); - g_object_unref (alloc); return TRUE; } @@ -1097,7 +1184,7 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) wbuf = gst_wl_linux_dmabuf_construct_wl_buffer (buffer, self->display, &self->drm_info); - if (!wbuf && !self->skip_dumb_buffer_copy) { + if (!wbuf && !self->skip_dma_buffer_copy) { /* DMABuf did not work, let try and make this a dmabuf, it does not matter * if it was a SHM since the compositor needs to copy that anyway, and * offloading the compositor from a copy helps maintaining a smoother @@ -1105,8 +1192,12 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) */ GstVideoFrame src, dst; - if (!gst_wayland_activate_drm_dumb_pool (self)) { - self->skip_dumb_buffer_copy = TRUE; + if (gst_wayland_activate_dmaheaps_pool (self)) { + GST_WARNING_OBJECT (self, "active dmaheaps pool"); + } else if (gst_wayland_activate_drm_dumb_pool (self)) { + GST_WARNING_OBJECT (self, "active drm dump pool"); + } else { + self->skip_dma_buffer_copy = TRUE; goto handle_shm; } @@ -1124,7 +1215,7 @@ gst_wayland_sink_show_frame (GstVideoSink * vsink, GstBuffer * buffer) if (G_UNLIKELY (!wbuf)) { GST_WARNING_OBJECT (self, "failed to import DRM Dumb dmabuf"); gst_clear_buffer (&to_render); - self->skip_dumb_buffer_copy = TRUE; + self->skip_dma_buffer_copy = TRUE; goto handle_shm; } diff --git a/ext/wayland/gstwaylandsink.h b/ext/wayland/gstwaylandsink.h index c8936186fa..d0ebe47ff8 100644 --- a/ext/wayland/gstwaylandsink.h +++ b/ext/wayland/gstwaylandsink.h @@ -74,7 +74,7 @@ struct _GstWaylandSink GstVideoOrientationMethod current_rotate_method; gchar *drm_device; - gboolean skip_dumb_buffer_copy; + gboolean skip_dma_buffer_copy; /* fps print support */ guint64 frame_showed; diff --git a/ext/wayland/meson.build b/ext/wayland/meson.build index f0a501a2c0..36c3968a00 100644 --- a/ext/wayland/meson.build +++ b/ext/wayland/meson.build @@ -2,11 +2,17 @@ wl_sources = [ 'gstwaylandsink.c' ] +dmabufheapsallocator_flags = [] + +if cc.has_header('linux/dma-heap.h') and cc.has_header('gst/allocators/gstdmabufheaps.h', dependencies: gstallocators_dep) + dmabufheapsallocator_flags += ['-DHAVE_DMABUFHEAPS_ALLOCATOR'] +endif + if use_wayland gstwaylandsink = library('gstwaylandsink', wl_sources + protocols_files, - c_args : gst_plugins_bad_args + ['-DGST_USE_UNSTABLE_API'], + c_args : gst_plugins_bad_args + dmabufheapsallocator_flags + ['-DGST_USE_UNSTABLE_API'], include_directories : [configinc], dependencies : [gst_dep, gstvideo_dep, gstwayland_dep, wl_client_dep, wl_protocol_dep, libdrm_dep], -- 2.25.1 From 2b4c363b2048c75a9745a358976202812352c9d3 Mon Sep 17 00:00:00 2001 From: Hou Qi Date: Thu, 28 Mar 2024 14:17:01 +0800 Subject: [PATCH 47/65] Revert "jpegparse: pass colorimetry in caps" imx-jpeg driver only supports below colorimetry when try_fmt, which can be transformed to gst mapping 1:4:7:1. If jpegparse adds colorimetry=sRGB to caps, it causes negotiation failure between jpegparse and v4l2h264dec for this pipeline "video/x-raw,format=BGR ! v4l2jpegenc ! jpegparse ! v4l2jpegdec" pix_mp->colorspace = V4L2_COLORSPACE_SRGB; pix_mp->ycbcr_enc = V4L2_YCBCR_ENC_601; pix_mp->xfer_func = V4L2_XFER_FUNC_SRGB; pix_mp->quantization = V4L2_QUANTIZATION_FULL_RANGE; This reverts commit 75a3e0b52ae4ecf62faeed7a46415edd5e8c61f4. --- gst/jpegformat/gstjpegparse.c | 9 --------- gst/jpegformat/gstjpegparse.h | 1 - 2 files changed, 10 deletions(-) diff --git a/gst/jpegformat/gstjpegparse.c b/gst/jpegformat/gstjpegparse.c index 1dda8c5f1a..7a79d4bc9d 100644 --- a/gst/jpegformat/gstjpegparse.c +++ b/gst/jpegformat/gstjpegparse.c @@ -247,9 +247,6 @@ gst_jpeg_parse_set_sink_caps (GstBaseParse * bparse, GstCaps * caps) parse->field_order = gst_video_field_order_from_string (field_order); } - g_clear_pointer (&parse->colorimetry, g_free); - parse->colorimetry = g_strdup (gst_structure_get_string (s, "colorimetry")); - return TRUE; } @@ -763,11 +760,6 @@ gst_jpeg_parse_set_new_caps (GstJpegParse * parse) sampling_to_string (parse->sampling), NULL); } - if (parse->colorimetry) { - gst_caps_set_simple (caps, "colorimetry", G_TYPE_STRING, parse->colorimetry, - NULL); - } - gst_caps_set_simple (caps, "interlace-mode", G_TYPE_STRING, gst_video_interlace_mode_to_string (parse->interlace_mode), NULL); @@ -1037,7 +1029,6 @@ gst_jpeg_parse_stop (GstBaseParse * bparse) } gst_clear_buffer (&parse->codec_data); gst_clear_caps (&parse->prev_caps); - g_clear_pointer (&parse->colorimetry, g_free); return TRUE; } diff --git a/gst/jpegformat/gstjpegparse.h b/gst/jpegformat/gstjpegparse.h index d6a88fdaf3..20ef6e9ac5 100644 --- a/gst/jpegformat/gstjpegparse.h +++ b/gst/jpegformat/gstjpegparse.h @@ -64,7 +64,6 @@ struct _GstJpegParse { gint orig_width, orig_height; GstBuffer *codec_data; - char *colorimetry; GstVideoInterlaceMode interlace_mode; GstVideoFieldOrder field_order; guint field; -- 2.25.1 From 286822e61bf228402b7dbdc7f192b8159c928f67 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 23 Sep 2020 19:39:54 +0800 Subject: [PATCH 48/65] MMFMWK-7954 waylandsink: default to play video fullscreen upstream status: imx specific --- gst-libs/gst/wayland/gstwlutils.c | 15 +++++--- gst-libs/gst/wayland/gstwlutils.h | 8 +++- gst-libs/gst/wayland/gstwlwindow.c | 62 +++++++++++++++++++++++++++--- gst-libs/gst/wayland/gstwlwindow.h | 12 ++++++ 4 files changed, 85 insertions(+), 12 deletions(-) diff --git a/gst-libs/gst/wayland/gstwlutils.c b/gst-libs/gst/wayland/gstwlutils.c index 7c556a8a8a..63e3c6f0c5 100644 --- a/gst-libs/gst/wayland/gstwlutils.c +++ b/gst-libs/gst/wayland/gstwlutils.c @@ -38,13 +38,13 @@ #define WESTON_INI "/etc/xdg/weston/weston.ini" gboolean -gst_wl_init_buffer_scale (gint display_width, gint display_height, - guint * scale) +gst_wl_init_surface_state (GstWlDisplay * display, GstWlWindow * self) { gchar path[] = WESTON_INI; gchar line[512], *p, *section = NULL, *size = NULL; gint fd, n, i; gint desktop_width, desktop_height; + gint display_width, display_height; gboolean found_config = FALSE; gboolean ret = TRUE; struct stat filestat; @@ -124,11 +124,16 @@ gst_wl_init_buffer_scale (gint display_width, gint display_height, } /* FIXME: only support buffer scale 2 and 1 */ + display_width = gst_wl_display_get_width (display); + display_height = gst_wl_display_get_height (display); if (display_width > 0 && display_height > 0) { - *scale = display_width / desktop_width; - if (*scale != 1 && *scale != 2) { - *scale = 1; + gst_wl_window_set_scale (self, display_width / desktop_width); + if (gst_wl_window_get_scale (self) != 1 && gst_wl_window_get_scale (self) != 2) { + ret = FALSE; + goto out; } + gst_wl_window_set_fullscreen_width (self, desktop_width); + gst_wl_window_set_fullscreen_height (self, desktop_height - PANEL_HEIGH); } else { ret = FALSE; goto out; diff --git a/gst-libs/gst/wayland/gstwlutils.h b/gst-libs/gst/wayland/gstwlutils.h index 285dcaeca2..90c3b0ecde 100644 --- a/gst-libs/gst/wayland/gstwlutils.h +++ b/gst-libs/gst/wayland/gstwlutils.h @@ -23,9 +23,15 @@ #include +#include "gstwlwindow.h" +#include "gstwldisplay.h" + G_BEGIN_DECLS -gboolean gst_wl_init_buffer_scale(gint display_width, gint display_height, guint * scale); +/* FIXME: try to get from wayland server */ +#define PANEL_HEIGH 32 + +gboolean gst_wl_init_surface_state(GstWlDisplay * display, GstWlWindow * self); G_END_DECLS #endif \ No newline at end of file diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index 68afdca28b..a95f947c2f 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -100,6 +100,8 @@ typedef struct _GstWlWindowPrivate /* mouse location when click */ gint pointer_x, pointer_y; + /* fullscreen window size */ + gint fullscreen_width, fullscreen_height; } GstWlWindowPrivate; G_DEFINE_TYPE_WITH_CODE (GstWlWindow, gst_wl_window, G_TYPE_OBJECT, @@ -394,6 +396,8 @@ gst_wl_window_init (GstWlWindow * self) priv->src_width = -1; priv->src_height = 0; priv->scale = 1; + priv->fullscreen_width = -1; + priv->fullscreen_height = -1; } static void @@ -511,8 +515,13 @@ gst_wl_window_new_internal (GstWlDisplay * display, GMutex * render_lock) width = gst_wl_display_get_width (display); height = gst_wl_display_get_height (display); - if (!gst_wl_init_buffer_scale (width, height, &priv->scale)) { - GST_WARNING ("init buffer scale fail, fallback to scale=%d", priv->scale); + if (!gst_wl_init_surface_state (display, self)) { + priv->fullscreen_width = width; + priv->fullscreen_height = height - PANEL_HEIGH; + priv->scale = 1; + GST_WARNING + ("init surface_state fail, fallback to scale=%d fullscreen (%dx%d)", + priv->scale, priv->fullscreen_width, priv->fullscreen_height); } return self; @@ -620,12 +629,15 @@ gst_wl_window_new_toplevel (GstWlDisplay * display, const GstVideoInfo * info, if (preferred_width > 0 && preferred_height > 0) { width = preferred_width; height = preferred_height; - } else { + } else if (priv->fullscreen_width <= 0) { /* set the initial size to be the same as the reported video size */ - width = + width = gst_util_uint64_scale_int_round (info->width, info->par_n, - info->par_d) / priv->scale; - height = info->height / priv->scale; + info->par_d); + height = info->height; + } else { + width = priv->fullscreen_width; + height = priv->fullscreen_height; } gst_wl_window_set_render_rectangle (self, 0, 0, width, height); @@ -1182,3 +1194,41 @@ gst_wl_window_set_rotate_method (GstWlWindow * self, gst_wl_window_update_geometry (self); } + +void +gst_wl_window_set_scale (GstWlWindow * self, gint scale) +{ + GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); + + priv->scale = scale; + + gst_wl_window_update_geometry (self); +} + +guint +gst_wl_window_get_scale (GstWlWindow * self) +{ + GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); + + return priv->scale; +} + +void +gst_wl_window_set_fullscreen_width (GstWlWindow * self, gint fullscreen_width) +{ + GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); + + priv->fullscreen_width = fullscreen_width; + + gst_wl_window_update_geometry (self); +} + +void +gst_wl_window_set_fullscreen_height (GstWlWindow * self, gint fullscreen_height) +{ + GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); + + priv->fullscreen_height = fullscreen_height; + + gst_wl_window_update_geometry (self); +} diff --git a/gst-libs/gst/wayland/gstwlwindow.h b/gst-libs/gst/wayland/gstwlwindow.h index 611853bc1c..d3be2f56b8 100644 --- a/gst-libs/gst/wayland/gstwlwindow.h +++ b/gst-libs/gst/wayland/gstwlwindow.h @@ -89,4 +89,16 @@ GST_WL_API void gst_wl_window_set_rotate_method (GstWlWindow *self, GstVideoOrientationMethod rotate_method); +GST_WL_API +void gst_wl_window_set_scale (GstWlWindow * self, gint scale); + +GST_WL_API +guint gst_wl_window_get_scale (GstWlWindow * self); + +GST_WL_API +void gst_wl_window_set_fullscreen_width (GstWlWindow * self, gint fullscreen_width); + +GST_WL_API +void gst_wl_window_set_fullscreen_height (GstWlWindow * self, gint fullscreen_height); + G_END_DECLS -- 2.25.1 From 1607dcd670d5a1b2f16d91dc17989daf2b2058f4 Mon Sep 17 00:00:00 2001 From: Hou Qi Date: Mon, 22 Apr 2024 14:32:48 +0800 Subject: [PATCH 49/65] MMFMWK-9362 waylandsink: correct fps calculation upstream status: imx specific --- ext/wayland/gstwaylandsink.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index 88766b99cd..8b35aa1545 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -1369,7 +1369,8 @@ dst_map_failed: } done: { - self->frame_showed++; + if (ret == GST_FLOW_OK) + self->frame_showed++; g_mutex_unlock (&self->render_lock); return ret; } -- 2.25.1 From 2d0a9ea3d8947aa7cbcef82b2ab52f80779c38b9 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 24 Apr 2024 01:10:36 +0900 Subject: [PATCH 50/65] MMFMWK-9366 wlwindow: using GCond to avoid dropping video buffer This change in 1.24.x will break upstream stage buffer usage in wlwindow, but it can avoid buffer drop cause by decoding speed is much faster than display fps. upstream status: imx specific --- gst-libs/gst/wayland/gstwlwindow.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index a95f947c2f..fc19c65646 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -63,6 +63,9 @@ typedef struct _GstWlWindowPrivate GCond configure_cond; GMutex configure_mutex; + gboolean redraw_pending; + GCond redraw_wait; + struct wl_shell_surface *shell_surface; struct zwp_blending_v1 *blend_func; @@ -387,7 +390,9 @@ gst_wl_window_init (GstWlWindow * self) GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); priv->configured = TRUE; + priv->redraw_pending = FALSE; g_cond_init (&priv->configure_cond); + g_cond_init (&priv->redraw_wait); g_mutex_init (&priv->configure_mutex); g_mutex_init (&priv->window_lock); @@ -412,7 +417,13 @@ gst_wl_window_finalize (GObject * gobject) if (priv->staged_buffer) gst_wl_buffer_unref_buffer (priv->staged_buffer); + g_mutex_lock (&priv->window_lock); + priv->redraw_pending = FALSE; + g_cond_signal (&priv->redraw_wait); + g_mutex_unlock (&priv->window_lock); + g_cond_clear (&priv->configure_cond); + g_cond_clear (&priv->redraw_wait); g_mutex_clear (&priv->configure_mutex); g_mutex_clear (&priv->window_lock); @@ -856,6 +867,8 @@ frame_redraw_callback (void *data, struct wl_callback *callback, uint32_t time) g_mutex_lock (&priv->window_lock); next_buffer = priv->next_buffer = priv->staged_buffer; priv->staged_buffer = NULL; + priv->redraw_pending = FALSE; + g_cond_signal (&priv->redraw_wait); g_mutex_unlock (&priv->window_lock); if (next_buffer || priv->clear_window) @@ -978,6 +991,9 @@ gst_wl_window_render (GstWlWindow * self, GstWlBuffer * buffer, if (G_UNLIKELY (info)) priv->next_video_info = gst_video_info_copy (info); + while (priv->redraw_pending) + g_cond_wait (&priv->redraw_wait, &priv->window_lock); + if (priv->next_buffer && priv->staged_buffer) { GST_LOG_OBJECT (self, "buffer %p dropped (replaced)", priv->staged_buffer); gst_wl_buffer_unref_buffer (priv->staged_buffer); @@ -986,6 +1002,7 @@ gst_wl_window_render (GstWlWindow * self, GstWlBuffer * buffer, if (!priv->next_buffer) { priv->next_buffer = buffer; + priv->redraw_pending = TRUE; priv->commit_callback = gst_wl_display_sync (priv->display, &commit_listener, self); wl_display_flush (gst_wl_display_get_display (priv->display)); -- 2.25.1 From b4d1cc6af803fa10dfecefbd817b21d98595a253 Mon Sep 17 00:00:00 2001 From: Chao Guo Date: Tue, 30 Apr 2024 18:14:20 +0900 Subject: [PATCH 51/65] wlwindow: Add mutex for surface commit to avoid interference between multi-threads When setting video rotation mode or video size during playback, there are multiple threads performing surface commit. Threads will affect each other, causing weston to report errors. Add a mutex commit_lock to protect surface commit tasks of different threads. When a thread commits a video surface, other threads must not commit this surface. upstream status: pending Signed-off-by: Chao Guo --- gst-libs/gst/wayland/gstwlwindow.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index fc19c65646..b18259dd8f 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -95,6 +95,8 @@ typedef struct _GstWlWindowPrivate struct wl_callback *frame_callback; struct wl_callback *commit_callback; + GMutex commit_lock; + /* the coordinate of video crop */ gint src_x, src_y, src_width, src_height; @@ -395,6 +397,7 @@ gst_wl_window_init (GstWlWindow * self) g_cond_init (&priv->redraw_wait); g_mutex_init (&priv->configure_mutex); g_mutex_init (&priv->window_lock); + g_mutex_init (&priv->commit_lock); priv->src_x = 0; priv->src_y = 0; @@ -426,6 +429,7 @@ gst_wl_window_finalize (GObject * gobject) g_cond_clear (&priv->redraw_wait); g_mutex_clear (&priv->configure_mutex); g_mutex_clear (&priv->window_lock); + g_mutex_clear (&priv->commit_lock); if (priv->xdg_toplevel) xdg_toplevel_destroy (priv->xdg_toplevel); @@ -903,6 +907,7 @@ gst_wl_window_commit_buffer (GstWlWindow * self, GstWlBuffer * buffer) gst_wl_window_set_opaque (self, info); } + g_mutex_lock (&priv->commit_lock); if (G_LIKELY (buffer)) { current_gstbuffer = gst_wl_buffer_get_current_gstbuffer (buffer); used_by_compositor = gst_wl_buffer_get_used_by_compositor (buffer); @@ -951,6 +956,7 @@ gst_wl_window_commit_buffer (GstWlWindow * self, GstWlBuffer * buffer) priv->next_video_info = NULL; } + g_mutex_unlock (&priv->commit_lock); } static void @@ -1104,14 +1110,17 @@ gst_wl_window_update_geometry (GstWlWindow * self) return; if (priv->scaled_width != 0) { + g_mutex_lock (&priv->commit_lock); wl_subsurface_set_sync (priv->video_subsurface); gst_wl_window_resize_video_surface (self, TRUE); } wl_surface_commit (priv->area_surface_wrapper); - if (priv->scaled_width != 0) + if (priv->scaled_width != 0) { wl_subsurface_set_desync (priv->video_subsurface); + g_mutex_unlock (&priv->commit_lock); + } } void -- 2.25.1 From cbe8c7b29a493f52a643c83c17b30a55bf15af4c Mon Sep 17 00:00:00 2001 From: Elliot Chen Date: Mon, 20 May 2024 19:10:37 +0900 Subject: [PATCH 52/65] MMFMWK-9380 autovideoconvert: fix double unref For all error cases in the function gst_base_auto_convert_activate_element, the element is also dereferenced. And no need to deference the element again if this function return false. upstream status: accepted https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6865 Signed-off-by: Elliot Chen --- gst/autoconvert/gstbaseautoconvert.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/gst/autoconvert/gstbaseautoconvert.c b/gst/autoconvert/gstbaseautoconvert.c index 4eb181b37f..eecc493056 100644 --- a/gst/autoconvert/gstbaseautoconvert.c +++ b/gst/autoconvert/gstbaseautoconvert.c @@ -869,8 +869,6 @@ gst_base_auto_convert_sink_setcaps (GstBaseAutoConvert * self, GstCaps * caps, if (gst_base_auto_convert_activate_element (self, element, caps)) { res = TRUE; break; - } else { - gst_object_unref (element); } } -- 2.25.1 From f8ad29bcb169bae2d937c1780eddf16d74ff921a Mon Sep 17 00:00:00 2001 From: Elliot Chen Date: Wed, 24 Apr 2024 15:54:04 +0900 Subject: [PATCH 53/65] MMFMWK-9066 gstplay: query seek information again in playing state for live stream upstream status: accept https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6681 Signed-off-by: Elliot Chen --- gst-libs/gst/play/gstplay.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/gst-libs/gst/play/gstplay.c b/gst-libs/gst/play/gstplay.c index 124f043a4d..d998d40994 100644 --- a/gst-libs/gst/play/gstplay.c +++ b/gst-libs/gst/play/gstplay.c @@ -1466,6 +1466,27 @@ state_changed_cb (G_GNUC_UNUSED GstBus * bus, GstMessage * msg, on_duration_changed (self, duration); } } + /* Try to query seek information again for live stream */ + if (self->is_live) { + gboolean seekable = FALSE; + gboolean updated = FALSE; + + GstQuery *query = gst_query_new_seeking (GST_FORMAT_TIME); + if (gst_element_query (self->playbin, query)) + gst_query_parse_seeking (query, NULL, &seekable, NULL, NULL); + gst_query_unref (query); + + g_mutex_lock (&self->lock); + if (self->media_info && seekable != self->media_info->seekable) { + self->media_info->seekable = seekable; + updated = TRUE; + } + g_mutex_unlock (&self->lock); + + if (updated) { + on_media_info_updated (self); + } + } /* api_bus_post_message (self, GST_PLAY_MESSAGE_POSITION_UPDATED, */ /* GST_PLAY_MESSAGE_DATA_POSITION, GST_TYPE_CLOCK_TIME, 0, NULL); */ -- 2.25.1 From 33d38e94dfef1835f644fb635065eb27d726f3a0 Mon Sep 17 00:00:00 2001 From: Elliot Chen Date: Fri, 24 May 2024 12:16:33 +0900 Subject: [PATCH 54/65] MMFMWK-9382 autovideoconvert: should not forward the allocation query if no element is selected upstream status: accept https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6927 Signed-off-by: Elliot Chen --- gst/autoconvert/gstbaseautoconvert.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gst/autoconvert/gstbaseautoconvert.c b/gst/autoconvert/gstbaseautoconvert.c index eecc493056..b928c115fb 100644 --- a/gst/autoconvert/gstbaseautoconvert.c +++ b/gst/autoconvert/gstbaseautoconvert.c @@ -1150,6 +1150,16 @@ gst_base_auto_convert_sink_query (GstPad * pad, GstObject * parent, return ret; } + /* Should not forward the allocation query downstream directly + * if no subelement is selected, otherwise it can influence + * the downstream allocation choices and upstream buffer usage. + */ + if (GST_QUERY_TYPE (query) == GST_QUERY_ALLOCATION) { + GST_DEBUG_OBJECT (self, + "no subelement is selected yet, can't answer ALLOCATION query"); + return FALSE; + } + ignore_acceptcaps_failure: if (GST_QUERY_TYPE (query) == GST_QUERY_ACCEPT_CAPS) { -- 2.25.1 From 0a3b0f58fa8b808615b3536aaafe65b4111eca8c Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 14 Feb 2024 09:43:35 -0300 Subject: [PATCH 55/65] autoconvert: Fix race condition when creating sub elements There was a case where the element would get destroyed while being added to the hash table of elements Part-of: --- gst/autoconvert/gstbaseautoconvert.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gst/autoconvert/gstbaseautoconvert.c b/gst/autoconvert/gstbaseautoconvert.c index b928c115fb..6a9efc4138 100644 --- a/gst/autoconvert/gstbaseautoconvert.c +++ b/gst/autoconvert/gstbaseautoconvert.c @@ -500,13 +500,12 @@ gst_base_auto_convert_add_element (GstBaseAutoConvert * self, g_assert (filter_info->subbin); - - element = filter_info->subbin; + element = gst_object_ref (filter_info->subbin); GST_DEBUG_OBJECT (self, "Adding element %s to the baseautoconvert bin", filter_info->name); pads = internal_pads_new (self, GST_OBJECT_NAME (element)); - g_hash_table_insert (self->elements, element, pads); + g_hash_table_insert (self->elements, element, internal_pads_ref (pads)); gst_pad_set_chain_function (pads->sink, GST_DEBUG_FUNCPTR (gst_base_auto_convert_internal_sink_chain)); @@ -522,7 +521,9 @@ gst_base_auto_convert_add_element (GstBaseAutoConvert * self, gst_pad_set_query_function (pads->src, GST_DEBUG_FUNCPTR (gst_base_auto_convert_internal_src_query)); - return gst_object_ref (element); + internal_pads_unref (pads); + + return element; } static GstElement * -- 2.25.1 From af2dea8331f972f082f97dece41919bba863d6ef Mon Sep 17 00:00:00 2001 From: Elliot Chen Date: Fri, 24 May 2024 11:44:15 +0900 Subject: [PATCH 56/65] MMFMWK-9381 autovideoconvert: select i.MX videoconvert plugins to convert upstream status: imx specific Signed-off-by: Elliot Chen --- gst/autoconvert/gstautovideoconvert.c | 91 +++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/gst/autoconvert/gstautovideoconvert.c b/gst/autoconvert/gstautovideoconvert.c index cf5468316c..037c53b2ba 100644 --- a/gst/autoconvert/gstautovideoconvert.c +++ b/gst/autoconvert/gstautovideoconvert.c @@ -71,6 +71,97 @@ gst_auto_video_convert_init (GstAutoVideoConvert * autovideoconvert) { /* *INDENT-OFF* */ static const GstAutoVideoFilterGenerator gen[] = { + { + .first_elements = { NULL }, + .colorspace_converters = { "imxvideoconvert_g2d", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 2, + }, + { + .first_elements = { NULL }, + .colorspace_converters = { "imxvideoconvert_ipu", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 2, + }, + { + .first_elements = { NULL }, + .colorspace_converters = { "imxvideoconvert_pxp", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 2, + }, + { + .first_elements = { NULL }, + .colorspace_converters = { "imxvideoconvert_ocl", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 2, + }, + { + .first_elements = { NULL }, + .colorspace_converters = {"imxvideoconvert_g2d", "imxvideoconvert_ipu", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 1, + }, + { + .first_elements = { NULL }, + .colorspace_converters = {"imxvideoconvert_g2d", "imxvideoconvert_pxp", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 1, + }, + { + .first_elements = { NULL }, + .colorspace_converters = {"imxvideoconvert_g2d", "imxvideoconvert_ocl", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 1, + }, + { + .first_elements = { NULL }, + .colorspace_converters = {"imxvideoconvert_ipu", "imxvideoconvert_g2d", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 1, + }, + { + .first_elements = { NULL }, + .colorspace_converters = {"imxvideoconvert_ipu", "imxvideoconvert_pxp", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 1, + }, + { + .first_elements = { NULL }, + .colorspace_converters = {"imxvideoconvert_pxp", "imxvideoconvert_g2d", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 1, + }, + { + .first_elements = { NULL }, + .colorspace_converters = {"imxvideoconvert_pxp", "imxvideoconvert_ipu", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 1, + }, + { + .first_elements = { NULL }, + .colorspace_converters = {"imxvideoconvert_ocl", "imxvideoconvert_g2d", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 1, + }, + { + .first_elements = { NULL }, + .colorspace_converters = {"imxvideoconvert_ocl", "imxvideoconvert_pxp", NULL }, + .last_elements = { NULL }, + .filters = { NULL }, + .rank = GST_RANK_PRIMARY + 1, + }, { .first_elements = { "bayer2rgb", NULL}, .colorspace_converters = { "videoconvertscale", NULL }, -- 2.25.1 From e2b5bfcc556cbce79d57832cdc9c5856ba9eeb6c Mon Sep 17 00:00:00 2001 From: Chao Guo Date: Fri, 31 May 2024 11:26:32 +0900 Subject: [PATCH 57/65] LF-12564 wlwindow: fix the issue of video hang after resizing by mouse When resizing video by mouse, there are two threads (render/display thread) that may wait for each other to release the mutex (render_lock/sync_mutex) they hold, which will cause the video pipeline to hang. Due to the update of gstreamer, the 'surface_commit' in the render thread is migrated to the display thread, so delete the render_lock in the display thread, so that the render thread does not need to wait for the display thread. Signed-off-by: Chao Guo --- gst-libs/gst/wayland/gstwlwindow.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/gst-libs/gst/wayland/gstwlwindow.c b/gst-libs/gst/wayland/gstwlwindow.c index b18259dd8f..8d0dd2ee0c 100644 --- a/gst-libs/gst/wayland/gstwlwindow.c +++ b/gst-libs/gst/wayland/gstwlwindow.c @@ -252,7 +252,6 @@ handle_xdg_toplevel_configure (void *data, struct xdg_toplevel *xdg_toplevel, int32_t width, int32_t height, struct wl_array *states) { GstWlWindow *self = data; - GstWlWindowPrivate *priv = gst_wl_window_get_instance_private (self); const uint32_t *state; GST_DEBUG ("XDG toplevel got a \"configure\" event, [ %d, %d ].", @@ -271,9 +270,7 @@ handle_xdg_toplevel_configure (void *data, struct xdg_toplevel *xdg_toplevel, if (width <= 2 * RESIZE_MARGIN || height <= 2 * RESIZE_MARGIN) return; - g_mutex_lock (priv->render_lock); gst_wl_window_set_render_rectangle (self, 0, 0, width, height); - g_mutex_unlock (priv->render_lock); } static const struct xdg_toplevel_listener xdg_toplevel_listener = { -- 2.25.1 From cef0d2869e42c30660dc2d1c3eaddfbe09c17892 Mon Sep 17 00:00:00 2001 From: Elliot Chen Date: Tue, 28 May 2024 19:48:12 +0900 Subject: [PATCH 58/65] MMFMWK-9385 autovideoconvert: need check caps negotiation when selecting elements upstream status: pending https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6949 Signed-off-by: Elliot Chen --- gst/autoconvert/gstbaseautoconvert.c | 72 ++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/gst/autoconvert/gstbaseautoconvert.c b/gst/autoconvert/gstbaseautoconvert.c index 6a9efc4138..5b0c691e8b 100644 --- a/gst/autoconvert/gstbaseautoconvert.c +++ b/gst/autoconvert/gstbaseautoconvert.c @@ -104,6 +104,9 @@ static gboolean gst_base_auto_convert_internal_src_query (GstPad * pad, GstObject * parent, GstQuery * query); static GList *gst_base_auto_convert_get_or_load_filters_info (GstBaseAutoConvert * self); +static gboolean +gst_base_auto_convert_check_negotiation (GstBaseAutoConvert * self, + GstCaps * caps, GstEvent * caps_event); G_DECLARE_FINAL_TYPE (GstBaseAutoConvertPad, gst_base_auto_convert_pad, GST, BASE_AUTO_CONVERT_PAD, GstPad); @@ -598,7 +601,7 @@ gst_base_auto_convert_get_element_internal_pads (GstBaseAutoConvert * self, static gboolean gst_base_auto_convert_activate_element (GstBaseAutoConvert * self, - GstElement * element, GstCaps * caps) + GstElement * element, GstCaps * caps, GstEvent * caps_event) { gboolean res = TRUE; GstPad *sinkpad = NULL, *srcpad = NULL; @@ -685,6 +688,13 @@ gst_base_auto_convert_activate_element (GstBaseAutoConvert * self, gst_pad_sticky_events_foreach (self->sinkpad, sticky_event_push, self); + /* Should check caps negotiation result if we can really use it */ + if (caps_event && + !gst_base_auto_convert_check_negotiation (self, caps, caps_event)) { + GST_DEBUG_OBJECT (element, "Check caps negotiation failed"); + goto error; + } + gst_pad_push_event (self->sinkpad, gst_event_new_reconfigure ()); GST_INFO_OBJECT (self, "Selected element %s", @@ -754,6 +764,47 @@ gst_auto_convert_get_filter_info (GstBaseAutoConvert * self, return NULL; } +static gboolean +gst_base_auto_convert_check_negotiation (GstBaseAutoConvert * self, + GstCaps * caps, GstEvent * caps_event) +{ + GstPad *internal_srcpad; + GstPad *internal_sinkpad; + gboolean ret = FALSE; + + internal_srcpad = gst_base_auto_convert_get_internal_srcpad (self); + internal_sinkpad = gst_base_auto_convert_get_internal_sinkpad (self); + if (!internal_srcpad || !internal_sinkpad) { + GST_DEBUG_OBJECT (self, "Could not get internal src or sink pad"); + goto done; + } + + /* Send caps event to trigger caps negotiation + * and get the internal sink caps to check caps + * negotiation result + */ + if (!gst_pad_push_event (internal_srcpad, gst_event_ref (caps_event))) { + GST_DEBUG_OBJECT (self, "Sending caps event failed, %s:%s", + GST_DEBUG_PAD_NAME (internal_srcpad)); + goto done; + } + + if (!gst_pad_get_current_caps (internal_sinkpad)) { + GST_DEBUG_OBJECT (self, "Could not get caps, %s:%s", + GST_DEBUG_PAD_NAME (internal_sinkpad)); + } else { + GST_DEBUG_OBJECT (self, "check caps negotiation success"); + ret = TRUE; + } + +done: + if (internal_srcpad) + gst_object_unref (internal_srcpad); + if (internal_sinkpad) + gst_object_unref (internal_sinkpad); + return ret; +} + /* * If there is already an internal element, it will try to call set_caps on it * @@ -764,7 +815,7 @@ gst_auto_convert_get_filter_info (GstBaseAutoConvert * self, static gboolean gst_base_auto_convert_sink_setcaps (GstBaseAutoConvert * self, GstCaps * caps, - gboolean check_downstream) + gboolean check_downstream, GstEvent * caps_event) { GList *tmp; GstCaps *other_caps = NULL; @@ -811,6 +862,9 @@ gst_base_auto_convert_sink_setcaps (GstBaseAutoConvert * self, GstCaps * caps, GST_DEBUG_OBJECT (self, "Filter %s can intersect", filter_info->name); } + if (res && caps_event) + res = gst_base_auto_convert_check_negotiation (self, caps, caps_event); + if (res) { /* If we can set the new caps on the current element, * then we just get out @@ -867,7 +921,8 @@ gst_base_auto_convert_sink_setcaps (GstBaseAutoConvert * self, GstCaps * caps, continue; /* And make it the current child */ - if (gst_base_auto_convert_activate_element (self, element, caps)) { + if (gst_base_auto_convert_activate_element (self, element, caps, + caps_event)) { res = TRUE; break; } @@ -1027,7 +1082,8 @@ gst_base_auto_convert_sink_chain (GstPad * pad, GstObject * parent, GST_INFO_OBJECT (parent, "Needs reconfigure."); /* if we need to reconfigure we pretend new caps arrived. This * will reconfigure the transform with the new output format. */ - if (sinkcaps && !gst_base_auto_convert_sink_setcaps (self, sinkcaps, TRUE)) { + if (sinkcaps + && !gst_base_auto_convert_sink_setcaps (self, sinkcaps, TRUE, NULL)) { gst_clear_caps (&sinkcaps); GST_ERROR_OBJECT (self, "Could not reconfigure."); @@ -1085,11 +1141,9 @@ gst_base_auto_convert_sink_event (GstPad * pad, GstObject * parent, GstCaps *caps; gst_event_parse_caps (event, &caps); - ret = gst_base_auto_convert_sink_setcaps (self, caps, FALSE); - if (!ret) { - gst_event_unref (event); - return ret; - } + ret = gst_base_auto_convert_sink_setcaps (self, caps, FALSE, event); + gst_event_unref (event); + return ret; } internal_srcpad = gst_base_auto_convert_get_internal_srcpad (self); -- 2.25.1 From 4842f0529bcbce5abdf6f53b755cf212fa112ccd Mon Sep 17 00:00:00 2001 From: Hou Qi Date: Wed, 28 Aug 2024 09:09:01 +0800 Subject: [PATCH 59/65] MMFMWK-9409 gstplay: fix one un-seekable aac stream loop playback random hang When gstplay handle eos_cb, it first posts message to gplay2, then change state to stopped. When gplay2 parses this eos bus message, it sets rate to 1 which causes seek_position is set to 0:00:07.424000000, then stop_sync of current item and play_sync of next item. There is chance that gstplay receives playing_to_paused bus message owing to timing issue between gstplay change_state and gplay2 stop_sync. It triggers seek then sends error bus message. stop_sync->start_sync->stop_sync causes urisourcebin and typefind deadlock. To avoid this issue, only seek if stream is seekable when state is changing playing_to_paused. upstream status: accepted part-of: --- gst-libs/gst/play/gstplay.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/play/gstplay.c b/gst-libs/gst/play/gstplay.c index d998d40994..8061882aeb 100644 --- a/gst-libs/gst/play/gstplay.c +++ b/gst-libs/gst/play/gstplay.c @@ -1433,8 +1433,12 @@ state_changed_cb (G_GNUC_UNUSED GstBus * bus, GstMessage * msg, } if (self->seek_position != GST_CLOCK_TIME_NONE) { - GST_DEBUG_OBJECT (self, "Seeking now that we reached PAUSED state"); - gst_play_seek_internal_locked (self); + if (!self->media_info->seekable) { + GST_DEBUG_OBJECT (self, "Media is not seekable"); + } else { + GST_DEBUG_OBJECT (self, "Seeking now that we reached PAUSED state"); + gst_play_seek_internal_locked (self); + } g_mutex_unlock (&self->lock); } else if (!self->seek_pending) { g_mutex_unlock (&self->lock); -- 2.25.1 From 886180323aa80d7cf373bcaa2cb386dce916f38a Mon Sep 17 00:00:00 2001 From: Hou Qi Date: Tue, 20 Aug 2024 08:52:23 +0800 Subject: [PATCH 60/65] LF-13224 jpegparse: Implement ::get_sink_caps vfunc to propagate downstream caps constraints upstream upstream status: pending Part-of: --- gst/jpegformat/gstjpegparse.c | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/gst/jpegformat/gstjpegparse.c b/gst/jpegformat/gstjpegparse.c index 7a79d4bc9d..0da6bf4ce6 100644 --- a/gst/jpegformat/gstjpegparse.c +++ b/gst/jpegformat/gstjpegparse.c @@ -99,6 +99,8 @@ gst_jpeg_parse_handle_frame (GstBaseParse * bparse, GstBaseParseFrame * frame, gint * skipsize); static gboolean gst_jpeg_parse_set_sink_caps (GstBaseParse * parse, GstCaps * caps); +static GstCaps *gst_jpeg_parse_get_sink_caps (GstBaseParse * parse, + GstCaps * caps); static gboolean gst_jpeg_parse_sink_event (GstBaseParse * parse, GstEvent * event); static gboolean gst_jpeg_parse_start (GstBaseParse * parse); @@ -167,6 +169,7 @@ gst_jpeg_parse_class_init (GstJpegParseClass * klass) gstbaseparse_class->start = gst_jpeg_parse_start; gstbaseparse_class->stop = gst_jpeg_parse_stop; gstbaseparse_class->set_sink_caps = gst_jpeg_parse_set_sink_caps; + gstbaseparse_class->get_sink_caps = gst_jpeg_parse_get_sink_caps; gstbaseparse_class->sink_event = gst_jpeg_parse_sink_event; gstbaseparse_class->handle_frame = gst_jpeg_parse_handle_frame; @@ -202,6 +205,58 @@ parse_avid (GstJpegParse * parse, const guint8 * data, guint16 len) gst_video_field_order_to_string (parse->field_order)); } +static void +remove_fields (GstCaps * caps) +{ + guint i, n; + + n = gst_caps_get_size (caps); + for (i = 0; i < n; i++) { + GstStructure *s = gst_caps_get_structure (caps, i); + + gst_structure_remove_fields (s, "parsed", "colorimetry", NULL); + } +} + +static GstCaps * +gst_jpeg_parse_get_sink_caps (GstBaseParse * bparse, GstCaps * filter) +{ + GstJpegParse *parse = GST_JPEG_PARSE_CAST (bparse); + GstCaps *peercaps, *templ; + GstCaps *res; + + templ = gst_pad_get_pad_template_caps (GST_BASE_PARSE_SINK_PAD (bparse)); + if (filter) { + GstCaps *fcopy = gst_caps_copy (filter); + remove_fields (fcopy); + peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (bparse), fcopy); + gst_caps_unref (fcopy); + } else { + peercaps = gst_pad_peer_query_caps (GST_BASE_PARSE_SRC_PAD (bparse), NULL); + } + + if (peercaps) { + peercaps = gst_caps_make_writable (peercaps); + remove_fields (peercaps); + res = gst_caps_intersect_full (peercaps, templ, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (templ); + gst_caps_unref (peercaps); + } else { + res = templ; + } + + if (filter) { + GstCaps *tmp = gst_caps_intersect_full (filter, res, + GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (res); + res = tmp; + } + + GST_DEBUG_OBJECT (parse, "returning sink caps %" GST_PTR_FORMAT, res); + + return res; +} + static gboolean gst_jpeg_parse_set_sink_caps (GstBaseParse * bparse, GstCaps * caps) { -- 2.25.1 From 3c7aa8dd2cee634772824a4229e7c3e07a8b57e9 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Thu, 26 Sep 2024 17:31:58 +0900 Subject: [PATCH 61/65] Add SCR.txt and LICENSE.txt for gstreamer 1.24.7 --- LICENSE.txt | 503 ++++++++++++++++++++++++++++++++++++++++++++++++++++ SCR.txt | 9 + 2 files changed, 512 insertions(+) create mode 100644 LICENSE.txt create mode 100644 SCR.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000000..efce2a87c3 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,503 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + diff --git a/SCR.txt b/SCR.txt new file mode 100644 index 0000000000..2efa2b58f2 --- /dev/null +++ b/SCR.txt @@ -0,0 +1,9 @@ +Package: gst-plugins-bad.git +Version: 1.24.7.imx +Outgoing License: LGPL-2.1 +License File: LICENSE.txt +Type of Content: source +Description and comments: Open Source Multimedia Farmework +Release Location: https://github.com/nxp-imx/gstreamer -b lf-6.6.36-2.1.0 +Origin: NXP (LGPL-2.1) + GStreamer (LGPL-2.1+) - http://gstreamer.freedesktop.org/src/gstreamer/ -- 2.25.1 From 983dc335b129de5e8ade97cf93644da8d706e47c Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Tue, 8 Oct 2024 11:22:55 +0900 Subject: [PATCH 62/65] LF-13498 wllinuxdmabuf: change timeout from 1s to 10s current implementation put the wl_dmabuf creation outside the wldisplay thread, but wldisplay thread maybe blocked by fence wait or something else. It easily cause timeout, so change timeout from 1s to 10s. upstream status: imx specific --- gst-libs/gst/wayland/gstwllinuxdmabuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/wayland/gstwllinuxdmabuf.c b/gst-libs/gst/wayland/gstwllinuxdmabuf.c index 1e0dc02548..108d91e06f 100644 --- a/gst-libs/gst/wayland/gstwllinuxdmabuf.c +++ b/gst-libs/gst/wayland/gstwllinuxdmabuf.c @@ -211,7 +211,7 @@ gst_wl_linux_dmabuf_construct_wl_buffer (GstBuffer * buf, /* Wait for the request answer */ wl_display_flush (gst_wl_display_get_display (display)); data.wbuf = (gpointer) 0x1; - timeout = g_get_monotonic_time () + G_TIME_SPAN_SECOND; + timeout = g_get_monotonic_time () + 10 * G_TIME_SPAN_SECOND; while (data.wbuf == (gpointer) 0x1) { if (!g_cond_wait_until (&data.cond, &data.lock, timeout)) { GST_ERROR_OBJECT (mem->allocator, "zwp_linux_buffer_params_v1 time out"); -- 2.25.1 From cdf213ef6ae3470eb0d95539043c166b8f276bbc Mon Sep 17 00:00:00 2001 From: Hou Qi Date: Thu, 17 Oct 2024 13:59:53 +0800 Subject: [PATCH 63/65] Update release branch in SCR.txt to lf-6.6.52-2.2.0 --- SCR.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SCR.txt b/SCR.txt index 2efa2b58f2..6e0145c059 100644 --- a/SCR.txt +++ b/SCR.txt @@ -4,6 +4,6 @@ Outgoing License: LGPL-2.1 License File: LICENSE.txt Type of Content: source Description and comments: Open Source Multimedia Farmework -Release Location: https://github.com/nxp-imx/gstreamer -b lf-6.6.36-2.1.0 +Release Location: https://github.com/nxp-imx/gstreamer -b lf-6.6.52-2.2.0 Origin: NXP (LGPL-2.1) GStreamer (LGPL-2.1+) - http://gstreamer.freedesktop.org/src/gstreamer/ -- 2.25.1 From bba296b75dfb3e7ae4c836255edbd2b3c085c269 Mon Sep 17 00:00:00 2001 From: Hou Qi Date: Fri, 18 Oct 2024 14:01:40 +0800 Subject: [PATCH 64/65] MMFMWK-9417 waylandsink: create shm buffer without padding When using wrong weston.ini "use-g2d=1" instead of "use-g2d=TRUE", waylandsink will create internal buffer pool. In order to avoid video-info calculated width and padding width mismatch, no need to add width and height padding when creating shm buffer pool. upstream status: imx specific --- ext/wayland/gstwaylandsink.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/wayland/gstwaylandsink.c b/ext/wayland/gstwaylandsink.c index 8b35aa1545..1a6778db18 100644 --- a/ext/wayland/gstwaylandsink.c +++ b/ext/wayland/gstwaylandsink.c @@ -708,6 +708,7 @@ gst_wayland_update_pool (GstWaylandSink * self, GstAllocator * allocator) gint w = GST_VIDEO_INFO_WIDTH (&self->video_info); gint h = GST_VIDEO_INFO_HEIGHT (&self->video_info); GstStructure *config; + gboolean is_shm = GST_IS_SHM_ALLOCATOR (allocator); /* Pools with outstanding buffer cannot be reconfigured, so we must use * a new pool. */ @@ -722,7 +723,7 @@ gst_wayland_update_pool (GstWaylandSink * self, GstAllocator * allocator) gst_buffer_pool_config_set_params (config, self->caps, size, 2, 0); gst_buffer_pool_config_set_allocator (config, allocator, NULL); - if (!ISALIGNED (w, 16) || !ISALIGNED (h, 16)) { + if (!is_shm && (!ISALIGNED (w, 16) || !ISALIGNED (h, 16))) { GstVideoAlignment alignment; memset (&alignment, 0, sizeof (GstVideoAlignment)); -- 2.25.1 From f620551789344a612803626901d28720f8e3cab2 Mon Sep 17 00:00:00 2001 From: Akshaya Maran Date: Tue, 18 Nov 2025 10:50:25 +0530 Subject: [PATCH 65/65] gst-plugins-bad: Fix declaration after label build issue --- gst/autoselect/gstautoselect.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gst/autoselect/gstautoselect.c b/gst/autoselect/gstautoselect.c index 9c9b847218..8572c61941 100755 --- a/gst/autoselect/gstautoselect.c +++ b/gst/autoselect/gstautoselect.c @@ -324,6 +324,8 @@ gst_auto_select_set_property (GObject * object, { GstAutoSelect *autoselect = GST_AUTO_SELECT (object); GstState cur_state; + GList *old_factories; + GList *new_factories; gst_element_get_state (GST_ELEMENT (autoselect), &cur_state, NULL, 0); if (cur_state == GST_STATE_NULL) { @@ -332,8 +334,8 @@ gst_auto_select_set_property (GObject * object, G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; case PROP_FACTORIES: - GList * old_factories = g_atomic_pointer_get (&autoselect->factories); - GList *new_factories = g_value_get_pointer (value); + old_factories = g_atomic_pointer_get (&autoselect->factories); + new_factories = g_value_get_pointer (value); new_factories = g_list_copy (new_factories); if (g_atomic_pointer_compare_and_exchange (&autoselect->factories, -- 2.25.1