From db03767b33192d573e7da70273d0e248aa6776f5 Mon Sep 17 00:00:00 2001
From: Mingke Wang <mingke.wang@freescale.com>
Date: Fri, 16 Oct 2015 19:31:32 +0800
Subject: [PATCH 01/93] basetextoverlay: make memory copy when video buffer's
 memory is ready only

1. since gst_buffer_make_writable just lookup the refcount to determine if
   a buffer is writable, and it will use _gst_buffer_copy() which don't
   perform a deep memory copy even if the flag of a memory is set to
   GST_MEMORY_FLAG_READONLY. So, we detect the memory flag and use
   gst_buffer_copy_region with GST_BUFFER_COPY_DEEP parameter to perform
   deep memory copy. if the allocator of a memory don't support mem_copy
   interface, the it will return NULL, if this case, we can use
   gst_buffer_make_writable() to get a shared memory buffer or the orignal
   buffer if the buffer's refcount is 1.

Signed-off-by: Mingke Wang <mingke.wang@freescale.com>
(cherry picked from commit dee3fd559f856526c49f35ed0498534bf19527d3)
---
 ext/pango/gstbasetextoverlay.c | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/ext/pango/gstbasetextoverlay.c b/ext/pango/gstbasetextoverlay.c
index 1000ec8a0..08c6b97c0 100644
--- a/ext/pango/gstbasetextoverlay.c
+++ b/ext/pango/gstbasetextoverlay.c
@@ -2316,16 +2316,44 @@ gst_base_text_overlay_push_frame (GstBaseTextOverlay * overlay,
     }
   }
 
-  video_frame = gst_buffer_make_writable (video_frame);
-
   if (overlay->attach_compo_to_buffer) {
     GST_DEBUG_OBJECT (overlay, "Attaching text overlay image to video buffer");
+    video_frame = gst_buffer_make_writable (video_frame);
     gst_buffer_add_video_overlay_composition_meta (video_frame,
         overlay->composition);
     /* FIXME: emulate shaded background box if want_shading=true */
     goto done;
   }
 
+  gint idx = 0;
+  gboolean mem_rdonly = FALSE;
+  GstMemory *mem;
+
+  while (mem = gst_buffer_get_memory(video_frame, idx++)) {
+    if (GST_MEMORY_IS_READONLY(mem)) {
+      gst_memory_unref (mem);
+      mem_rdonly = TRUE;
+      break;
+    }
+    gst_memory_unref (mem);
+  }
+
+  if (mem_rdonly) {
+    GstBuffer *new_buf = gst_buffer_copy_region (video_frame,
+        GST_BUFFER_COPY_ALL | GST_BUFFER_COPY_DEEP, 0, -1);
+
+    if (!new_buf) {
+      GST_WARNING_OBJECT(overlay,
+                "buffer memory read only, but copy memory failed");
+      goto done;
+    } else {
+      gst_buffer_unref (video_frame);
+      video_frame = new_buf;
+    }
+  } else {
+    video_frame = gst_buffer_make_writable (video_frame);
+  }
+
   if (!gst_video_frame_map (&frame, &overlay->info, video_frame,
           GST_MAP_READWRITE))
     goto invalid_frame;
-- 
2.34.1


From 64fbf213afc4d80d3e1a961cf2ec7a67bb23e1a1 Mon Sep 17 00:00:00 2001
From: Mingke Wang <mingke.wang@freescale.com>
Date: Thu, 19 Mar 2015 14:15:25 +0800
Subject: [PATCH 02/93] gstplaysink: don't set async of custom text-sink to
 false

set async to false lead to A/V sync problem when seeking.
the preroll need use GAP event instead of set async to false.

Upstream Status: Inappropriate [i.MX specific]

Signed-off-by: Mingke Wang <mingke.wang@freescale.com>
(cherry picked from commit 8ec4cab5637f44baaf4a2c39bb308db29c9793d9)
---
 gst/playback/gstplaysink.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gst/playback/gstplaysink.c b/gst/playback/gstplaysink.c
index f4d7a0f8e..f2dc6168b 100644
--- a/gst/playback/gstplaysink.c
+++ b/gst/playback/gstplaysink.c
@@ -2485,7 +2485,7 @@ gen_text_chain (GstPlaySink * playsink)
           G_TYPE_BOOLEAN);
       if (elem) {
         /* make sure the sparse subtitles don't participate in the preroll */
-        g_object_set (elem, "async", FALSE, NULL);
+        //g_object_set (elem, "async", FALSE, NULL);
         GST_DEBUG_OBJECT (playsink, "adding custom text sink");
         gst_bin_add (bin, chain->sink);
         /* NOTE streamsynchronizer needs streams decoupled */
-- 
2.34.1


From bcb366c9c42bf347f63ba1753ba13e2fab4c8617 Mon Sep 17 00:00:00 2001
From: Mingke Wang <mingke.wang@freescale.com>
Date: Thu, 19 Mar 2015 14:17:10 +0800
Subject: [PATCH 03/93] ssaparse: enhance SSA text lines parsing.

some parser will pass in the original ssa text line which starts with "Dialog:"
and there's are maybe multiple Dialog lines in one input buffer.

Upstream Status: Inappropriate [i.MX specific]

Signed-off-by: Mingke Wang <mingke.wang@freescale.com>
(cherry picked from commit 3fa862b08a6eaca29776e7616afb2473a6e792a3)
---
 gst/subparse/gstssaparse.c | 150 +++++++++++++++++++++++++++++++++----
 1 file changed, 134 insertions(+), 16 deletions(-)

diff --git a/gst/subparse/gstssaparse.c b/gst/subparse/gstssaparse.c
index 42fbb42b9..2562c7b5b 100644
--- a/gst/subparse/gstssaparse.c
+++ b/gst/subparse/gstssaparse.c
@@ -270,6 +270,7 @@ gst_ssa_parse_remove_override_codes (GstSsaParse * parse, gchar * txt)
  * gst_ssa_parse_push_line:
  * @parse: caller element
  * @txt: text to push
+ * @size: text size need to be parse
  * @start: timestamp for the buffer
  * @duration: duration for the buffer
  *
@@ -279,27 +280,133 @@ gst_ssa_parse_remove_override_codes (GstSsaParse * parse, gchar * txt)
  * Returns: result of the push of the created buffer
  */
 static GstFlowReturn
-gst_ssa_parse_push_line (GstSsaParse * parse, gchar * txt,
+gst_ssa_parse_push_line (GstSsaParse * parse, gchar * txt, gint size,
     GstClockTime start, GstClockTime duration)
 {
   GstFlowReturn ret;
   GstBuffer *buf;
-  gchar *t, *escaped;
+  gchar *t, *text, *p, *escaped, *p_start, *p_end;
   gint num, i, len;
+  GstClockTime start_time = G_MAXUINT64, end_time = 0;
 
-  num = atoi (txt);
-  GST_LOG_OBJECT (parse, "Parsing line #%d at %" GST_TIME_FORMAT,
-      num, GST_TIME_ARGS (start));
-
-  /* skip all non-text fields before the actual text */
+  p = text = g_malloc(size + 1);
+  *p = '\0';
   t = txt;
-  for (i = 0; i < 8; ++i) {
-    t = strchr (t, ',');
+
+  /* there are may have multiple dialogue lines at a time */
+  while (*t) {
+    /* ignore leading white space characters */
+    while (isspace(*t))
+      t++;
+
+    /* ignore Format: and Style: lines */
+    if (strncmp(t, "Format:", 7) == 0 || strncmp(t, "Style:", 6) == 0) {
+      while (*t != '\0' && *t != '\n') {
+        t++;
+      }
+    }
+
+    if (*t == '\0')
+      break;
+
+    /* continue with next line */
+    if (*t == '\n') {
+      t++;
+      continue;
+    }
+
+    if(strncmp(t, "Dialogue:", 9) != 0) {
+      /* not started with "Dialogue:", it must be a line trimmed by demuxer */
+      num = atoi (t);
+      GST_LOG_OBJECT (parse, "Parsing line #%d at %" GST_TIME_FORMAT,
+          num, GST_TIME_ARGS (start));
+
+      /* skip all non-text fields before the actual text */
+      for (i = 0; i < 8; ++i) {
+        t = strchr (t, ',');
+        if (t == NULL)
+          break;
+        ++t;
+      }
+    } else {
+      /* started with "Dialogue:", update timestamp and duration */
+      /* time format are like Dialog:Mark,0:00:01.02,0:00:03.04,xx,xxx,... */
+      guint hour, min, sec, msec;
+      GstClockTime tmp;
+      gchar t_str[12] = {0};
+
+      /* find the first ',' */
+      p_start = strchr (t, ',');
+      if (p_start)
+        p_end = strchr (++p_start, ',');
+
+      if (p_start && p_end) {
+        /* copy text between first ',' and second ',' */
+        strncpy(t_str, p_start, p_end - p_start);
+        if (sscanf (t_str, "%u:%u:%u.%u", &hour, &min, &sec, &msec) == 4) {
+          tmp = ((hour*3600) + (min*60) + sec) * GST_SECOND + msec*GST_MSECOND;
+          GST_DEBUG_OBJECT (parse, "Get start time:%02d:%02d:%02d:%03d\n",
+              hour, min, sec, msec);
+          if (start_time > tmp)
+            start_time = tmp;
+        } else {
+          GST_WARNING_OBJECT (parse,
+              "failed to parse ssa start timestamp string :%s", t_str);
+        }
+
+        p_start = p_end;
+        p_end = strchr (++p_start, ',');
+        if (p_end) {
+          /* copy text between second ',' and third ',' */
+          strncpy(t_str, p_start, p_end - p_start);
+          if (sscanf (t_str, "%u:%u:%u.%u", &hour, &min, &sec, &msec) == 4) {
+            tmp = ((hour*3600) + (min*60) + sec)*GST_SECOND + msec*GST_MSECOND;
+            GST_DEBUG_OBJECT(parse, "Get end time:%02d:%02d:%02d:%03d\n",
+                hour, min, sec, msec);
+            if (end_time < tmp)
+              end_time = tmp;
+          } else {
+            GST_WARNING_OBJECT (parse,
+                "failed to parse ssa end timestamp string :%s", t_str);
+          }
+        }
+      }
+
+      /* now skip all non-text fields before the actual text */
+      for (i = 0; i <= 8; ++i) {
+        t = strchr (t, ',');
+        if (t == NULL)
+          break;
+        ++t;
+      }
+    }
+
+    /* line end before expected number of ',', not a Dialogue line */
     if (t == NULL)
-      return GST_FLOW_ERROR;
-    ++t;
+      break;
+
+    /* if not the first line, and the last character of previous line is '\0',
+     * then replace it with '\N' */
+    if (p != text && *p == '\0') {
+      *p++ = '\\';
+      *p++ = 'N';
+    }
+
+    /* copy all actual text of this line */
+    while ((*t != '\0') && (*t != '\n'))
+      *p++ = *t++;
+
+    /* add a terminator at the end */
+    *p = '\0';
+  }
+
+  /* not valid text found in this buffer return OK to let caller unref buffer */
+  if (strlen(text) <= 0) {
+    GST_WARNING_OBJECT (parse, "Not valid text found in this buffer\n");
+    return GST_FLOW_ERROR;
   }
 
+  t = text;
   GST_LOG_OBJECT (parse, "Text : %s", t);
 
   if (gst_ssa_parse_remove_override_codes (parse, t)) {
@@ -317,13 +424,22 @@ gst_ssa_parse_push_line (GstSsaParse * parse, gchar * txt,
   gst_buffer_fill (buf, 0, escaped, len + 1);
   gst_buffer_set_size (buf, len);
   g_free (escaped);
+  g_free(t);
+
+  if (start_time != G_MAXUINT64)
+    GST_BUFFER_TIMESTAMP (buf) = start_time;
+  else
+    GST_BUFFER_TIMESTAMP (buf) = start;
 
-  GST_BUFFER_TIMESTAMP (buf) = start;
-  GST_BUFFER_DURATION (buf) = duration;
+  if (end_time > start_time)
+    GST_BUFFER_DURATION (buf) = end_time - start_time;
+  else
+    GST_BUFFER_DURATION (buf) = duration;
 
   GST_LOG_OBJECT (parse, "Pushing buffer with timestamp %" GST_TIME_FORMAT
-      " and duration %" GST_TIME_FORMAT, GST_TIME_ARGS (start),
-      GST_TIME_ARGS (duration));
+      " and duration %" GST_TIME_FORMAT,
+      GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf)),
+      GST_TIME_ARGS (GST_BUFFER_DURATION (buf)));
 
   ret = gst_pad_push (parse->srcpad, buf);
 
@@ -343,6 +459,7 @@ gst_ssa_parse_chain (GstPad * sinkpad, GstObject * parent, GstBuffer * buf)
   GstClockTime ts;
   gchar *txt;
   GstMapInfo map;
+  gint size;
 
   if (G_UNLIKELY (!parse->framed))
     goto not_framed;
@@ -360,13 +477,14 @@ gst_ssa_parse_chain (GstPad * sinkpad, GstObject * parent, GstBuffer * buf)
   /* make double-sure it's 0-terminated and all */
   gst_buffer_map (buf, &map, GST_MAP_READ);
   txt = g_strndup ((gchar *) map.data, map.size);
+  size = map.size;
   gst_buffer_unmap (buf, &map);
 
   if (txt == NULL)
     goto empty_text;
 
   ts = GST_BUFFER_TIMESTAMP (buf);
-  ret = gst_ssa_parse_push_line (parse, txt, ts, GST_BUFFER_DURATION (buf));
+  ret = gst_ssa_parse_push_line (parse, txt, size, ts, GST_BUFFER_DURATION (buf));
 
   if (ret != GST_FLOW_OK && GST_CLOCK_TIME_IS_VALID (ts)) {
     GstSegment segment;
-- 
2.34.1


From 4dd5c6d2f435e2c6eaac6fe0b10bc23dd270c005 Mon Sep 17 00:00:00 2001
From: Mingke Wang <mingke.wang@freescale.com>
Date: Thu, 19 Mar 2015 14:20:26 +0800
Subject: [PATCH 04/93] subparse: set need_segment after sink pad received
 GST_EVENT_SEGMENT

subparse works in push mode, chain funciton will be called once
up stream element finished the seeking and flushing.
if set need_segment flag in src pad event handler, the segment
event will be pushed earlier, result in the subtitle text will
be send out to down stream from the beginning.

Upstream Status: Inappropriate [i.MX specific]

Signed-off-by: Mingke Wang <mingke.wang@freescale.com>
(cherry picked from commit 8454a4d27fb2bc6f7bfac7fd2fb2ebc62de4cb9e)
---
 gst/subparse/gstsubparse.c | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/gst/subparse/gstsubparse.c b/gst/subparse/gstsubparse.c
index 994cf62d1..995a416bd 100644
--- a/gst/subparse/gstsubparse.c
+++ b/gst/subparse/gstsubparse.c
@@ -273,24 +273,20 @@ gst_sub_parse_src_event (GstPad * pad, GstObject * parent, GstEvent * event)
         goto beach;
       }
 
+      /* Apply the seek to our segment */
+      gst_segment_do_seek (&self->segment, rate, format, flags,
+          start_type, start, stop_type, stop, &update);
+
+      GST_DEBUG_OBJECT (self, "segment after seek: %" GST_SEGMENT_FORMAT,
+          &self->segment);
+
       /* Convert that seek to a seeking in bytes at position 0,
          FIXME: could use an index */
       ret = gst_pad_push_event (self->sinkpad,
           gst_event_new_seek (rate, GST_FORMAT_BYTES, flags,
               GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_NONE, 0));
 
-      if (ret) {
-        /* Apply the seek to our segment */
-        gst_segment_do_seek (&self->segment, rate, format, flags,
-            start_type, start, stop_type, stop, &update);
-
-        GST_DEBUG_OBJECT (self, "segment after seek: %" GST_SEGMENT_FORMAT,
-            &self->segment);
-
-        /* will mark need_segment when receiving segment from upstream,
-         * after FLUSH and all that has happened,
-         * rather than racing with chain */
-      } else {
+      if (!ret) {
         GST_WARNING_OBJECT (self, "seek to 0 bytes failed");
       }
 
@@ -1805,9 +1801,10 @@ gst_sub_parse_sink_event (GstPad * pad, GstObject * parent, GstEvent * event)
       gst_event_parse_segment (event, &s);
       if (s->format == GST_FORMAT_TIME)
         gst_event_copy_segment (event, &self->segment);
-      GST_DEBUG_OBJECT (self, "newsegment (%s)",
-          gst_format_get_name (self->segment.format));
+      GST_DEBUG_OBJECT (self, "newsegment (%s) %" GST_SEGMENT_FORMAT,
+          gst_format_get_name (self->segment.format), &self->segment);
       self->segment_seqnum = gst_event_get_seqnum (event);
+      self->need_segment = TRUE;
 
       /* if not time format, we'll either start with a 0 timestamp anyway or
        * it's following a seek in which case we'll have saved the requested
-- 
2.34.1


From c2b9aa80153705705c3035212013bac5ab205b31 Mon Sep 17 00:00:00 2001
From: zhouming <b42586@freescale.com>
Date: Wed, 14 May 2014 10:16:20 +0800
Subject: [PATCH 05/93] ENGR00312515: get caps from src pad when query caps

https://bugzilla.gnome.org/show_bug.cgi?id=728312

Upstream Status: Pending

Signed-off-by: zhouming <b42586@freescale.com>
(cherry picked from commit d68b1bb512ec3270b4b049475b39df1a0ad55bb1)
---
 gst-libs/gst/tag/gsttagdemux.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/gst-libs/gst/tag/gsttagdemux.c b/gst-libs/gst/tag/gsttagdemux.c
index ef0ff90be..53f05c8b4 100644
--- a/gst-libs/gst/tag/gsttagdemux.c
+++ b/gst-libs/gst/tag/gsttagdemux.c
@@ -1796,6 +1796,19 @@ gst_tag_demux_pad_query (GstPad * pad, GstObject * parent, GstQuery * query)
       }
       break;
     }
+    case GST_QUERY_CAPS:
+    {
+
+      /* We can hijack caps query if we typefind already */
+      if (demux->priv->src_caps) {
+        gst_query_set_caps_result (query, demux->priv->src_caps);
+        res = TRUE;
+      } else {
+        res = gst_pad_query_default (pad, parent, query);
+      }
+      break;
+    }
+
     default:
       res = gst_pad_query_default (pad, parent, query);
       break;
-- 
2.34.1


From 9a518c682eef82686b7ac33dbcd9ab22479f72fd Mon Sep 17 00:00:00 2001
From: Jian Li <jian.li@freescale.com>
Date: Mon, 23 Jun 2014 14:14:07 +0800
Subject: [PATCH 06/93] LF-8635 playbin3/playbin2: remove default deinterlace
 flag

remove default deinterlace flag in playbin3 for i.MX SoCs

upstream status: imx specific
---
 gst/playback/gstplaybin2.c | 3 +--
 gst/playback/gstplaybin3.c | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/gst/playback/gstplaybin2.c b/gst/playback/gstplaybin2.c
index 7f843bd44..7cef7d51a 100644
--- a/gst/playback/gstplaybin2.c
+++ b/gst/playback/gstplaybin2.c
@@ -507,8 +507,7 @@ struct _GstPlayBinClass
 #define DEFAULT_SUBURI            NULL
 #define DEFAULT_SOURCE            NULL
 #define DEFAULT_FLAGS             GST_PLAY_FLAG_AUDIO | GST_PLAY_FLAG_VIDEO | GST_PLAY_FLAG_TEXT | \
-                                  GST_PLAY_FLAG_SOFT_VOLUME | GST_PLAY_FLAG_DEINTERLACE | \
-                                  GST_PLAY_FLAG_SOFT_COLORBALANCE
+                                  GST_PLAY_FLAG_SOFT_VOLUME
 #define DEFAULT_N_VIDEO           0
 #define DEFAULT_CURRENT_VIDEO     -1
 #define DEFAULT_N_AUDIO           0
diff --git a/gst/playback/gstplaybin3.c b/gst/playback/gstplaybin3.c
index 3e735c3a3..e3d876bf0 100644
--- a/gst/playback/gstplaybin3.c
+++ b/gst/playback/gstplaybin3.c
@@ -397,8 +397,7 @@ struct _GstPlayBin3Class
 #define DEFAULT_URI               NULL
 #define DEFAULT_SUBURI            NULL
 #define DEFAULT_FLAGS             GST_PLAY_FLAG_AUDIO | GST_PLAY_FLAG_VIDEO | GST_PLAY_FLAG_TEXT | \
-                                  GST_PLAY_FLAG_SOFT_VOLUME | GST_PLAY_FLAG_DEINTERLACE | \
-                                  GST_PLAY_FLAG_SOFT_COLORBALANCE | GST_PLAY_FLAG_BUFFERING
+                                  GST_PLAY_FLAG_SOFT_VOLUME | GST_PLAY_FLAG_BUFFERING
 #define DEFAULT_CURRENT_VIDEO     -1
 #define DEFAULT_CURRENT_AUDIO     -1
 #define DEFAULT_CURRENT_TEXT      -1
-- 
2.34.1


From 889efdb69be9a9347aee31ba46e6525200bdb749 Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@freescale.com>
Date: Wed, 21 Oct 2015 16:35:43 +0800
Subject: [PATCH 07/93] taglist not send to down stream if all the frame
 corrupted

https://bugzilla.gnome.org/show_bug.cgi?id=737246

Upstream status: Pending

Signed-off-by: Jian Li <lj.qfy.sh@gmail.com>
(cherry picked from commit 8467a918fdeef633fcbe0fa3e8d936cd921fc25a)
---
 gst-libs/gst/audio/gstaudiodecoder.c | 9 +++++++++
 gst-libs/gst/video/gstvideodecoder.c | 8 ++++++++
 2 files changed, 17 insertions(+)

diff --git a/gst-libs/gst/audio/gstaudiodecoder.c b/gst-libs/gst/audio/gstaudiodecoder.c
index 14a603e77..3ffd591a5 100644
--- a/gst-libs/gst/audio/gstaudiodecoder.c
+++ b/gst-libs/gst/audio/gstaudiodecoder.c
@@ -2508,6 +2508,15 @@ gst_audio_decoder_sink_eventfunc (GstAudioDecoder * dec, GstEvent * event)
             ("no valid frames found"));
       }
 
+      /* send taglist if no valid frame is decoded util EOS */
+      if (dec->priv->taglist && dec->priv->taglist_changed) {
+        GST_DEBUG_OBJECT (dec, "codec tag %" GST_PTR_FORMAT, dec->priv->taglist);
+        if (!gst_tag_list_is_empty (dec->priv->taglist))
+          gst_audio_decoder_push_event (dec,
+              gst_event_new_tag (gst_tag_list_ref (dec->priv->taglist)));
+        dec->priv->taglist_changed = FALSE;
+      }
+
       /* Forward EOS because no buffer or serialized event will come after
        * EOS and nothing could trigger another _finish_frame() call. */
       if (dec->priv->pending_events)
diff --git a/gst-libs/gst/video/gstvideodecoder.c b/gst-libs/gst/video/gstvideodecoder.c
index 74bcab1e7..df12e89b9 100644
--- a/gst-libs/gst/video/gstvideodecoder.c
+++ b/gst-libs/gst/video/gstvideodecoder.c
@@ -1427,6 +1427,14 @@ gst_video_decoder_sink_event_default (GstVideoDecoder * decoder,
        * parent class' ::sink_event() until a later time.
        */
       forward_immediate = TRUE;
+
+      /* send taglist if no valid frame is decoded util EOS */
+      if (decoder->priv->tags && decoder->priv->tags_changed) {
+        gst_video_decoder_push_event (decoder,
+            gst_event_new_tag (gst_tag_list_ref (decoder->priv->tags)));
+        decoder->priv->tags_changed = FALSE;
+      }
+
       break;
     }
     case GST_EVENT_GAP:
-- 
2.34.1


From b5606a73b7800427b160e1a98d57c9ab02c7248e Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@freescale.com>
Date: Mon, 15 Dec 2014 16:52:07 +0800
Subject: [PATCH 08/93] handle audio/video decoder error

When there is input data and no output data to the end of the stream, it will
send GST_ELEMENT_ERROR, So the clips playing will quit.
However, if only one of the tracks is corrupt, there is no need to quit other
tracks playing.

The patch comments the GST_ELEMENT_ERROR() and just add GST_ERROR_OBJECT()
information instead.

https://bugzilla.gnome.org/show_bug.cgi?id=741542

Upstream Status: Pending

Signed-off-by: Lyon Wang <lyon.wang@freescale.com>
(cherry picked from commit 19e771001a499442bee7da7e5f20d285afc51684)
---
 gst-libs/gst/audio/gstaudiodecoder.c | 5 +++--
 gst-libs/gst/video/gstvideodecoder.c | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/gst-libs/gst/audio/gstaudiodecoder.c b/gst-libs/gst/audio/gstaudiodecoder.c
index 3ffd591a5..daac3acaa 100644
--- a/gst-libs/gst/audio/gstaudiodecoder.c
+++ b/gst-libs/gst/audio/gstaudiodecoder.c
@@ -2503,9 +2503,10 @@ gst_audio_decoder_sink_eventfunc (GstAudioDecoder * dec, GstEvent * event)
       GST_AUDIO_DECODER_STREAM_UNLOCK (dec);
 
       if (dec->priv->ctx.had_input_data && !dec->priv->ctx.had_output_data) {
-        GST_ELEMENT_ERROR (dec, STREAM, DECODE,
+        /* GST_ELEMENT_ERROR (dec, STREAM, DECODE,
             ("No valid frames decoded before end of stream"),
-            ("no valid frames found"));
+            ("no valid frames found")); */
+        GST_ERROR_OBJECT(dec, "No valid frames decoded before end of stream");
       }
 
       /* send taglist if no valid frame is decoded util EOS */
diff --git a/gst-libs/gst/video/gstvideodecoder.c b/gst-libs/gst/video/gstvideodecoder.c
index df12e89b9..4fa7449d3 100644
--- a/gst-libs/gst/video/gstvideodecoder.c
+++ b/gst-libs/gst/video/gstvideodecoder.c
@@ -1413,9 +1413,10 @@ gst_video_decoder_sink_event_default (GstVideoDecoder * decoder,
 
       /* Error out even if EOS was ok when we had input, but no output */
       if (ret && priv->had_input_data && !priv->had_output_data) {
-        GST_ELEMENT_ERROR (decoder, STREAM, DECODE,
+        /* GST_ELEMENT_ERROR (decoder, STREAM, DECODE,
             ("No valid frames decoded before end of stream"),
-            ("no valid frames found"));
+            ("no valid frames found")); */
+        GST_ERROR_OBJECT(decoder, "No valid frames decoded before end of stream");
       }
 
       /* Forward EOS immediately. This is required because no
-- 
2.34.1


From 2b382d451982b5892df19e8e9aaea9ab52153fba Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@freescale.com>
Date: Tue, 17 Nov 2015 14:56:47 +0800
Subject: [PATCH 09/93] gstaudiobasesink print warning istead of return ERROR.

For those clips with corrupt audio track,
there might be no output from audio decoder
and thus the audio track have no chance to negotiate.
We can just print error warning instead of return ERROR,
so that other track can be played normally

https://bugzilla.gnome.org/show_bug.cgi?id=758215

Upstream Status:  Pending

Signed-off-by: Lyon Wang <lyon.wang@freescale.com>
(cherry picked from commit 089b03a9c5f055f79346e30fc0e618f5287bb7b0)
---
 gst-libs/gst/audio/gstaudiobasesink.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/gst-libs/gst/audio/gstaudiobasesink.c b/gst-libs/gst/audio/gstaudiobasesink.c
index 537425da1..e6fc003e1 100644
--- a/gst-libs/gst/audio/gstaudiobasesink.c
+++ b/gst-libs/gst/audio/gstaudiobasesink.c
@@ -1114,10 +1114,15 @@ gst_audio_base_sink_wait_event (GstBaseSink * bsink, GstEvent * event)
     case GST_EVENT_GAP:
       /* We must have a negotiated format before starting the ringbuffer */
       if (G_UNLIKELY (!gst_audio_ring_buffer_is_acquired (sink->ringbuffer))) {
-        GST_ELEMENT_ERROR (sink, STREAM, FORMAT, (NULL),
+  /*      GST_ELEMENT_ERROR (sink, STREAM, FORMAT, (NULL),
             ("Sink not negotiated before %s event.",
                 GST_EVENT_TYPE_NAME (event)));
+
         return GST_FLOW_ERROR;
+   */
+        /* consider there might be chance that corrupt audio track without output buffer and not negotiated.
+             We'd better not return error and quit play, video track can keep playing.*/
+        GST_ERROR_OBJECT(sink, "Sink not negotiated before %s event.",GST_EVENT_TYPE_NAME (event));
       }
 
       gst_audio_base_sink_force_start (sink);
-- 
2.34.1


From a6beb8e304dcb56d1c87df50e786da39d56bb5b9 Mon Sep 17 00:00:00 2001
From: Song Bing <b06498@freescale.com>
Date: Mon, 11 Jan 2016 14:51:17 +0800
Subject: [PATCH 10/93] MMFMWK-7030 [Linux_MX6QP_ARD]IMXCameraApp:When Enabled
 "save time to image" item, preview, find the time can not display completely.
 100%

As IPU need 8 pixels alignment, add one workaround in base text overlay
to generate 8 pixels alignment text video buffer. The side effect should
cause all text a little smaller.

Upstream Status: Inappropriate [i.MX specific]

Signed-off-by: Song Bing b06498@freescale.com
(cherry picked from commit 265d56846c51da30ab864ac6eb79a50005def560)
---
 ext/pango/gstbasetextoverlay.c | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ext/pango/gstbasetextoverlay.c b/ext/pango/gstbasetextoverlay.c
index 08c6b97c0..8b63ed54d 100644
--- a/ext/pango/gstbasetextoverlay.c
+++ b/ext/pango/gstbasetextoverlay.c
@@ -1727,6 +1727,10 @@ gst_base_text_overlay_render_pangocairo (GstBaseTextOverlay * overlay,
   GstBuffer *buffer;
   GstMapInfo map;
 
+  /* i.MX special, will cause text a little small */
+  overlay->width = GST_ROUND_DOWN_8 (overlay->width);
+  overlay->height = GST_ROUND_DOWN_8 (overlay->height);
+
   if (overlay->auto_adjust_size) {
     /* 640 pixel is default */
     scalef_x = scalef_y = (double) (overlay->width) / DEFAULT_SCALE_BASIS;
@@ -1887,6 +1891,9 @@ gst_base_text_overlay_render_pangocairo (GstBaseTextOverlay * overlay,
   scalef_x *= overlay->render_scale;
   scalef_y *= overlay->render_scale;
 
+ GST_DEBUG_OBJECT (overlay, "Rendering with width %d and height %d "
+      , width, height);
+
   if (width <= 0 || height <= 0) {
     GST_DEBUG_OBJECT (overlay,
         "Overlay is outside video frame. Skipping text rendering");
-- 
2.34.1


From c24a9887e005049b6c7dac696c98c5eb5fcf4611 Mon Sep 17 00:00:00 2001
From: Song Bing <bing.song@nxp.com>
Date: Wed, 13 Sep 2017 13:37:17 -0800
Subject: [PATCH 11/93] dmabuf: set fd memory to keep mapped

set fd memory to keep mapped.

Upstream Status: Pending

https://bugzilla.gnome.org/show_bug.cgi?id=768794
(cherry picked from commit eeb24c2307f219b5accbd0afcf84c6701408defd)
---
 gst-libs/gst/allocators/gstdmabuf.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gst-libs/gst/allocators/gstdmabuf.c b/gst-libs/gst/allocators/gstdmabuf.c
index 2793b73c8..15de4c83e 100644
--- a/gst-libs/gst/allocators/gstdmabuf.c
+++ b/gst-libs/gst/allocators/gstdmabuf.c
@@ -156,7 +156,7 @@ gst_dmabuf_allocator_alloc (GstAllocator * allocator, gint fd, gsize size)
 {
   g_return_val_if_fail (GST_IS_DMABUF_ALLOCATOR (allocator), NULL);
 
-  return gst_fd_allocator_alloc (allocator, fd, size, GST_FD_MEMORY_FLAG_NONE);
+  return gst_fd_allocator_alloc (allocator, fd, size, GST_FD_MEMORY_FLAG_KEEP_MAPPED);
 }
 
 /**
-- 
2.34.1


From 21331b44b79c0dd760abe9c0d03b5a872a215112 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Mon, 6 Nov 2017 15:05:47 +0800
Subject: [PATCH 12/93] fbmemory: need unmap if mapping flags are not subset of
 previous

upstream status: Pending
https://bugzilla.gnome.org/show_bug.cgi?id=789952

(cherry picked from commit e8723f5fbe30d481f63d7c8500de0afdcdc25d21)
---
 gst-libs/gst/allocators/gstfdmemory.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/gst-libs/gst/allocators/gstfdmemory.c b/gst-libs/gst/allocators/gstfdmemory.c
index f2963999e..ae3112101 100644
--- a/gst-libs/gst/allocators/gstfdmemory.c
+++ b/gst-libs/gst/allocators/gstfdmemory.c
@@ -97,15 +97,20 @@ gst_fd_mem_map (GstMemory * gmem, gsize maxsize, GstMapFlags flags)
     if ((mem->mmapping_flags & prot) == prot) {
       ret = mem->data;
       mem->mmap_count++;
+      goto out;
     } else if ((mem->flags & GST_FD_MEMORY_FLAG_KEEP_MAPPED)
         && mem->mmap_count == 0
         && mprotect (mem->data, gmem->maxsize, prot) == 0) {
       ret = mem->data;
       mem->mmapping_flags = prot;
       mem->mmap_count++;
+      goto out;
+    } else {
+      /* if mapping flags is not a subset, need unmap first */
+      munmap ((void *) mem->data, gmem->maxsize);
+      mem->data = NULL;
+      mem->mmap_count = 0;
     }
-
-    goto out;
   }
 
   if (mem->fd != -1) {
-- 
2.34.1


From 7d06e27e26925d0800cff1489bcc84f06f4c9a7f Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Thu, 30 Nov 2017 17:43:20 +0800
Subject: [PATCH 13/93] basetextoverlay: need avoid idx exceed memory block
 number

when check whether video buffer is read only, the gst_buffer_get_memory call
should make sure idx don't exceed the total memory block number

upstream status: i.mx specific

Signed-off-by: Haihua Hu <jared.hu@nxp.com>
(cherry picked from commit f48f36f7d711a62d0eabc5cdec4f25b41d39c961)
---
 ext/pango/gstbasetextoverlay.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ext/pango/gstbasetextoverlay.c b/ext/pango/gstbasetextoverlay.c
index 8b63ed54d..4683b7070 100644
--- a/ext/pango/gstbasetextoverlay.c
+++ b/ext/pango/gstbasetextoverlay.c
@@ -2336,7 +2336,9 @@ gst_base_text_overlay_push_frame (GstBaseTextOverlay * overlay,
   gboolean mem_rdonly = FALSE;
   GstMemory *mem;
 
-  while (mem = gst_buffer_get_memory(video_frame, idx++)) {
+  gint n_mem = gst_buffer_n_memory (video_frame);
+
+  while (idx < n_mem && (mem = gst_buffer_get_memory(video_frame, idx++))) {
     if (GST_MEMORY_IS_READONLY(mem)) {
       gst_memory_unref (mem);
       mem_rdonly = TRUE;
-- 
2.34.1


From 5c07f810547b57b3edb8688983bdde768dc9cb1d Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@nxp.com>
Date: Thu, 8 Feb 2018 15:26:06 +0800
Subject: [PATCH 14/93] Move gstimxcommon.h into gst-plugins-base

MMFMWK-7883

Move gstimxcommon.h into base for future imx specified dev

Signed-off-by: Lyon Wang <lyon.wang@nxp.com>
(cherry picked from commit d39f75c96e0d5a16d50beb39982e36ee8d89f2df)
---
 gst-libs/gst/gstimxcommon.h | 358 ++++++++++++++++++++++++++++++++++++
 gst-libs/gst/meson.build    |   5 +
 2 files changed, 363 insertions(+)
 create mode 100644 gst-libs/gst/gstimxcommon.h

diff --git a/gst-libs/gst/gstimxcommon.h b/gst-libs/gst/gstimxcommon.h
new file mode 100644
index 000000000..a7c729b0d
--- /dev/null
+++ b/gst-libs/gst/gstimxcommon.h
@@ -0,0 +1,358 @@
+/*
+ * Copyright (c) 2013-2016, Freescale Semiconductor, Inc. All rights reserved.
+ * Copyright 2017, 2018 NXP
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __IMX_COMMON_H__
+#define __IMX_COMMON_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/utsname.h>
+#include <gst/gst.h>
+#include <string.h>
+
+#define IMX_GST_PLUGIN_AUTHOR "Multimedia Team <shmmmw@freescale.com>"
+#define IMX_GST_PLUGIN_PACKAGE_NAME "Freescle Gstreamer Multimedia Plugins"
+#define IMX_GST_PLUGIN_PACKAGE_ORIG "http://www.freescale.com"
+#define IMX_GST_PLUGIN_LICENSE "LGPL"
+
+#define IMX_GST_PLUGIN_RANK (GST_RANK_PRIMARY+1)
+
+#define IMX_GST_PLUGIN_DEFINE(name, description, initfunc)\
+  GST_PLUGIN_DEFINE(GST_VERSION_MAJOR,\
+      GST_VERSION_MINOR,\
+      name.imx,\
+      description,\
+      initfunc,\
+      VERSION,\
+      IMX_GST_PLUGIN_LICENSE,\
+      IMX_GST_PLUGIN_PACKAGE_NAME, IMX_GST_PLUGIN_PACKAGE_ORIG)
+
+#define CHIPCODE(a,b,c,d)( (((unsigned int)((a)))<<24) | (((unsigned int)((b)))<<16)|(((unsigned int)((c)))<<8)|(((unsigned int)((d)))))
+typedef enum
+{
+  CC_MX23 = CHIPCODE ('M', 'X', '2', '3'),
+  CC_MX25 = CHIPCODE ('M', 'X', '2', '5'),
+  CC_MX27 = CHIPCODE ('M', 'X', '2', '7'),
+  CC_MX28 = CHIPCODE ('M', 'X', '2', '8'),
+  CC_MX31 = CHIPCODE ('M', 'X', '3', '1'),
+  CC_MX35 = CHIPCODE ('M', 'X', '3', '5'),
+  CC_MX37 = CHIPCODE ('M', 'X', '3', '7'),
+  CC_MX50 = CHIPCODE ('M', 'X', '5', '0'),
+  CC_MX51 = CHIPCODE ('M', 'X', '5', '1'),
+  CC_MX53 = CHIPCODE ('M', 'X', '5', '3'),
+  CC_MX6Q = CHIPCODE ('M', 'X', '6', 'Q'),
+  CC_MX60 = CHIPCODE ('M', 'X', '6', '0'),
+  CC_MX6SL = CHIPCODE ('M', 'X', '6', '1'),
+  CC_MX6SX = CHIPCODE ('M', 'X', '6', '2'),
+  CC_MX6UL = CHIPCODE ('M', 'X', '6', '3'),
+  CC_MX6SLL = CHIPCODE ('M', 'X', '6', '4'),
+  CC_MX7D = CHIPCODE ('M', 'X', '7', 'D'),
+  CC_MX7ULP = CHIPCODE ('M', 'X', '7', 'U'),
+  CC_MX8 = CHIPCODE ('M', 'X', '8', '0'),
+  CC_MX8QM = CHIPCODE ('M', 'X', '8', '1'),
+  CC_MX8QXP = CHIPCODE ('M', 'X', '8', '3'),
+  CC_MX8M = CHIPCODE ('M', 'X', '8', '2'),
+  CC_UNKN = CHIPCODE ('U', 'N', 'K', 'N')
+
+} CHIP_CODE;
+
+typedef struct {
+  CHIP_CODE code;
+  int chip_num;
+} CPU_INFO;
+
+typedef struct {
+  CHIP_CODE code;
+  char *name;
+} SOC_INFO;
+
+typedef struct {
+  CHIP_CODE chip_name;
+  gboolean g3d;
+  gboolean g2d;
+  gboolean ipu;
+  gboolean pxp;
+  gboolean vpu;
+  gboolean dpu;
+  gboolean dcss;
+} IMXV4l2FeatureMap;
+
+typedef enum {
+  G3D = 1,
+  G2D,
+  IPU,
+  PXP,
+  VPU,
+  DPU,
+  DCSS
+} CHIP_FEATURE;
+
+
+#define HAS_G3D() check_feature(imx_chip_code(), G3D)
+#define HAS_G2D() check_feature(imx_chip_code(), G2D)
+#define HAS_IPU() check_feature(imx_chip_code(), IPU)
+#define HAS_PXP() check_feature(imx_chip_code(), PXP)
+#define HAS_VPU() check_feature(imx_chip_code(), VPU)
+#define HAS_DPU() check_feature(imx_chip_code(), DPU)
+#define HAS_DCSS() check_feature(imx_chip_code(), DCSS)
+
+#define IS_HANTRO() (CC_MX8M == imx_chip_code())
+#define IS_AMPHION() (CC_MX8QXP == imx_chip_code())
+
+
+/* define rotate and flip glib enum for overlaysink and imxv4l2sink */
+typedef enum
+{
+  GST_IMX_ROTATION_0 = 0,
+  GST_IMX_ROTATION_90,
+  GST_IMX_ROTATION_180,
+  GST_IMX_ROTATION_270,
+  GST_IMX_ROTATION_HFLIP,
+  GST_IMX_ROTATION_VFLIP
+}GstImxRotateMethod;
+
+/*=============================================================================
+FUNCTION:               get_chipname
+
+DESCRIPTION:            To get chipname from /proc/cpuinfo
+
+ARGUMENTS PASSED: STR of chipname
+
+RETURN VALUE:            chip code
+=============================================================================*/
+//*
+
+static CPU_INFO cpu_info[] = {
+  {CC_MX23, 0x23},
+  {CC_MX25, 0x25},
+  {CC_MX27, 0x27},
+  {CC_MX28, 0x28},
+  {CC_MX31, 0x31},
+  {CC_MX35, 0x35},
+  {CC_MX37, 0x37},
+  {CC_MX50, 0x50},
+  {CC_MX51, 0x51},
+  {CC_MX53, 0x53},
+  {CC_MX6Q, 0x61},
+  {CC_MX6Q, 0x63},
+  {CC_MX60, 0x60}
+};
+
+static CHIP_CODE getChipCodeFromCpuinfo (void)
+{
+  FILE *fp = NULL;
+  char buf[100], *p, *rev;
+  char chip_name[3];
+  int len = 0, i;
+  int chip_num = -1;
+  CHIP_CODE cc = CC_UNKN;
+  fp = fopen ("/proc/cpuinfo", "r");
+  if (fp == NULL) {
+    return cc;
+  }
+  while (!feof (fp)) {
+    p = fgets (buf, 100, fp);
+    p = strstr (buf, "Revision");
+    if (p != NULL) {
+      rev = index (p, ':');
+      if (rev != NULL) {
+        rev++;
+        chip_num = strtoul (rev, NULL, 16);
+        chip_num >>= 12;
+        break;
+      }
+    }
+  }
+
+  fclose (fp);
+
+  if (chip_num < 0) {
+    return cc;
+  }
+
+  int num = sizeof(cpu_info) / sizeof(CPU_INFO);
+  for(i=0; i<num; i++) {
+    if(chip_num == cpu_info[i].chip_num) {
+      cc = cpu_info[i].code;
+      break;
+    }
+  }
+
+  return cc;
+}
+
+static SOC_INFO soc_info[] = {
+  {CC_MX23, "i.MX23"},
+  {CC_MX25, "i.MX25"},
+  {CC_MX27, "i.MX27"},
+  {CC_MX28, "i.MX28"},
+  {CC_MX31, "i.MX31"},
+  {CC_MX35, "i.MX35"},
+  {CC_MX37, "i.MX37"},
+  {CC_MX50, "i.MX50"},
+  {CC_MX51, "i.MX51"},
+  {CC_MX53, "i.MX53"},
+  {CC_MX6Q, "i.MX6DL"},
+  {CC_MX6Q, "i.MX6Q"},
+  {CC_MX6Q, "i.MX6QP"},
+  {CC_MX6SL, "i.MX6SL"},
+  {CC_MX6SLL, "i.MX6SLL"},
+  {CC_MX6SX, "i.MX6SX"},
+  {CC_MX6UL, "i.MX6UL"},
+  {CC_MX6UL, "i.MX6ULL"},
+  {CC_MX7D, "i.MX7D"},
+  {CC_MX7ULP, "i.MX7ULP"},
+  {CC_MX8, "i.MX8DV"},
+  {CC_MX8QM, "i.MX8QM"},
+  {CC_MX8QXP, "i.MX8QXP"},
+  {CC_MX8M, "i.MX8MQ"},
+};
+
+static CHIP_CODE getChipCodeFromSocid (void)
+{
+  FILE *fp = NULL;
+  char soc_name[100];
+  CHIP_CODE code = CC_UNKN;
+
+  fp = fopen("/sys/devices/soc0/soc_id", "r");
+  if (fp == NULL) {
+    g_print("open /sys/devices/soc0/soc_id failed.\n");
+    return  CC_UNKN;
+  }
+
+  if (fscanf(fp, "%100s", soc_name) != 1) {
+    g_print("fscanf soc_id failed.\n");
+    fclose(fp);
+    return CC_UNKN;
+  }
+  fclose(fp);
+
+  //GST_INFO("SOC is %s\n", soc_name);
+
+  int num = sizeof(soc_info) / sizeof(SOC_INFO);
+  int i;
+  for(i=0; i<num; i++) {
+    if(!strcmp(soc_name, soc_info[i].name)) {
+      code = soc_info[i].code;
+      break;
+    }
+  }
+
+  return code;
+}
+
+#define KERN_VER(a, b, c) (((a) << 16) + ((b) << 8) + (c))
+
+static CHIP_CODE gimx_chip_code = CC_UNKN;
+
+static CHIP_CODE imx_chip_code (void)
+{
+  struct utsname sys_name;
+  int kv, kv_major, kv_minor, kv_rel;
+  char soc_name[255];
+  int rev_major, rev_minor;
+  int idx, num;
+
+  if (gimx_chip_code != CC_UNKN)
+    return gimx_chip_code;
+
+  if (uname(&sys_name) < 0) {
+    g_print("get kernel version via uname failed.\n");
+    return CC_UNKN;
+  }
+
+  if (sscanf(sys_name.release, "%d.%d.%d", &kv_major, &kv_minor, &kv_rel) != 3) {
+    g_print("sscanf kernel version failed.\n");
+    return CC_UNKN;
+  }
+
+  kv = ((kv_major << 16) + (kv_minor << 8) + kv_rel);
+  //GST_INFO("kernel:%s, %d.%d.%d\n", sys_name.release, kv_major, kv_minor, kv_rel);
+
+  if (kv < KERN_VER(3, 10, 0))
+    gimx_chip_code = getChipCodeFromCpuinfo();
+  else
+    gimx_chip_code = getChipCodeFromSocid();
+
+  return gimx_chip_code;
+}
+
+static IMXV4l2FeatureMap g_imxv4l2feature_maps[] = {
+  /* chip_name, g3d, g2d, ipu, pxp, vpu, dpu, dcss*/
+  {CC_MX6Q, TRUE, TRUE, TRUE, FALSE, TRUE, FALSE, FALSE},
+  {CC_MX6SL, FALSE, TRUE, FALSE, TRUE, FALSE, FALSE, FALSE},
+  {CC_MX6SLL, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE},
+  {CC_MX6SX, TRUE, TRUE, FALSE, TRUE, FALSE, FALSE, FALSE},
+  {CC_MX6UL, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE},
+  {CC_MX7D, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE},
+  {CC_MX7ULP, TRUE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE},
+  {CC_MX8, TRUE, TRUE, FALSE, FALSE, FALSE, TRUE, FALSE},
+  {CC_MX8QM, TRUE, TRUE, FALSE, FALSE, FALSE, TRUE, FALSE},
+  {CC_MX8QXP, TRUE, TRUE, FALSE, FALSE, TRUE, TRUE, FALSE},
+  {CC_MX8M, TRUE, FALSE, FALSE, FALSE, TRUE, FALSE, TRUE},
+};
+
+
+static gboolean check_feature(CHIP_CODE chip_name, CHIP_FEATURE feature)
+{
+  int i;
+  gboolean ret = FALSE;
+  for (i=0; i<sizeof(g_imxv4l2feature_maps)/sizeof(IMXV4l2FeatureMap); i++) {	
+    if ( chip_name== g_imxv4l2feature_maps[i].chip_name) {
+      switch (feature) {
+        case G3D:
+          ret = g_imxv4l2feature_maps[i].g3d;
+          break;
+        case G2D:
+          ret = g_imxv4l2feature_maps[i].g2d;
+          break;
+        case IPU:
+          ret = g_imxv4l2feature_maps[i].ipu;
+          break;
+        case PXP:
+          ret = g_imxv4l2feature_maps[i].pxp;
+          break;
+        case VPU:
+          ret = g_imxv4l2feature_maps[i].vpu;
+          break;
+        case DPU:
+          ret = g_imxv4l2feature_maps[i].dpu;
+          break;
+        case DCSS:
+          ret = g_imxv4l2feature_maps[i].dcss;
+          break;
+        default:
+          break;
+      }
+      break;
+    }
+  }
+  return ret;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __IMX_COMMON_H__ */ 
diff --git a/gst-libs/gst/meson.build b/gst-libs/gst/meson.build
index 42c0a6fca..ff8e3af11 100644
--- a/gst-libs/gst/meson.build
+++ b/gst-libs/gst/meson.build
@@ -11,3 +11,8 @@ subdir('app')
 subdir('allocators')
 # FIXME: gl deps are automagic
 subdir('gl')
+
+imxcommon_header = [
+  'gstimxcommon.h',
+]
+install_headers(imxcommon_header)
\ No newline at end of file
-- 
2.34.1


From a049b175cdbb1073772d27b92da0ddad0a639c3b Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@nxp.com>
Date: Thu, 3 May 2018 17:28:02 +0800
Subject: [PATCH 15/93] gstimxcommon: add some platform marcos

add 8mm,8mp,8mn,8ulp,imx93,imx95

upstream status: imx specific

Signed-off-by: Haihua Hu <jared.hu@nxp.com>
---
 gst-libs/gst/gstimxcommon.h | 33 ++++++++++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/gst-libs/gst/gstimxcommon.h b/gst-libs/gst/gstimxcommon.h
index a7c729b0d..e2a0aa295 100644
--- a/gst-libs/gst/gstimxcommon.h
+++ b/gst-libs/gst/gstimxcommon.h
@@ -73,6 +73,12 @@ typedef enum
   CC_MX8QM = CHIPCODE ('M', 'X', '8', '1'),
   CC_MX8QXP = CHIPCODE ('M', 'X', '8', '3'),
   CC_MX8M = CHIPCODE ('M', 'X', '8', '2'),
+  CC_MX8MM = CHIPCODE ('M', 'X', '8', '4'),
+  CC_MX8MN = CHIPCODE ('M', 'X', '8', '5'),
+  CC_MX8MP = CHIPCODE ('M', 'X', '8', '6'),
+  CC_MX8ULP = CHIPCODE ('M', 'X', '8', 'U'),
+  CC_MX93 = CHIPCODE ('M', 'X', '9', '3'),
+  CC_MX95 = CHIPCODE ('M', 'X', '9', '5'),
   CC_UNKN = CHIPCODE ('U', 'N', 'K', 'N')
 
 } CHIP_CODE;
@@ -117,8 +123,17 @@ typedef enum {
 #define HAS_DPU() check_feature(imx_chip_code(), DPU)
 #define HAS_DCSS() check_feature(imx_chip_code(), DCSS)
 
-#define IS_HANTRO() (CC_MX8M == imx_chip_code())
-#define IS_AMPHION() (CC_MX8QXP == imx_chip_code())
+#define IS_HANTRO() ((CC_MX8M == imx_chip_code()) || (CC_MX8MM == imx_chip_code()) || (CC_MX8MP == imx_chip_code()) )
+#define IS_AMPHION() ((CC_MX8QM == imx_chip_code()) || (CC_MX8QXP == imx_chip_code()))
+#define IS_IMX8MM() (CC_MX8MM == imx_chip_code())
+#define IS_IMX8MN() (CC_MX8MN == imx_chip_code())
+#define IS_IMX8MQ() (CC_MX8M == imx_chip_code())
+#define IS_IMX8MP() (CC_MX8MP == imx_chip_code())
+#define IS_IMX8ULP() (CC_MX8ULP == imx_chip_code())
+#define IS_IMX93() (CC_MX93 == imx_chip_code())
+#define IS_IMX95() (CC_MX95 == imx_chip_code())
+#define IS_IMX8Q() ((CC_MX8QM == imx_chip_code()) || (CC_MX8QXP == imx_chip_code()))
+#define IS_IMX6Q() (CC_MX6Q == imx_chip_code())
 
 
 /* define rotate and flip glib enum for overlaysink and imxv4l2sink */
@@ -227,6 +242,12 @@ static SOC_INFO soc_info[] = {
   {CC_MX8QM, "i.MX8QM"},
   {CC_MX8QXP, "i.MX8QXP"},
   {CC_MX8M, "i.MX8MQ"},
+  {CC_MX8MM, "i.MX8MM"},
+  {CC_MX8MN, "i.MX8MN"},
+  {CC_MX8MP, "i.MX8MP"},
+  {CC_MX8ULP, "i.MX8ULP"},
+  {CC_MX93, "i.MX93"},
+  {CC_MX95, "i.MX95"},
 };
 
 static CHIP_CODE getChipCodeFromSocid (void)
@@ -308,9 +329,15 @@ static IMXV4l2FeatureMap g_imxv4l2feature_maps[] = {
   {CC_MX7D, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE},
   {CC_MX7ULP, TRUE, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE},
   {CC_MX8, TRUE, TRUE, FALSE, FALSE, FALSE, TRUE, FALSE},
-  {CC_MX8QM, TRUE, TRUE, FALSE, FALSE, FALSE, TRUE, FALSE},
+  {CC_MX8QM, TRUE, TRUE, FALSE, FALSE, TRUE, TRUE, FALSE},
   {CC_MX8QXP, TRUE, TRUE, FALSE, FALSE, TRUE, TRUE, FALSE},
   {CC_MX8M, TRUE, FALSE, FALSE, FALSE, TRUE, FALSE, TRUE},
+  {CC_MX8MM, TRUE, TRUE, FALSE, FALSE, TRUE, FALSE, FALSE},
+  {CC_MX8MP, TRUE, TRUE, FALSE, FALSE, TRUE, FALSE, FALSE},
+  {CC_MX8MN, TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE},
+  {CC_MX8ULP, TRUE, TRUE, FALSE, TRUE, FALSE, FALSE, FALSE},
+  {CC_MX93, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE},
+  {CC_MX95, TRUE, TRUE, FALSE, FALSE, TRUE, TRUE, FALSE},
 };
 
 
-- 
2.34.1


From 24829f314fe89b44de27781f4838c40066fcbfce Mon Sep 17 00:00:00 2001
From: Song Bing <b06498@freescale.com>
Date: Mon, 8 Jun 2015 17:06:22 +0800
Subject: [PATCH 16/93] glfilter: Lost frame rate info when fixate caps
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Lost frame rate info when fixate caps. It will cause
down stream element fail, such avimux.

Upstream Status: Waiting for review.

https://bugzilla.gnome.org/show_bug.cgi?id=750545
(cherry picked from commit bfa1c93f45e45c0bed22816d0c01c09a7252a121)
---
 gst-libs/gst/gl/gstglfilter.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/gst-libs/gst/gl/gstglfilter.c b/gst-libs/gst/gl/gstglfilter.c
index cd79f1b54..edaf49927 100644
--- a/gst-libs/gst/gl/gstglfilter.c
+++ b/gst-libs/gst/gl/gstglfilter.c
@@ -242,7 +242,8 @@ gst_gl_filter_fixate_caps (GstBaseTransform * bt,
     GstPadDirection direction, GstCaps * caps, GstCaps * othercaps)
 {
   GstStructure *ins, *outs;
-  const GValue *from_par, *to_par;
+  const GValue *from_par, *to_par, *from_fps;
+  gint framerate_num, framerate_den;
   GValue fpar = { 0, }, tpar = {
     0,
   };
@@ -256,6 +257,16 @@ gst_gl_filter_fixate_caps (GstBaseTransform * bt,
   ins = gst_caps_get_structure (caps, 0);
   outs = gst_caps_get_structure (othercaps, 0);
 
+  /* replace frame rate */
+  from_fps = gst_structure_get_value (ins, "framerate");
+  if (from_fps) {
+      gst_structure_set_value (outs, "framerate", from_fps);
+  } else {
+    if (gst_structure_get_fraction (ins, "framerate", &framerate_num, &framerate_den))
+      gst_structure_set (outs, "framerate", GST_TYPE_FRACTION, framerate_num, framerate_den,
+          NULL);
+  }
+
   from_par = gst_structure_get_value (ins, "pixel-aspect-ratio");
   to_par = gst_structure_get_value (outs, "pixel-aspect-ratio");
 
-- 
2.34.1


From 8975bf32be89f76268db2edd45d189f9d60206fb Mon Sep 17 00:00:00 2001
From: Haihua Hu <b55597@freescale.com>
Date: Fri, 13 Nov 2015 10:51:25 +0800
Subject: [PATCH 17/93] support video crop for glimagesink

1.Add video crop meta copy in glupload
2.Calculate the new texture coordinate in vertices array and bind to buffer object
3.Make glimagesink only updating vertices array when video crop meta changed

Upstream-Status: Inappropriate [i.MX specific]

Signed-off-by: Haihua Hu <b55597@freescale.com>

(cherry picked from commit 5b19777803b2dea2348a92130c0c3c695655b5b1)
---
 ext/gl/gstglimagesink.c     | 58 +++++++++++++++++++++++++++++++++++++
 ext/gl/gstglimagesink.h     |  3 ++
 ext/gl/gstgluploadelement.c | 14 +++++++--
 3 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/ext/gl/gstglimagesink.c b/ext/gl/gstglimagesink.c
index 3cd0fbeb4..1e435857c 100644
--- a/ext/gl/gstglimagesink.c
+++ b/ext/gl/gstglimagesink.c
@@ -814,6 +814,8 @@ gst_glimage_sink_init (GstGLImageSink * glimage_sink)
   glimage_sink->handle_events = TRUE;
   glimage_sink->ignore_alpha = TRUE;
   glimage_sink->overlay_compositor = NULL;
+  glimage_sink->cropmeta = NULL;
+  glimage_sink->prev_cropmeta = NULL;
 
   glimage_sink->mview_output_mode = DEFAULT_MULTIVIEW_MODE;
   glimage_sink->mview_output_flags = DEFAULT_MULTIVIEW_FLAGS;
@@ -1386,6 +1388,12 @@ gst_glimage_sink_change_state (GstElement * element, GstStateChange transition)
 
       _set_other_context (glimage_sink, NULL);
       _set_display (glimage_sink, NULL);
+
+      glimage_sink->cropmeta = NULL;
+      if (glimage_sink->prev_cropmeta)
+        g_slice_free(GstVideoCropMeta, glimage_sink->prev_cropmeta);
+      glimage_sink->prev_cropmeta = NULL;
+
       break;
     default:
       break;
@@ -1899,6 +1907,8 @@ gst_glimage_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf)
       GST_VIDEO_SINK_WIDTH (glimage_sink),
       GST_VIDEO_SINK_HEIGHT (glimage_sink));
 
+  glimage_sink->cropmeta = gst_buffer_get_video_crop_meta (buf);
+
   /* Ask the underlying window to redraw its content */
   if (!gst_glimage_sink_redisplay (glimage_sink))
     goto redisplay_failed;
@@ -2432,6 +2442,54 @@ gst_glimage_sink_on_draw (GstGLImageSink * gl_sink)
 
     gst_gl_shader_use (gl_sink->redisplay_shader);
 
+    if (gl_sink->cropmeta) {
+      gint width = GST_VIDEO_SINK_WIDTH (gl_sink);
+      gint height = GST_VIDEO_SINK_HEIGHT (gl_sink);
+
+      if (!gl_sink->prev_cropmeta){
+        /* Initialize the previous crop meta and set all memroy to zero */
+        gl_sink->prev_cropmeta = (GstVideoCropMeta *) g_slice_new0(GstVideoCropMeta);
+      }
+
+      /* If crop meta not equal to the previous, recalculate the vertices */
+      if (gl_sink->prev_cropmeta->x != gl_sink->cropmeta->x
+        || gl_sink->prev_cropmeta->y != gl_sink->cropmeta->y
+        || gl_sink->prev_cropmeta->width != gl_sink->cropmeta->width
+        || gl_sink->prev_cropmeta->height != gl_sink->cropmeta->height){
+
+	GLfloat crop_vertices[] = {
+	     1.0f,  1.0f, 0.0f, 1.0f, 0.0f,
+	    -1.0f,  1.0f, 0.0f, 0.0f, 0.0f,
+	    -1.0f, -1.0f, 0.0f, 0.0f, 1.0f,
+	     1.0f, -1.0f, 0.0f, 1.0f, 1.0f
+	};
+
+        crop_vertices[8] = (float)(gl_sink->cropmeta->x) / width;
+        crop_vertices[9] = (float)(gl_sink->cropmeta->y) / height;
+
+        crop_vertices[3] = (float)(gl_sink->cropmeta->width + gl_sink->cropmeta->x) / width;
+        crop_vertices[4] = crop_vertices[9];
+
+        crop_vertices[13] = crop_vertices[8];
+        crop_vertices[14] = (float)(gl_sink->cropmeta->height + gl_sink->cropmeta->y) / height;
+
+        crop_vertices[18] = crop_vertices[3];
+        crop_vertices[19] = crop_vertices[14];
+
+        gl->BindBuffer (GL_ARRAY_BUFFER, gl_sink->vertex_buffer);
+        gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), crop_vertices,
+            GL_STATIC_DRAW);
+
+        gl->BindBuffer (GL_ARRAY_BUFFER, 0);
+
+        /* Store the previous crop meta */
+        gl_sink->prev_cropmeta->x = gl_sink->cropmeta->x;
+        gl_sink->prev_cropmeta->y = gl_sink->cropmeta->y;
+        gl_sink->prev_cropmeta->width = gl_sink->cropmeta->width;
+        gl_sink->prev_cropmeta->height = gl_sink->cropmeta->height;
+      }
+    }
+
     if (gl->GenVertexArrays)
       gl->BindVertexArray (gl_sink->vao);
     _bind_buffer (gl_sink);
diff --git a/ext/gl/gstglimagesink.h b/ext/gl/gstglimagesink.h
index 93014d2ad..a6f36190a 100644
--- a/ext/gl/gstglimagesink.h
+++ b/ext/gl/gstglimagesink.h
@@ -109,6 +109,9 @@ struct _GstGLImageSink
     guint window_width;
     guint window_height;
 
+    GstVideoCropMeta *cropmeta;
+    GstVideoCropMeta *prev_cropmeta;
+
     GstVideoRectangle display_rect;
 
     GstGLShader *redisplay_shader;
diff --git a/ext/gl/gstgluploadelement.c b/ext/gl/gstgluploadelement.c
index 215ee3184..c1e610ace 100644
--- a/ext/gl/gstgluploadelement.c
+++ b/ext/gl/gstgluploadelement.c
@@ -320,9 +320,19 @@ again:
   /* basetransform doesn't unref if they're the same */
   if (buffer == *outbuf)
     gst_buffer_unref (*outbuf);
-  else
+  else {
+    GstVideoCropMeta *incropmeta, *outcropmeta;
+    /* add video crop meta to out buffer if need */
+    incropmeta = gst_buffer_get_video_crop_meta (buffer);
+    if (incropmeta) {
+      outcropmeta = gst_buffer_add_video_crop_meta (*outbuf);
+      outcropmeta->x = incropmeta->x;
+      outcropmeta->y = incropmeta->y;
+      outcropmeta->width = incropmeta->width;
+      outcropmeta->height = incropmeta->height;
+    }
     bclass->copy_metadata (bt, buffer, *outbuf);
-
+  }
   return GST_FLOW_OK;
 }
 
-- 
2.34.1


From ff1cbf3e8ec64657fb8739c993b0ddace212010a Mon Sep 17 00:00:00 2001
From: Haihua Hu <b55597@freescale.com>
Date: Wed, 18 Nov 2015 15:10:22 +0800
Subject: [PATCH 18/93] Add fps print in glimagesink

In GST-1.6, Pipeline will set start time to 0 when state
change form PAUSE to READY, so get start time in state change
PLAYING_PAUSE.

Upstream-Status: Inappropriate [i.MX specific]

Signed-off-by: Haihua Hu <b55597@freescale.com>

(cherry picked from commit 4b58caceb54a0d4f05ed081213e27f9676b5d95b)
---
 ext/gl/gstglimagesink.c | 15 +++++++++++++++
 ext/gl/gstglimagesink.h |  4 ++++
 2 files changed, 19 insertions(+)

diff --git a/ext/gl/gstglimagesink.c b/ext/gl/gstglimagesink.c
index 1e435857c..400026949 100644
--- a/ext/gl/gstglimagesink.c
+++ b/ext/gl/gstglimagesink.c
@@ -816,6 +816,8 @@ gst_glimage_sink_init (GstGLImageSink * glimage_sink)
   glimage_sink->overlay_compositor = NULL;
   glimage_sink->cropmeta = NULL;
   glimage_sink->prev_cropmeta = NULL;
+  glimage_sink->frame_showed = 0;
+  glimage_sink->run_time = 0;
 
   glimage_sink->mview_output_mode = DEFAULT_MULTIVIEW_MODE;
   glimage_sink->mview_output_flags = DEFAULT_MULTIVIEW_FLAGS;
@@ -1298,7 +1300,10 @@ gst_glimage_sink_change_state (GstElement * element, GstStateChange transition)
 
   switch (transition) {
     case GST_STATE_CHANGE_PLAYING_TO_PAUSED:
+    {
+      glimage_sink->run_time = gst_element_get_start_time (GST_ELEMENT (glimage_sink));
       break;
+    }
     case GST_STATE_CHANGE_PAUSED_TO_READY:
     {
       GstBuffer *buf[2];
@@ -1394,6 +1399,14 @@ gst_glimage_sink_change_state (GstElement * element, GstStateChange transition)
         g_slice_free(GstVideoCropMeta, glimage_sink->prev_cropmeta);
       glimage_sink->prev_cropmeta = NULL;
 
+      if (glimage_sink->run_time > 0) {
+        g_print ("Total showed frames (%" G_GUINT64_FORMAT "), playing for (%"GST_TIME_FORMAT"), fps (%.3f).\n",
+                glimage_sink->frame_showed, GST_TIME_ARGS (glimage_sink->run_time),
+                (gfloat)GST_SECOND * glimage_sink->frame_showed / glimage_sink->run_time);
+      }
+
+      glimage_sink->frame_showed = 0;
+      glimage_sink->run_time = 0;
       break;
     default:
       break;
@@ -1921,6 +1934,8 @@ gst_glimage_sink_show_frame (GstVideoSink * vsink, GstBuffer * buf)
     return GST_FLOW_ERROR;
   }
 
+  glimage_sink->frame_showed++;
+
   return GST_FLOW_OK;
 
 /* ERRORS */
diff --git a/ext/gl/gstglimagesink.h b/ext/gl/gstglimagesink.h
index a6f36190a..1b25666da 100644
--- a/ext/gl/gstglimagesink.h
+++ b/ext/gl/gstglimagesink.h
@@ -132,6 +132,10 @@ struct _GstGLImageSink
     GstVideoOrientationMethod current_rotate_method;
     GstVideoOrientationMethod rotate_method;
     const gfloat *transform_matrix;
+
+    /* fps print support */
+    guint64 frame_showed;
+    GstClockTime run_time;
 };
 
 struct _GstGLImageSinkClass
-- 
2.34.1


From ca9e41435edb75a1c3c75ec28ecb9d51b507ad16 Mon Sep 17 00:00:00 2001
From: Haihua Hu <b55597@freescale.com>
Date: Thu, 25 Feb 2016 13:53:20 +0800
Subject: [PATCH 19/93] glcolorconvert: convert YUV to RGB use directviv

Add a property "disable_passthrough" in glcolorconvert for enable/disable passthrough.
When need convert YUV to RGB with directviv, set it to be TRUE.

Upstream Status: Inappropriate [i.MX specific]

Signed-off-by: Haihua Hu <b55597@freescale.com>

(cherry picked from commit ab2d96e1e891dfc1d584b73f2fd458c7c7e4bfa6)
---
 ext/gl/gstglcolorconvertelement.c   | 76 +++++++++++++++++++++++++++++
 ext/gl/gstglcolorconvertelement.h   |  2 +
 gst-libs/gst/gl/gstglcolorconvert.c |  6 ++-
 3 files changed, 83 insertions(+), 1 deletion(-)

diff --git a/ext/gl/gstglcolorconvertelement.c b/ext/gl/gstglcolorconvertelement.c
index f4b317644..13d62b303 100644
--- a/ext/gl/gstglcolorconvertelement.c
+++ b/ext/gl/gstglcolorconvertelement.c
@@ -40,8 +40,18 @@ G_DEFINE_TYPE_WITH_CODE (GstGLColorConvertElement, gst_gl_color_convert_element,
 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (glcolorconvert, "glcolorconvert",
     GST_RANK_NONE, GST_TYPE_GL_COLOR_CONVERT_ELEMENT, gl_element_init (plugin));
 
+enum
+{
+  GL_COLOR_CONVERT_PROP_0,
+  GL_COLOR_CONVERT_PROP_DISABLE_PASSTHROUGH
+};
+
+#define DISABLE_PASSTHROUGH_DAFAULT FALSE
+
 static gboolean gst_gl_color_convert_element_gl_set_caps (GstGLBaseFilter *
     base_filter, GstCaps * in_caps, GstCaps * out_caps);
+static gboolean gst_gl_color_convert_element_set_caps (GstBaseTransform * bt,
+    GstCaps * in_caps, GstCaps * out_caps);
 static GstCaps *gst_gl_color_convert_element_transform_caps (GstBaseTransform *
     bt, GstPadDirection direction, GstCaps * caps, GstCaps * filter);
 static gboolean gst_gl_color_convert_element_get_unit_size (GstBaseTransform *
@@ -61,6 +71,11 @@ static GstStateChangeReturn
 gst_gl_color_convert_element_change_state (GstElement * element,
     GstStateChange transition);
 
+static void gst_gl_color_convert_set_property (GObject * object,
+    guint prop_id, const GValue * value, GParamSpec * pspec);
+static void gst_gl_color_convert_get_property (GObject * object,
+    guint prop_id, GValue * value, GParamSpec * pspec);
+
 static GstStaticPadTemplate gst_gl_color_convert_element_src_pad_template =
 GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
@@ -92,8 +107,13 @@ gst_gl_color_convert_element_class_init (GstGLColorConvertElementClass * klass)
   GstGLBaseFilterClass *filter_class = GST_GL_BASE_FILTER_CLASS (klass);
   GstBaseTransformClass *bt_class = GST_BASE_TRANSFORM_CLASS (klass);
   GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
+  GObjectClass *object_class = G_OBJECT_CLASS (klass);
+
+  object_class->set_property = gst_gl_color_convert_set_property;
+  object_class->get_property = gst_gl_color_convert_get_property;
 
   bt_class->transform_caps = gst_gl_color_convert_element_transform_caps;
+  bt_class->set_caps = gst_gl_color_convert_element_set_caps;
   bt_class->get_unit_size = gst_gl_color_convert_element_get_unit_size;
   bt_class->filter_meta = gst_gl_color_convert_element_filter_meta;
   bt_class->decide_allocation = gst_gl_color_convert_element_decide_allocation;
@@ -111,6 +131,12 @@ gst_gl_color_convert_element_class_init (GstGLColorConvertElementClass * klass)
   gst_element_class_add_static_pad_template (element_class,
       &gst_gl_color_convert_element_sink_pad_template);
 
+  g_object_class_install_property (object_class,
+      GL_COLOR_CONVERT_PROP_DISABLE_PASSTHROUGH,
+      g_param_spec_boolean ("disable_passthrough", "Disable passthrough",
+          "Disable passthrough mode", DISABLE_PASSTHROUGH_DAFAULT,
+          G_PARAM_READWRITE));
+
   gst_element_class_set_metadata (element_class,
       "OpenGL color converter", "Filter/Converter/Video",
       "Converts between color spaces using OpenGL shaders",
@@ -125,6 +151,56 @@ gst_gl_color_convert_element_init (GstGLColorConvertElement * convert)
 {
   gst_base_transform_set_prefer_passthrough (GST_BASE_TRANSFORM (convert),
       TRUE);
+  convert->disable_passthrough = FALSE;
+}
+
+static void
+gst_gl_color_convert_set_property (GObject * object,
+    guint prop_id, const GValue * value, GParamSpec * pspec)
+{
+  GstGLColorConvertElement *convert = GST_GL_COLOR_CONVERT_ELEMENT (object);
+  switch (prop_id) {
+    case GL_COLOR_CONVERT_PROP_DISABLE_PASSTHROUGH:
+      convert->disable_passthrough = g_value_get_boolean (value);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+gst_gl_color_convert_get_property (GObject * object,
+    guint prop_id, GValue * value, GParamSpec * pspec)
+{
+  GstGLColorConvertElement *convert = GST_GL_COLOR_CONVERT_ELEMENT (object);
+  switch (prop_id) {
+    case GL_COLOR_CONVERT_PROP_DISABLE_PASSTHROUGH:
+      g_value_set_boolean (value, convert->disable_passthrough);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static gboolean
+gst_gl_color_convert_element_set_caps (GstBaseTransform * bt,
+    GstCaps * in_caps, GstCaps * out_caps)
+{
+  GstGLColorConvertElement *convert =
+      GST_GL_COLOR_CONVERT_ELEMENT (bt);
+
+  GST_BASE_TRANSFORM_CLASS (parent_class)->set_caps (bt, in_caps, out_caps);
+
+  if (gst_base_transform_is_passthrough (bt) && convert->disable_passthrough) {
+    /* if in passthrough mode and disable_passthrough is set to true, 
+     * set passthrough to FALSE*/
+    GST_DEBUG_OBJECT (convert, "Disable passthrough mode");
+    gst_base_transform_set_passthrough (bt, FALSE);
+  }
+
+  return TRUE;
 }
 
 static gboolean
diff --git a/ext/gl/gstglcolorconvertelement.h b/ext/gl/gstglcolorconvertelement.h
index 211871a1b..187b8731b 100644
--- a/ext/gl/gstglcolorconvertelement.h
+++ b/ext/gl/gstglcolorconvertelement.h
@@ -45,6 +45,8 @@ struct _GstGLColorConvertElement
   GstGLBaseFilter        parent;
 
   GstGLColorConvert *convert;
+
+  gboolean disable_passthrough;
 };
 
 struct _GstGLColorConvertElementClass
diff --git a/gst-libs/gst/gl/gstglcolorconvert.c b/gst-libs/gst/gl/gstglcolorconvert.c
index 99a080396..7f37c2501 100644
--- a/gst-libs/gst/gl/gstglcolorconvert.c
+++ b/gst-libs/gst/gl/gstglcolorconvert.c
@@ -909,7 +909,11 @@ _gst_gl_color_convert_set_caps_unlocked (GstGLColorConvert * convert,
   convert->priv->to_texture_target = to_target;
   convert->initted = FALSE;
 
-  convert->passthrough = passthrough;
+  /* We may disable passthrough via an external property
+   * By the way, when glconvertelement is in passthrough mode, 
+   * the plugin will not call gst_gl_color_convert_perform().*/
+
+  //convert->passthrough = passthrough;
 #ifndef GST_DISABLE_GST_DEBUG
   if (G_UNLIKELY (convert->passthrough))
     GST_DEBUG_OBJECT (convert,
-- 
2.34.1


From 6912a378c654a0070fd3e2ec5fadee5c2367e72b Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Wed, 13 Sep 2017 16:42:21 +0800
Subject: [PATCH 20/93] glupload: add crop meta support in dmafd uploader

get video crop meta from input buffer and update video info

upstream status: pending
https://bugzilla.gnome.org/show_bug.cgi?id=787616

(cherry picked from commit 8557c5e1ef98aa81118fed32802a20349ada00c8)
---
 gst-libs/gst/gl/gstglupload.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index d91898a1f..ee41a5cfb 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -1445,6 +1445,7 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
   GstVideoInfo *out_info = &dmabuf->out_info;
   guint n_planes;
   GstVideoMeta *meta;
+  GstVideoCropMeta *crop;
   guint n_mem;
   GstMemory *mems[GST_VIDEO_MAX_PLANES];
   GstMemory *previous_mem = NULL;
@@ -1455,6 +1456,7 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
 
   n_mem = gst_buffer_n_memory (buffer);
   meta = gst_buffer_get_video_meta (buffer);
+  crop = gst_buffer_get_video_crop_meta(buffer);
 
   if (!dmabuf->upload->context->gl_vtable->EGLImageTargetTexture2D)
     return FALSE;
@@ -1526,6 +1528,15 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
       in_info->stride[i] = meta->stride[i];
     }
   }
+  
+  if (crop) {
+    in_info->width = MIN (crop->width, in_info->width);
+    in_info->height = MIN (crop->height, in_info->height);
+
+    GST_DEBUG_OBJECT (dmabuf->upload, "got crop meta (%d)x(%d)",
+        in_info->width, in_info->height);
+    gst_buffer_remove_meta (buffer, (GstMeta *)crop);
+  }
 
   /* We cannot have multiple dmabuf per plane */
   if (n_mem > n_planes) {
-- 
2.34.1


From f32f9e74b7eeabf238ded09129498634a297947c Mon Sep 17 00:00:00 2001
From: Song Bing <bing.song@nxp.com>
Date: Fri, 23 Mar 2018 15:13:55 -0700
Subject: [PATCH 21/93] allocators: Add GstDmabufMeta

This meta can be added to the allocation query to indicate support
for dmabuf memory.  Optionally the drm_modifier can also be set in
the allocation query to indicate that the element supports some
certain set of drm modifiers (swizzling/tiling/compression/etc).
When the meta is attached to a buffer, the drm_modifier should
only be one which the src and sink both indicated support for in
the allocation query.

https://bugzilla.gnome.org/show_bug.cgi?id=779146

Conflicts:
	gst-libs/gst/allocators/meson.build
(cherry picked from commit 243e5bb8210bd4133bfcceab205680bb018e4348)
---
 gst-libs/gst/allocators/allocators.h    |   1 +
 gst-libs/gst/allocators/gstdmabufmeta.c | 161 ++++++++++++++++++++++++
 gst-libs/gst/allocators/gstdmabufmeta.h |  62 +++++++++
 gst-libs/gst/allocators/meson.build     |   2 +
 4 files changed, 226 insertions(+)
 create mode 100644 gst-libs/gst/allocators/gstdmabufmeta.c
 create mode 100644 gst-libs/gst/allocators/gstdmabufmeta.h

diff --git a/gst-libs/gst/allocators/allocators.h b/gst-libs/gst/allocators/allocators.h
index 48676e36c..09391718b 100644
--- a/gst-libs/gst/allocators/allocators.h
+++ b/gst-libs/gst/allocators/allocators.h
@@ -29,6 +29,7 @@
 #include <gst/allocators/gstphysmemory.h>
 #include <gst/allocators/gstdrmdumb.h>
 #include <gst/allocators/gstshmallocator.h>
+#include <gst/allocators/gstdmabufmeta.h>
 
 #endif /* __GST_ALLOCATORS_H__ */
 
diff --git a/gst-libs/gst/allocators/gstdmabufmeta.c b/gst-libs/gst/allocators/gstdmabufmeta.c
new file mode 100644
index 000000000..afd63a5d2
--- /dev/null
+++ b/gst-libs/gst/allocators/gstdmabufmeta.c
@@ -0,0 +1,161 @@
+/*
+ * GStreamer
+ * Copyright (C) 2017 Intel Corporation
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+/**
+ * SECTION:gstdmabufmeta
+ * @short_description: dmabuf metadata
+ * @see_also: #GstDmaBufAllocator
+ *
+ * #GstDmabufMeta carries metadata that goes along with
+ * dmabuf memory in the buffer, like drm modifier.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "gstdmabufmeta.h"
+
+/**
+ * gst_buffer_add_dmabuf_meta:
+ * @buffer: a #GstBuffer
+ * @modifier: the drm modifier
+ *
+ * Returns: (transfer none): the #GstDmabufMeta added to #GstBuffer
+ *
+ * Since: 1.12
+ */
+GstDmabufMeta *
+gst_buffer_add_dmabuf_meta (GstBuffer * buffer, guint64 drm_modifier)
+{
+  GstDmabufMeta *meta;
+
+  meta =
+      (GstDmabufMeta *) gst_buffer_add_meta ((buffer),
+      GST_DMABUF_META_INFO, NULL);
+
+  if (!meta)
+    return NULL;
+
+  meta->drm_modifier = drm_modifier;
+
+  return meta;
+}
+
+static gboolean
+gst_dmabuf_meta_transform (GstBuffer * dest, GstMeta * meta,
+    GstBuffer * buffer, GQuark type, gpointer data)
+{
+  GstDmabufMeta *dmeta, *smeta;
+
+  smeta = (GstDmabufMeta *) meta;
+
+  if (GST_META_TRANSFORM_IS_COPY (type)) {
+    GstMetaTransformCopy *copy = data;
+
+    if (!copy->region) {
+      /* only copy if the complete data is copied as well */
+      dmeta = gst_buffer_add_dmabuf_meta (dest, smeta->drm_modifier);
+      if (!dmeta)
+        return FALSE;
+    }
+  } else {
+    /* return FALSE, if transform type is not supported */
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+static void
+gst_dmabuf_meta_free (GstMeta * meta, GstBuffer * buffer)
+{
+  ((GstDmabufMeta *) meta)->drm_modifier = 0;
+
+  return;
+}
+
+static gboolean
+gst_dmabuf_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer)
+{
+  ((GstDmabufMeta *) meta)->drm_modifier = 0;
+
+  return TRUE;
+}
+
+GType
+gst_dmabuf_meta_api_get_type (void)
+{
+  static GType type = 0;
+  static const gchar *tags[] = { NULL };
+
+  if (g_once_init_enter (&type)) {
+    GType _type = gst_meta_api_type_register ("GstDmabufMetaAPI", tags);
+    g_once_init_leave (&type, _type);
+  }
+
+  return type;
+}
+
+const GstMetaInfo *
+gst_dmabuf_meta_get_info (void)
+{
+  static const GstMetaInfo *meta_info = NULL;
+
+  if (g_once_init_enter (&meta_info)) {
+    const GstMetaInfo *meta = gst_meta_register (GST_DMABUF_META_API_TYPE,
+        "GstDmabufMeta",
+        sizeof (GstDmabufMeta), gst_dmabuf_meta_init,
+        gst_dmabuf_meta_free,
+        gst_dmabuf_meta_transform);
+    g_once_init_leave (&meta_info, meta);
+  }
+
+  return meta_info;
+}
+
+void
+gst_query_add_allocation_dmabuf_meta (GstQuery * query, guint64 drm_modifier)
+{
+  guint index;
+  GstStructure *params;
+
+  if (!gst_query_find_allocation_meta (query, GST_DMABUF_META_API_TYPE, &index)) {
+    gchar *str =
+        g_strdup_printf ("GstDmabufMeta, dmabuf.drm_modifier=(guint64){ %"
+        G_GUINT64_FORMAT " };", drm_modifier);
+
+    params = gst_structure_new_from_string (str);
+    g_free (str);
+
+    gst_query_add_allocation_meta (query, GST_DMABUF_META_API_TYPE, params);
+    gst_structure_free (params);
+  } else {
+    GValue newlist = G_VALUE_INIT, drm_modifier_value = G_VALUE_INIT;
+
+    gst_query_parse_nth_allocation_meta (query, index,
+        (const GstStructure **) &params);
+    g_value_init (&drm_modifier_value, G_TYPE_UINT64);
+    g_value_set_uint64 (&drm_modifier_value, drm_modifier);
+    gst_value_list_merge (&newlist, gst_structure_get_value (params,
+            "dmabuf.drm_modifier"), &drm_modifier_value);
+    gst_structure_take_value (params, "dmabuf.drm_modifier", &newlist);
+  }
+}
diff --git a/gst-libs/gst/allocators/gstdmabufmeta.h b/gst-libs/gst/allocators/gstdmabufmeta.h
new file mode 100644
index 000000000..8c0d40c52
--- /dev/null
+++ b/gst-libs/gst/allocators/gstdmabufmeta.h
@@ -0,0 +1,62 @@
+/*
+ * GStreamer
+ * Copyright (C) 2017 Intel Corporation
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __GST_DMABUF_META_H__
+#define __GST_DMABUF_META_H__
+
+#include <gst/gststructure.h>
+#include <gst/gstmeta.h>
+#include <gst/gstquery.h>
+#include <gst/gstvalue.h>
+#include <gst/allocators/allocators-prelude.h>
+
+G_BEGIN_DECLS
+
+#define GST_DMABUF_META_API_TYPE (gst_dmabuf_meta_api_get_type())
+#define GST_DMABUF_META_INFO     (gst_dmabuf_meta_get_info())
+typedef struct _GstDmabufMeta GstDmabufMeta;
+
+/**
+ * GstDmabufMeta:
+ * @parent: the parent #GstMeta
+ * @modifier: DRM modifier
+ */
+struct _GstDmabufMeta
+{
+  GstMeta parent;
+
+  guint64 drm_modifier;
+};
+
+GST_ALLOCATORS_API
+GType gst_dmabuf_meta_api_get_type (void);
+GST_ALLOCATORS_API
+const GstMetaInfo *gst_dmabuf_meta_get_info (void);
+
+#define gst_buffer_get_dmabuf_meta(b) ((GstDmabufMeta*)gst_buffer_get_meta((b),GST_DMABUF_META_API_TYPE))
+
+GST_ALLOCATORS_API
+GstDmabufMeta * gst_buffer_add_dmabuf_meta (GstBuffer * buffer, guint64 drm_modifier);
+
+GST_ALLOCATORS_API
+void gst_query_add_allocation_dmabuf_meta (GstQuery * query, guint64 drm_modifier);
+
+G_END_DECLS
+#endif /* __GST_DMABUF_META_H__ */
diff --git a/gst-libs/gst/allocators/meson.build b/gst-libs/gst/allocators/meson.build
index 5180f6c77..84d66f769 100644
--- a/gst-libs/gst/allocators/meson.build
+++ b/gst-libs/gst/allocators/meson.build
@@ -6,6 +6,7 @@ gst_allocators_headers = files([
   'gstdmabuf.h',
   'gstdrmdumb.h',
   'gstshmallocator.h',
+  'gstdmabufmeta.h',
 ])
 install_headers(gst_allocators_headers, subdir : 'gstreamer-1.0/gst/allocators/')
 
@@ -15,6 +16,7 @@ gst_allocators_sources = files([
   'gstfdmemory.c',
   'gstphysmemory.c',
   'gstshmallocator.c',
+  'gstdmabufmeta.c',
 ])
 
 gst_allocators_cargs = [
-- 
2.34.1


From 0bf95e2c153e6ebcd9313225b6d24ed4d264d7ed Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@nxp.com>
Date: Mon, 2 Apr 2018 16:20:01 +0800
Subject: [PATCH 22/93] MMFMWK-7895 move physical meta to base

move to base since 1.14

Signed-off-by: Song Bing <bing.song@nxp.com>
(cherry picked from commit 95a023a28a653dbf024fe024c2bf2e91ea236e52)
---
 gst-libs/gst/allocators/gstphymemmeta.c | 105 ++++++++++++++++++++++++
 gst-libs/gst/allocators/gstphymemmeta.h |  58 +++++++++++++
 gst-libs/gst/allocators/meson.build     |   4 +-
 3 files changed, 166 insertions(+), 1 deletion(-)
 create mode 100644 gst-libs/gst/allocators/gstphymemmeta.c
 create mode 100644 gst-libs/gst/allocators/gstphymemmeta.h

diff --git a/gst-libs/gst/allocators/gstphymemmeta.c b/gst-libs/gst/allocators/gstphymemmeta.c
new file mode 100644
index 000000000..18e774d48
--- /dev/null
+++ b/gst-libs/gst/allocators/gstphymemmeta.c
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2014, Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "gstphymemmeta.h"
+
+GST_DEBUG_CATEGORY_STATIC (phy_mem_meta_debug);
+#define GST_CAT_DEFAULT phy_mem_meta_debug
+
+static gboolean
+gst_phy_mem_meta_transform (GstBuffer * dest, GstMeta * meta,
+    GstBuffer * buffer, GQuark type, gpointer data)
+{
+  GstPhyMemMeta *dmeta = NULL;
+  GstPhyMemMeta *smeta = NULL;
+
+  if (GST_META_TRANSFORM_IS_COPY (type)) {
+    smeta = (GstPhyMemMeta *) meta;
+    dmeta = GST_PHY_MEM_META_ADD (dest);
+
+    GST_DEBUG ("copy phy metadata");
+
+    dmeta->x_padding = smeta->x_padding;
+    dmeta->y_padding = smeta->y_padding;
+  } else if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type)) {
+    GstVideoMetaTransform *trans = data;
+    gint ow, oh, nw, nh;
+
+    smeta = (GstPhyMemMeta *) meta;
+    dmeta = GST_PHY_MEM_META_ADD (dest);
+
+    ow = GST_VIDEO_INFO_WIDTH (trans->in_info);
+    nw = GST_VIDEO_INFO_WIDTH (trans->out_info);
+    oh = GST_VIDEO_INFO_HEIGHT (trans->in_info);
+    nh = GST_VIDEO_INFO_HEIGHT (trans->out_info);
+
+    GST_DEBUG ("scaling phy metadata %dx%d -> %dx%d", ow, oh, nw, nh);
+
+    dmeta->x_padding = (smeta->x_padding * nw) / ow;
+    dmeta->y_padding = (smeta->y_padding * nh) / oh;
+  }
+
+  dmeta->rfc_luma_offset = smeta->rfc_luma_offset;
+  dmeta->rfc_chroma_offset = smeta->rfc_chroma_offset;
+
+  return TRUE;
+}
+
+GType
+gst_phy_mem_meta_api_get_type (void)
+{
+  static GType type = 0;
+  static const gchar *tags[] =
+      { GST_META_TAG_VIDEO_STR, GST_META_TAG_VIDEO_SIZE_STR,
+    GST_META_TAG_VIDEO_ORIENTATION_STR, NULL
+  };
+
+  if (g_once_init_enter (&type)) {
+    GType _type = gst_meta_api_type_register ("GstPhyMemMetaAPI", tags);
+    g_once_init_leave (&type, _type);
+  }
+  return type;
+}
+
+static gboolean
+gst_phy_mem_meta_init (GstMeta * meta, gpointer params, GstBuffer * buf)
+{
+  return TRUE;
+}
+
+const GstMetaInfo *
+gst_phy_mem_meta_get_info (void)
+{
+  static const GstMetaInfo *phy_mem_meta_info = NULL;
+
+  if (g_once_init_enter (&phy_mem_meta_info)) {
+    const GstMetaInfo *meta =
+        gst_meta_register (GST_PHY_MEM_META_API_TYPE, "GstPhyMemMeta",
+        sizeof (GstPhyMemMeta), (GstMetaInitFunction) gst_phy_mem_meta_init,
+        (GstMetaFreeFunction) NULL, gst_phy_mem_meta_transform);
+    GST_DEBUG_CATEGORY_INIT (phy_mem_meta_debug, "phymemmeta", 0,
+        "Freescale physical memory meta");
+    g_once_init_leave (&phy_mem_meta_info, meta);
+  }
+  return phy_mem_meta_info;
+}
diff --git a/gst-libs/gst/allocators/gstphymemmeta.h b/gst-libs/gst/allocators/gstphymemmeta.h
new file mode 100644
index 000000000..54f80ae48
--- /dev/null
+++ b/gst-libs/gst/allocators/gstphymemmeta.h
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2014, Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __GST_PHY_MEM_META_H__ 
+#define __GST_PHY_MEM_META_H__
+
+#include <gst/gst.h>
+#include <gst/video/video.h>
+#include <gst/video/gstvideometa.h>
+#include <gst/allocators/allocators-prelude.h>
+#include "gstphymemmeta.h"
+
+G_BEGIN_DECLS
+
+typedef struct _GstPhyMemMeta GstPhyMemMeta;
+
+#define GST_PHY_MEM_META_API_TYPE  (gst_phy_mem_meta_api_get_type())
+#define GST_PHY_MEM_META_INFO  (gst_phy_mem_meta_get_info())
+
+#define GST_PHY_MEM_META_GET(buffer)      ((GstPhyMemMeta *)gst_buffer_get_meta((buffer), gst_phy_mem_meta_api_get_type()))
+#define GST_PHY_MEM_META_ADD(buffer)      ((GstPhyMemMeta *)gst_buffer_add_meta((buffer), gst_phy_mem_meta_get_info(), NULL))
+#define GST_PHY_MEM_META_DEL(buffer)      (gst_buffer_remove_meta((buffer), gst_buffer_get_meta((buffer), gst_phy_mem_meta_api_get_type())))
+
+struct _GstPhyMemMeta
+{
+  GstMeta meta;
+  guint x_padding;
+  guint y_padding;
+  guint rfc_luma_offset;
+  guint rfc_chroma_offset;
+};
+
+GST_ALLOCATORS_API
+GType gst_phy_mem_meta_api_get_type(void);
+
+GST_ALLOCATORS_API
+GstMetaInfo const * gst_phy_mem_meta_get_info(void);
+
+G_END_DECLS
+
+#endif
+
diff --git a/gst-libs/gst/allocators/meson.build b/gst-libs/gst/allocators/meson.build
index 84d66f769..020122082 100644
--- a/gst-libs/gst/allocators/meson.build
+++ b/gst-libs/gst/allocators/meson.build
@@ -7,6 +7,7 @@ gst_allocators_headers = files([
   'gstdrmdumb.h',
   'gstshmallocator.h',
   'gstdmabufmeta.h',
+  'gstphymemmeta.h',
 ])
 install_headers(gst_allocators_headers, subdir : 'gstreamer-1.0/gst/allocators/')
 
@@ -17,6 +18,7 @@ gst_allocators_sources = files([
   'gstphysmemory.c',
   'gstshmallocator.c',
   'gstdmabufmeta.c',
+  'gstphymemmeta.c',
 ])
 
 gst_allocators_cargs = [
@@ -44,7 +46,7 @@ gstallocators = library('gstallocators-@0@'.format(api_version),
   soversion : soversion,
   darwin_versions : osxversion,
   install : true,
-  dependencies : [libdrm_dep, gst_dep],
+  dependencies : [libdrm_dep, gst_dep, video_dep],
 )
 
 pkg_name = 'gstreamer-allocators-1.0'
-- 
2.34.1


From 145f9ce50b273f10a904d52b9e5dbc74af3be5b0 Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@nxp.com>
Date: Tue, 10 Apr 2018 16:06:14 +0800
Subject: [PATCH 23/93] Add imx physical memory allocator

Upstream status: [i.MX specific]

Signed-off-by: Haihua Hu <jared.hu@nxp.com>

Conflicts:
	gst-libs/gst/allocators/meson.build
(cherry picked from commit 4cf3366a1c1f3ba4c5e5f2ad246e9bc49eb80565)
---
 gst-libs/gst/allocators/allocators.h         |   1 +
 gst-libs/gst/allocators/gstallocatorphymem.c | 339 +++++++++++++++++++
 gst-libs/gst/allocators/gstallocatorphymem.h |  72 ++++
 gst-libs/gst/allocators/meson.build          |   2 +
 4 files changed, 414 insertions(+)
 create mode 100755 gst-libs/gst/allocators/gstallocatorphymem.c
 create mode 100755 gst-libs/gst/allocators/gstallocatorphymem.h

diff --git a/gst-libs/gst/allocators/allocators.h b/gst-libs/gst/allocators/allocators.h
index 09391718b..356ae3cd5 100644
--- a/gst-libs/gst/allocators/allocators.h
+++ b/gst-libs/gst/allocators/allocators.h
@@ -30,6 +30,7 @@
 #include <gst/allocators/gstdrmdumb.h>
 #include <gst/allocators/gstshmallocator.h>
 #include <gst/allocators/gstdmabufmeta.h>
+#include <gst/allocators/gstallocatorphymem.h>
 
 #endif /* __GST_ALLOCATORS_H__ */
 
diff --git a/gst-libs/gst/allocators/gstallocatorphymem.c b/gst-libs/gst/allocators/gstallocatorphymem.c
new file mode 100755
index 000000000..75f976381
--- /dev/null
+++ b/gst-libs/gst/allocators/gstallocatorphymem.c
@@ -0,0 +1,339 @@
+/*
+ * Copyright (c) 2013-2015, Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include "gstallocatorphymem.h"
+#include "gstphysmemory.h"
+
+typedef struct
+{
+  GstMemory mem;
+  guint8 *vaddr;
+  guint8 *paddr;
+  PhyMemBlock block;
+} GstMemoryPhy;
+
+static int
+default_copy (GstAllocatorPhyMem * allocator, PhyMemBlock * dst_mem,
+    PhyMemBlock * src_mem, guint offset, guint size)
+{
+  GST_WARNING
+      ("No default copy implementation for physical memory allocator.\n");
+  return -1;
+}
+
+static gpointer
+gst_phymem_map (GstMemory * mem, gsize maxsize, GstMapFlags flags)
+{
+  GstMemoryPhy *phymem = (GstMemoryPhy *) mem;
+
+  if (GST_MEMORY_IS_READONLY (mem) && (flags & GST_MAP_WRITE)) {
+    GST_ERROR ("memory is read only");
+    return NULL;
+  }
+
+  return phymem->vaddr;
+}
+
+static void
+gst_phymem_unmap (GstMemory * mem)
+{
+  return;
+}
+
+static GstMemory *
+gst_phymem_copy (GstMemory * mem, gssize offset, gssize size)
+{
+  GstAllocatorPhyMemClass *klass;
+  GstMemoryPhy *src_mem = (GstMemoryPhy *) mem;
+
+  GstMemoryPhy *dst_mem = g_slice_alloc (sizeof (GstMemoryPhy));
+  if (dst_mem == NULL) {
+    GST_ERROR ("Can't allocate for GstMemoryPhy structure.\n");
+    return NULL;
+  }
+
+  klass = GST_ALLOCATOR_PHYMEM_CLASS (G_OBJECT_GET_CLASS (mem->allocator));
+  if (klass == NULL) {
+    GST_ERROR ("Can't get class from allocator object.\n");
+    return NULL;
+  }
+
+  if (klass->copy_phymem ((GstAllocatorPhyMem *) mem->allocator,
+          &dst_mem->block, &src_mem->block, offset, size) < 0) {
+    GST_WARNING ("Copy phymem %" G_GSSIZE_FORMAT " failed.\n", size);
+    return NULL;
+  }
+
+  GST_DEBUG ("copied phymem, vaddr(%p), paddr(%p), size(%" G_GSIZE_FORMAT ").\n",
+      dst_mem->block.vaddr, dst_mem->block.paddr, dst_mem->block.size);
+
+  dst_mem->vaddr = dst_mem->block.vaddr;
+  dst_mem->paddr = dst_mem->block.paddr;
+
+  gst_memory_init (GST_MEMORY_CAST (dst_mem),
+      mem->mini_object.flags & (~GST_MEMORY_FLAG_READONLY),
+      mem->allocator, NULL, mem->maxsize, mem->align, mem->offset, mem->size);
+
+  return (GstMemory *) dst_mem;
+}
+
+static GstMemory *
+gst_phymem_share (GstMemory * mem, gssize offset, gssize size)
+{
+  GST_ERROR ("Not implemented mem_share in gstallocatorphymem.\n");
+  return NULL;
+}
+
+static gboolean
+gst_phymem_is_span (GstMemory * mem1, GstMemory * mem2, gsize * offset)
+{
+  return FALSE;
+}
+
+static gpointer
+gst_phymem_get_phy (GstMemory * mem)
+{
+  GstMemoryPhy *phymem = (GstMemoryPhy *) mem;
+
+  return phymem->paddr;
+}
+
+static GstMemory *
+base_alloc (GstAllocator * allocator, gsize size, GstAllocationParams * params)
+{
+  GstAllocatorPhyMemClass *klass;
+  GstMemoryPhy *mem;
+  gsize maxsize, aoffset, offset, align, padding;
+  guint8 *data;
+
+  mem = g_slice_alloc (sizeof (GstMemoryPhy));
+  if (mem == NULL) {
+    GST_ERROR ("Can allocate for GstMemoryPhy structure.\n");
+    return NULL;
+  }
+
+  klass = GST_ALLOCATOR_PHYMEM_CLASS (G_OBJECT_GET_CLASS (allocator));
+  if (klass == NULL) {
+    GST_ERROR ("Can't get class from allocator object.\n");
+    return NULL;
+  }
+
+  GST_DEBUG
+      ("allocate params, prefix (%" G_GSIZE_FORMAT "), padding (%" G_GSIZE_FORMAT "),"
+        "align (%" G_GSIZE_FORMAT "), flags (%x).\n", params->prefix, params->padding,
+        params->align, params->flags);
+
+  maxsize = size + params->prefix + params->padding;
+  mem->block.size = maxsize;
+  if (klass->alloc_phymem ((GstAllocatorPhyMem *) allocator, &mem->block) < 0) {
+    GST_ERROR ("Allocate phymem %" G_GSIZE_FORMAT " failed.\n", maxsize);
+    return NULL;
+  }
+
+  GST_DEBUG ("allocated phymem, vaddr(%p), paddr(%p), size(%" G_GSIZE_FORMAT ").\n",
+      mem->block.vaddr, mem->block.paddr, mem->block.size);
+
+  data = mem->block.vaddr;
+  offset = params->prefix;
+  align = params->align;
+  /* do alignment */
+  if ((aoffset = ((guintptr) data & align))) {
+    aoffset = (align + 1) - aoffset;
+    data += aoffset;
+    maxsize -= aoffset;
+  }
+  mem->vaddr = mem->block.vaddr + aoffset;
+  mem->paddr = mem->block.paddr + aoffset;
+
+  GST_DEBUG ("aligned vaddr(%p), paddr(%p), size(%" G_GSIZE_FORMAT ").\n",
+      mem->block.vaddr, mem->block.paddr, mem->block.size);
+
+  if (offset && (params->flags & GST_MEMORY_FLAG_ZERO_PREFIXED))
+    memset (data, 0, offset);
+
+  padding = maxsize - (offset + size);
+  if (padding && (params->flags & GST_MEMORY_FLAG_ZERO_PADDED))
+    memset (data + offset + size, 0, padding);
+
+  gst_memory_init (GST_MEMORY_CAST (mem), params->flags, allocator, NULL,
+      maxsize, align, offset, size);
+
+  return (GstMemory *) mem;
+}
+
+static void
+base_free (GstAllocator * allocator, GstMemory * mem)
+{
+  GstAllocatorPhyMemClass *klass;
+  GstMemoryPhy *phymem;
+
+  klass = GST_ALLOCATOR_PHYMEM_CLASS (G_OBJECT_GET_CLASS (allocator));
+  if (klass == NULL) {
+    GST_ERROR ("Can't get class from allocator object, can't free %p\n", mem);
+    return;
+  }
+
+  phymem = (GstMemoryPhy *) mem;
+
+  GST_DEBUG ("free phymem, vaddr(%p), paddr(%p), size(%" G_GSIZE_FORMAT ").\n",
+      phymem->block.vaddr, phymem->block.paddr, phymem->block.size);
+
+  klass->free_phymem ((GstAllocatorPhyMem *) allocator, &phymem->block);
+  g_slice_free1 (sizeof (GstMemoryPhy), mem);
+
+  return;
+}
+
+static int
+default_alloc (GstAllocatorPhyMem * allocator, PhyMemBlock * phy_mem)
+{
+  GST_ERROR
+      ("No default allocating implementation for physical memory allocation.\n");
+  return -1;
+}
+
+static int
+default_free (GstAllocatorPhyMem * allocator, PhyMemBlock * phy_mem)
+{
+  GST_ERROR
+      ("No default free implementation for physical memory allocation.\n");
+  return -1;
+}
+
+static guintptr
+gst_allocator_phymem_get_phys_addr (GstPhysMemoryAllocator * allocator,
+    GstMemory * mem)
+{
+  return (guintptr) gst_phymem_get_phy (mem);
+}
+
+static void
+gst_allocator_phymem_iface_init (gpointer g_iface)
+{
+  GstPhysMemoryAllocatorInterface *iface = g_iface;
+  iface->get_phys_addr = gst_allocator_phymem_get_phys_addr;
+}
+
+G_DEFINE_TYPE_WITH_CODE (GstAllocatorPhyMem, gst_allocator_phymem,
+    GST_TYPE_ALLOCATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_PHYS_MEMORY_ALLOCATOR,
+        gst_allocator_phymem_iface_init));
+
+static void
+gst_allocator_phymem_class_init (GstAllocatorPhyMemClass * klass)
+{
+  GstAllocatorClass *allocator_class;
+
+  allocator_class = (GstAllocatorClass *) klass;
+
+  allocator_class->alloc = base_alloc;
+  allocator_class->free = base_free;
+  klass->alloc_phymem = default_alloc;
+  klass->free_phymem = default_free;
+  klass->copy_phymem = default_copy;
+}
+
+static void
+gst_allocator_phymem_init (GstAllocatorPhyMem * allocator)
+{
+  GstAllocator *alloc = GST_ALLOCATOR_CAST (allocator);
+
+  alloc->mem_map = gst_phymem_map;
+  alloc->mem_unmap = gst_phymem_unmap;
+  alloc->mem_copy = gst_phymem_copy;
+  alloc->mem_share = gst_phymem_share;
+  alloc->mem_is_span = gst_phymem_is_span;
+}
+
+
+//global functions
+
+gboolean
+gst_buffer_is_phymem (GstBuffer * buffer)
+{
+  gboolean ret = FALSE;
+
+  GstMemory *mem = gst_buffer_get_memory (buffer, 0);
+  if (mem == NULL) {
+    GST_ERROR ("Not get memory from buffer.\n");
+    return FALSE;
+  }
+
+  if (GST_IS_ALLOCATOR_PHYMEM (mem->allocator)) {
+    if (NULL == ((GstMemoryPhy *) mem)->block.paddr) {
+      GST_WARNING ("physical address in memory block is invalid");
+      ret = FALSE;
+    } else {
+      ret = TRUE;
+    }
+  }
+
+  gst_memory_unref (mem);
+
+  return ret;
+}
+
+PhyMemBlock *
+gst_buffer_query_phymem_block (GstBuffer * buffer)
+{
+  GstMemory *mem;
+  GstMemoryPhy *memphy;
+  PhyMemBlock *memblk;
+
+  mem = gst_buffer_get_memory (buffer, 0);
+  if (mem == NULL) {
+    GST_ERROR ("Not get memory from buffer.\n");
+    return NULL;
+  }
+
+  if (!GST_IS_ALLOCATOR_PHYMEM (mem->allocator)) {
+    gst_memory_unref (mem);
+    return NULL;
+  }
+
+  memphy = (GstMemoryPhy *) mem;
+  memblk = &memphy->block;
+
+  gst_memory_unref (mem);
+
+  return memblk;
+}
+
+PhyMemBlock *
+gst_memory_query_phymem_block (GstMemory * mem)
+{
+  GstMemoryPhy *memphy;
+  PhyMemBlock *memblk;
+
+  if (!mem)
+    return NULL;
+
+  if (!GST_IS_ALLOCATOR_PHYMEM (mem->allocator))
+    return NULL;
+
+  memphy = (GstMemoryPhy *) mem;
+  memblk = &memphy->block;
+
+  return memblk;
+}
diff --git a/gst-libs/gst/allocators/gstallocatorphymem.h b/gst-libs/gst/allocators/gstallocatorphymem.h
new file mode 100755
index 000000000..0ced166fa
--- /dev/null
+++ b/gst-libs/gst/allocators/gstallocatorphymem.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2013-2015, Freescale Semiconductor, Inc. All rights reserved.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __ALLOCATOR_PHYMEM_H__
+#define __ALLOCATOR_PHYMEM_H__
+
+#include <gst/gst.h>
+#include <gst/gstallocator.h>
+#include <gst/allocators/allocators-prelude.h>
+
+#define PAGE_ALIGN(x) (((x) + 4095) & ~4095)
+
+#define GST_TYPE_ALLOCATOR_PHYMEM             (gst_allocator_phymem_get_type())
+#define GST_ALLOCATOR_PHYMEM(obj)             (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_ALLOCATOR_PHYMEM, GstAllocatorPhyMem))
+#define GST_ALLOCATOR_PHYMEM_CLASS(klass)     (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_ALLOCATOR_PHYMEM, GstAllocatorPhyMemClass))
+#define GST_IS_ALLOCATOR_PHYMEM(obj)          (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_ALLOCATOR_PHYMEM))
+#define GST_IS_ALLOCATOR_PHYMEM_CLASS(klass)  (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_ALLOCATOR_PHYMEM))
+
+typedef struct _GstAllocatorPhyMem GstAllocatorPhyMem;
+typedef struct _GstAllocatorPhyMemClass GstAllocatorPhyMemClass;
+
+/* also change gst-libs/gst/gl/gstglvivdirecttexture.c in gst-plugins-bad git
+ * if changed below structure */
+typedef struct {
+  guint8 *vaddr;
+  guint8 *paddr;
+  guint8 *caddr;
+  gsize size;
+  gpointer *user_data;
+} PhyMemBlock;
+
+struct _GstAllocatorPhyMem {
+  GstAllocator parent;
+};
+
+struct _GstAllocatorPhyMemClass {
+  GstAllocatorClass parent_class;
+  int (*alloc_phymem) (GstAllocatorPhyMem *allocator, PhyMemBlock *phy_mem);
+  int (*free_phymem) (GstAllocatorPhyMem *allocator, PhyMemBlock *phy_mem);
+  int (*copy_phymem) (GstAllocatorPhyMem *allocator, PhyMemBlock *det_mem,
+                      PhyMemBlock *src_mem, guint offset, guint size);
+};
+
+GST_ALLOCATORS_API
+GType gst_allocator_phymem_get_type (void);
+
+GST_ALLOCATORS_API
+gboolean gst_buffer_is_phymem (GstBuffer *buffer);
+
+GST_ALLOCATORS_API
+PhyMemBlock *gst_buffer_query_phymem_block (GstBuffer *buffer);
+
+GST_ALLOCATORS_API
+PhyMemBlock *gst_memory_query_phymem_block (GstMemory *mem);
+
+#endif
diff --git a/gst-libs/gst/allocators/meson.build b/gst-libs/gst/allocators/meson.build
index 020122082..428ba645e 100644
--- a/gst-libs/gst/allocators/meson.build
+++ b/gst-libs/gst/allocators/meson.build
@@ -8,6 +8,7 @@ gst_allocators_headers = files([
   'gstshmallocator.h',
   'gstdmabufmeta.h',
   'gstphymemmeta.h',
+  'gstallocatorphymem.h',
 ])
 install_headers(gst_allocators_headers, subdir : 'gstreamer-1.0/gst/allocators/')
 
@@ -19,6 +20,7 @@ gst_allocators_sources = files([
   'gstshmallocator.c',
   'gstdmabufmeta.c',
   'gstphymemmeta.c',
+  'gstallocatorphymem.c',
 ])
 
 gst_allocators_cargs = [
-- 
2.34.1


From 492ce439feee3f79079cbdc153ed0e5a5f28e4fe Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@nxp.com>
Date: Wed, 11 Apr 2018 16:55:11 +0800
Subject: [PATCH 24/93] gstimxcommon: Update IMX_GST_PLUGIN_DEFINE define for
 GST1.14

- As macro GST_PLUGIN_DEFINE is upgraded in GST1.14
(change C structure into function
commit e7ede5a487, https://bugzilla.gnome.org/show_bug.cgi?id=779344)

So our macor cannot use "." in name field

-- i.MX specific

Signed-off-by: Lyon Wang <lyon.wang@nxp.com>
(cherry picked from commit 26c5c0be1b4a34520e28dbb16e8fba5ebc8e61f4)
---
 gst-libs/gst/gstimxcommon.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 mode change 100644 => 100755 gst-libs/gst/gstimxcommon.h

diff --git a/gst-libs/gst/gstimxcommon.h b/gst-libs/gst/gstimxcommon.h
old mode 100644
new mode 100755
index e2a0aa295..2d05ca344
--- a/gst-libs/gst/gstimxcommon.h
+++ b/gst-libs/gst/gstimxcommon.h
@@ -41,7 +41,7 @@ extern "C" {
 #define IMX_GST_PLUGIN_DEFINE(name, description, initfunc)\
   GST_PLUGIN_DEFINE(GST_VERSION_MAJOR,\
       GST_VERSION_MINOR,\
-      name.imx,\
+      name, \
       description,\
       initfunc,\
       VERSION,\
-- 
2.34.1


From 95117c917d0fe02080d633c6d8d58ab01052fd94 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Wed, 11 Apr 2018 14:54:52 +0800
Subject: [PATCH 25/93] Accelerate gldownload with directviv

1.Propose a physical buffer pool to upstream in gldownload
2.Bind the physical buffer with texture via dirctviv
3.In gldownload, wrap the physical buffer to gstbuffer, pass to
  downstream plugins.
4.Add some configure check for g2d and phymem

From GST-1.14, add GST_GL_API ahead api to export symbol
add include "gstglfuncs.h" when need call gl api

Upstream-Status: Inappropriate [i.MX specific]

Conflicts:
	gst-libs/gst/gl/meson.build
(cherry picked from commit 2647a29b358520f53abbd7ddeeef32e554878352)
---
 ext/gl/gstgldownloadelement.c       | 184 ++++++++------
 gst-libs/gst/gl/gstglbufferpool.c   |  23 +-
 gst-libs/gst/gl/gstglconfig.h.meson |   1 +
 gst-libs/gst/gl/gstglphymemory.c    | 375 ++++++++++++++++++++++++++++
 gst-libs/gst/gl/gstglphymemory.h    |  49 ++++
 gst-libs/gst/gl/meson.build         |  19 +-
 6 files changed, 567 insertions(+), 84 deletions(-)
 create mode 100644 gst-libs/gst/gl/gstglphymemory.c
 create mode 100644 gst-libs/gst/gl/gstglphymemory.h

diff --git a/ext/gl/gstgldownloadelement.c b/ext/gl/gstgldownloadelement.c
index 1a3bb7eaa..fb545bd17 100644
--- a/ext/gl/gstgldownloadelement.c
+++ b/ext/gl/gstgldownloadelement.c
@@ -32,6 +32,10 @@
 #include "gstglelements.h"
 #include "gstgldownloadelement.h"
 
+#if GST_GL_HAVE_PHYMEM
+#include <gst/gl/gstglphymemory.h>
+#endif
+
 GST_DEBUG_CATEGORY_STATIC (gst_gl_download_element_debug);
 #define GST_CAT_DEFAULT gst_gl_download_element_debug
 
@@ -775,7 +779,8 @@ gst_gl_buffer_pool_nvmm_new (GstGLContext * context)
 G_DEFINE_TYPE_WITH_CODE (GstGLDownloadElement, gst_gl_download_element,
     GST_TYPE_GL_BASE_FILTER,
     GST_DEBUG_CATEGORY_INIT (gst_gl_download_element_debug, "gldownloadelement",
-        0, "download element"););
+        0, "download element");
+    );
 GST_ELEMENT_REGISTER_DEFINE_WITH_CODE (gldownload, "gldownload",
     GST_RANK_NONE, GST_TYPE_GL_DOWNLOAD_ELEMENT, gl_element_init (plugin));
 
@@ -1232,6 +1237,25 @@ gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
   GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
   GstGLSyncMeta *in_sync_meta;
   gint i, n;
+  GstGLMemory *glmem;
+
+#if GST_GL_HAVE_PHYMEM
+  glmem = gst_buffer_peek_memory (inbuf, 0);
+  if (gst_is_gl_physical_memory (glmem)) {
+    GstCaps *src_caps;
+    GstVideoInfo info;
+    GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
+
+    src_caps = gst_pad_get_current_caps (bt->srcpad);
+
+    gst_video_info_from_caps (&info, src_caps);
+    *outbuf = gst_gl_phymem_buffer_to_gstbuffer (context, &info, inbuf);
+
+    GST_DEBUG_OBJECT (dl, "gl download with direct viv.");
+
+    return GST_FLOW_OK;
+  }
+#endif /* GST_GL_HAVE_PHYMEM */
 
   *outbuf = inbuf;
 
@@ -1362,6 +1386,84 @@ gst_gl_download_element_transform_meta (GstBaseTransform * bt,
       meta, inbuf);
 }
 
+static gboolean
+gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
+    GstQuery * decide_query, GstQuery * query)
+{
+  GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
+  GstGLDownloadElement *download = GST_GL_DOWNLOAD_ELEMENT (bt);
+  GstAllocationParams params;
+  GstAllocator *allocator = NULL;
+  GstBufferPool *pool = NULL;
+  guint n_pools, i;
+  GstVideoInfo info;
+  GstCaps *caps;
+  GstStructure *config;
+  gsize size;
+
+  gst_query_parse_allocation (query, &caps, NULL);
+  if (!gst_video_info_from_caps (&info, caps)) {
+    GST_WARNING_OBJECT (bt, "invalid caps specified");
+    return FALSE;
+  }
+
+  GST_DEBUG_OBJECT (bt, "video format is %s",
+      gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&info)));
+
+  gst_allocation_params_init (&params);
+
+#if GST_GL_HAVE_PHYMEM
+  if (gst_is_gl_physical_memory_supported_fmt (&info)) {
+    allocator = gst_phy_mem_allocator_obtain ();
+    GST_DEBUG_OBJECT (bt, "obtain physical memory allocator %p.", allocator);
+  }
+#endif /* GST_GL_HAVE_PHYMEM */
+
+  if (!allocator)
+    allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR_NAME);
+
+  if (!allocator) {
+    GST_ERROR_OBJECT (bt, "Can't obtain gl memory allocator.");
+    return FALSE;
+  }
+
+  gst_query_add_allocation_param (query, allocator, &params);
+  gst_object_unref (allocator);
+
+  n_pools = gst_query_get_n_allocation_pools (query);
+  for (i = 0; i < n_pools; i++) {
+    gst_query_parse_nth_allocation_pool (query, i, &pool, NULL, NULL, NULL);
+    gst_object_unref (pool);
+    pool = NULL;
+  }
+
+  //new buffer pool
+  pool = gst_gl_buffer_pool_new (context);
+  config = gst_buffer_pool_get_config (pool);
+
+  /* the normal size of a frame */
+  size = info.size;
+  gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
+  gst_buffer_pool_config_set_gl_min_free_queue_size (config, 1);
+  gst_buffer_pool_config_add_option (config,
+      GST_BUFFER_POOL_OPTION_GL_SYNC_META);
+
+  if (!gst_buffer_pool_set_config (pool, config)) {
+    gst_object_unref (pool);
+    GST_WARNING_OBJECT (bt, "failed setting config");
+    return FALSE;
+  }
+
+  GST_DEBUG_OBJECT (download, "create pool %p", pool);
+
+  //propose 3 buffers for better performance
+  gst_query_add_allocation_pool (query, pool, size, 3, 0);
+
+  gst_object_unref (pool);
+
+  return TRUE;
+}
+
 static gboolean
 gst_gl_download_element_decide_allocation (GstBaseTransform * trans,
     GstQuery * query)
@@ -1402,86 +1504,6 @@ gst_gl_download_element_src_event (GstBaseTransform * bt, GstEvent * event)
   return GST_BASE_TRANSFORM_CLASS (parent_class)->src_event (bt, event);
 }
 
-static gboolean
-gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
-    GstQuery * decide_query, GstQuery * query)
-{
-  GstBufferPool *pool = NULL;
-  GstCaps *caps;
-  GstGLContext *context;
-  GstStructure *config;
-  GstVideoInfo info;
-  gsize size;
-
-  if (!GST_BASE_TRANSFORM_CLASS (parent_class)->propose_allocation (bt,
-          decide_query, query))
-    return FALSE;
-
-  gst_query_parse_allocation (query, &caps, NULL);
-  if (caps == NULL)
-    goto invalid_caps;
-
-  context = GST_GL_BASE_FILTER (bt)->context;
-  if (!context) {
-    GST_ERROR_OBJECT (context, "got no GLContext");
-    return FALSE;
-  }
-
-  if (!gst_video_info_from_caps (&info, caps))
-    goto invalid_caps;
-
-#if GST_GL_HAVE_PLATFORM_EGL && defined(HAVE_NVMM)
-  if (!pool && decide_query) {
-    GstCaps *decide_caps;
-
-    gst_query_parse_allocation (decide_query, &decide_caps, NULL);
-    if (decide_caps && gst_caps_get_size (decide_caps) > 0) {
-      GstCapsFeatures *features = gst_caps_get_features (decide_caps, 0);
-
-      if (gst_caps_features_contains (features, GST_CAPS_FEATURE_MEMORY_NVMM)) {
-        pool = gst_gl_buffer_pool_nvmm_new (context);
-        GST_INFO_OBJECT (bt, "have NVMM downstream, proposing NVMM "
-            "pool %" GST_PTR_FORMAT, pool);
-      }
-    }
-  }
-#endif
-  if (!pool) {
-    pool = gst_gl_buffer_pool_new (context);
-  }
-  config = gst_buffer_pool_get_config (pool);
-
-  /* the normal size of a frame */
-  size = info.size;
-  gst_buffer_pool_config_set_params (config, caps, size, 0, 0);
-  gst_buffer_pool_config_set_gl_min_free_queue_size (config, 1);
-
-  if (!gst_buffer_pool_set_config (pool, config)) {
-    gst_object_unref (pool);
-    goto config_failed;
-  }
-
-  if (context->gl_vtable->FenceSync)
-    gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, NULL);
-
-  gst_query_add_allocation_pool (query, pool, size, 1, 0);
-
-  gst_object_unref (pool);
-  return TRUE;
-
-invalid_caps:
-  {
-    GST_ERROR_OBJECT (bt, "Invalid Caps specified");
-    return FALSE;
-  }
-config_failed:
-  {
-    GST_ERROR_OBJECT (bt, "failed setting config");
-    return FALSE;
-  }
-}
-
-
 static void
 gst_gl_download_element_finalize (GObject * object)
 {
diff --git a/gst-libs/gst/gl/gstglbufferpool.c b/gst-libs/gst/gl/gstglbufferpool.c
index f0dfda787..08b49f7e0 100644
--- a/gst-libs/gst/gl/gstglbufferpool.c
+++ b/gst-libs/gst/gl/gstglbufferpool.c
@@ -28,6 +28,10 @@
 #include "gstglsyncmeta.h"
 #include "gstglutils.h"
 
+#if GST_GL_HAVE_PHYMEM
+#include <gst/gl/gstglphymemory.h>
+#endif
+
 #define DEFAULT_FREE_QUEUE_MIN_DEPTH 0
 
 /**
@@ -139,7 +143,11 @@ gst_gl_buffer_pool_set_config (GstBufferPool * pool, GstStructure * config)
     gst_object_unref (priv->allocator);
 
   if (allocator) {
-    if (!GST_IS_GL_MEMORY_ALLOCATOR (allocator)) {
+    if (!GST_IS_GL_MEMORY_ALLOCATOR (allocator)
+#if GST_GL_HAVE_PHYMEM
+        && (g_strcmp0 (allocator->mem_type, GST_GL_PHY_MEM_ALLOCATOR) != 0)
+#endif
+        ) {
       gst_object_unref (allocator);
       goto wrong_allocator;
     } else {
@@ -322,11 +330,24 @@ gst_gl_buffer_pool_alloc (GstBufferPool * pool, GstBuffer ** buffer,
   if (!(buf = gst_buffer_new ())) {
     goto no_buffer;
   }
+#if GST_GL_HAVE_PHYMEM
+  if ((g_strcmp0 (priv->allocator->mem_type, GST_GL_PHY_MEM_ALLOCATOR) == 0)) {
+    if (!gst_gl_physical_memory_setup_buffer (priv->allocator, buf,
+            priv->gl_params)) {
+      GST_ERROR_OBJECT (pool, "Can't create physcial buffer.");
+      return GST_FLOW_ERROR;
+    }
+    goto done;
+  }
+#endif
 
   alloc = GST_GL_MEMORY_ALLOCATOR (priv->allocator);
   if (!gst_gl_memory_setup_buffer (alloc, buf, priv->gl_params, NULL, NULL, 0))
     goto mem_create_failed;
 
+#if GST_GL_HAVE_PHYMEM
+done:
+#endif
   if (priv->add_glsyncmeta)
     gst_buffer_add_gl_sync_meta (glpool->context, buf);
 
diff --git a/gst-libs/gst/gl/gstglconfig.h.meson b/gst-libs/gst/gl/gstglconfig.h.meson
index aefd93aa8..9f0b40cee 100644
--- a/gst-libs/gst/gl/gstglconfig.h.meson
+++ b/gst-libs/gst/gl/gstglconfig.h.meson
@@ -33,6 +33,7 @@ G_BEGIN_DECLS
 
 #mesondefine GST_GL_HAVE_DMABUF
 #mesondefine GST_GL_HAVE_VIV_DIRECTVIV
+#mesondefine GST_GL_HAVE_PHYMEM
 
 #mesondefine GST_GL_HAVE_GLEGLIMAGEOES
 #mesondefine GST_GL_HAVE_GLCHAR
diff --git a/gst-libs/gst/gl/gstglphymemory.c b/gst-libs/gst/gl/gstglphymemory.c
new file mode 100644
index 000000000..432b060d6
--- /dev/null
+++ b/gst-libs/gst/gl/gstglphymemory.c
@@ -0,0 +1,375 @@
+/*
+ * GStreamer
+ * Copyright (c) 2015, Freescale Semiconductor, Inc.
+ * Copyright 2017 NXP
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "gstglphymemory.h"
+#include "gstglfuncs.h"
+#include <g2d.h>
+
+GST_DEBUG_CATEGORY_STATIC (GST_CAT_GL_PHY_MEMORY);
+#define GST_CAT_DEFAULT GST_CAT_GL_PHY_MEMORY
+
+#ifndef GL_BGRA_EXT
+#define GL_BGRA_EXT                                             0x80E1
+#endif
+#ifndef GL_VIV_YV12
+#define GL_VIV_YV12                                             0x8FC0
+#endif
+#ifndef GL_VIV_NV12
+#define GL_VIV_NV12                                             0x8FC1
+#endif
+#ifndef GL_VIV_YUY2
+#define GL_VIV_YUY2                                             0x8FC2
+#endif
+#ifndef GL_VIV_UYVY
+#define GL_VIV_UYVY                                             0x8FC3
+#endif
+#ifndef GL_VIV_NV21
+#define GL_VIV_NV21                                             0x8FC4
+#endif
+#ifndef GL_VIV_I420
+#define GL_VIV_I420                                             0x8FC5
+#endif
+
+typedef void (*TexDirectVIVMap) (GLenum Target, GLsizei Width, GLsizei Height,
+    GLenum Format, GLvoid ** Logical, const GLuint * Physical);
+typedef void (*TexDirectInvalidateVIV) (GLenum Target);
+static TexDirectVIVMap pTexDirectVIVMap = NULL;
+static TexDirectInvalidateVIV pTexDirectInvalidateVIV = NULL;
+
+typedef struct
+{
+  guint tex_id;
+  guint w;
+  guint h;
+  guint fmt;
+  void *vaddr;
+  guint paddr;
+  gboolean ret;
+} DirectVIVData;
+
+typedef struct _GstPhyMemAllocator GstPhyMemAllocator;
+typedef struct _GstPhyMemAllocatorClass GstPhyMemAllocatorClass;
+
+struct _GstPhyMemAllocator
+{
+  GstAllocatorPhyMem parent;
+};
+
+struct _GstPhyMemAllocatorClass
+{
+  GstAllocatorPhyMemClass parent_class;
+};
+
+GType gst_phy_mem_allocator_get_type (void);
+G_DEFINE_TYPE (GstPhyMemAllocator, gst_phy_mem_allocator,
+    GST_TYPE_ALLOCATOR_PHYMEM);
+
+static int
+alloc_phymem (GstAllocatorPhyMem * allocator, PhyMemBlock * memblk)
+{
+  struct g2d_buf *pbuf = NULL;
+
+  memblk->size = PAGE_ALIGN (memblk->size);
+
+  pbuf = g2d_alloc (memblk->size, 0);
+  if (!pbuf) {
+    GST_ERROR ("G2D allocate %" G_GSIZE_FORMAT " bytes memory failed: %s",
+        memblk->size, strerror (errno));
+    return -1;
+  }
+
+  memblk->vaddr = (guchar *) pbuf->buf_vaddr;
+  memblk->paddr = (guchar *) (uintptr_t) pbuf->buf_paddr;
+  memblk->user_data = (gpointer) pbuf;
+  GST_DEBUG ("G2D allocated memory (%p)", memblk->paddr);
+
+  return 1;
+}
+
+static int
+free_phymem (GstAllocatorPhyMem * allocator, PhyMemBlock * memblk)
+{
+  GST_DEBUG ("G2D free memory (%p)", memblk->paddr);
+  gint ret = g2d_free ((struct g2d_buf *) (memblk->user_data));
+  memblk->user_data = NULL;
+  memblk->vaddr = NULL;
+  memblk->paddr = NULL;
+  memblk->size = 0;
+
+  return ret;
+}
+
+static void
+gst_phy_mem_allocator_class_init (GstPhyMemAllocatorClass * klass)
+{
+  GstAllocatorPhyMemClass *phy_allocator_klass =
+      (GstAllocatorPhyMemClass *) klass;
+
+  phy_allocator_klass->alloc_phymem = alloc_phymem;
+  phy_allocator_klass->free_phymem = free_phymem;
+}
+
+static void
+gst_phy_mem_allocator_init (GstPhyMemAllocator * allocator)
+{
+  GstAllocator *alloc = GST_ALLOCATOR_CAST (allocator);
+
+  alloc->mem_type = GST_GL_PHY_MEM_ALLOCATOR;
+}
+
+
+static gpointer
+gst_phy_mem_allocator_init_instance (gpointer data)
+{
+  GstAllocator *allocator =
+      g_object_new (gst_phy_mem_allocator_get_type (), NULL);
+
+  GST_DEBUG_CATEGORY_INIT (GST_CAT_GL_PHY_MEMORY, "glphymemory", 0,
+      "GLPhysical Memory");
+
+  gst_allocator_register (GST_GL_PHY_MEM_ALLOCATOR, gst_object_ref (allocator));
+
+  return allocator;
+}
+
+static void
+_finish_texture (GstGLContext * ctx, gpointer * data)
+{
+  GstGLFuncs *gl = ctx->gl_vtable;
+
+  gl->Finish ();
+}
+
+static void
+_do_viv_direct_tex_bind_mem (GstGLContext * ctx, DirectVIVData * data)
+{
+  GstGLFuncs *gl = ctx->gl_vtable;
+
+  GST_DEBUG ("viv direct bind, tex_id %d, fmt: %d, res: (%dx%d)", data->tex_id,
+      data->fmt, data->w, data->h);
+  GST_DEBUG ("Physical memory buffer, vaddr: %p, paddr: %d", data->vaddr,
+      data->paddr);
+
+  gl->BindTexture (GL_TEXTURE_2D, data->tex_id);
+  pTexDirectVIVMap (GL_TEXTURE_2D, data->w, data->h, data->fmt, &data->vaddr,
+      &data->paddr);
+  pTexDirectInvalidateVIV (GL_TEXTURE_2D);
+  data->ret = TRUE;
+}
+
+static GLenum
+_directviv_video_format_to_gl_format (GstVideoFormat format)
+{
+  switch (format) {
+    case GST_VIDEO_FORMAT_I420:
+      return GL_VIV_I420;
+    case GST_VIDEO_FORMAT_YV12:
+      return GL_VIV_YV12;
+    case GST_VIDEO_FORMAT_NV12:
+      return GL_VIV_NV12;
+    case GST_VIDEO_FORMAT_NV21:
+      return GL_VIV_NV21;
+    case GST_VIDEO_FORMAT_YUY2:
+      return GL_VIV_YUY2;
+    case GST_VIDEO_FORMAT_UYVY:
+      return GL_VIV_UYVY;
+    case GST_VIDEO_FORMAT_RGB16:
+      return GL_RGB565;
+    case GST_VIDEO_FORMAT_RGBA:
+      return GL_RGBA;
+    case GST_VIDEO_FORMAT_BGRA:
+      return GL_BGRA_EXT;
+    case GST_VIDEO_FORMAT_RGBx:
+      return GL_RGBA;
+    case GST_VIDEO_FORMAT_BGRx:
+      return GL_BGRA_EXT;
+    default:
+      return 0;
+  }
+}
+
+static void
+gst_gl_phy_mem_destroy (GstMemory * mem)
+{
+  gst_memory_unref (mem);
+}
+
+
+GstAllocator *
+gst_phy_mem_allocator_obtain (void)
+{
+  static GOnce once = G_ONCE_INIT;
+
+  g_once (&once, gst_phy_mem_allocator_init_instance, NULL);
+
+  g_return_val_if_fail (once.retval != NULL, NULL);
+
+  return (GstAllocator *) (g_object_ref (once.retval));
+}
+
+gboolean
+gst_is_gl_physical_memory (GstMemory * mem)
+{
+  GstGLBaseMemory *glmem;
+  g_return_val_if_fail (gst_is_gl_memory (mem), FALSE);
+
+  glmem = (GstGLBaseMemory *) mem;
+
+  if (glmem->user_data
+      && GST_IS_MINI_OBJECT_TYPE (glmem->user_data, GST_TYPE_MEMORY))
+    return gst_memory_is_type ((GstMemory *) glmem->user_data,
+        GST_GL_PHY_MEM_ALLOCATOR);
+  else
+    return FALSE;
+}
+
+gboolean
+gst_is_gl_physical_memory_supported_fmt (GstVideoInfo * info)
+{
+  if (GST_VIDEO_INFO_IS_RGB (info)
+      && _directviv_video_format_to_gl_format (GST_VIDEO_INFO_FORMAT (info))) {
+    return TRUE;
+  } else
+    return FALSE;
+}
+
+gboolean
+gst_gl_physical_memory_setup_buffer (GstAllocator * allocator,
+    GstBuffer * buffer, GstGLVideoAllocationParams * params)
+{
+  GstGLBaseMemoryAllocator *gl_alloc;
+  GstMemory *mem = NULL;
+  PhyMemBlock *memblk = NULL;
+  GstGLMemory *glmem = NULL;
+  gsize size;
+
+  GstVideoInfo *info = params->v_info;
+  GstVideoAlignment *valign = params->valign;
+
+  GST_DEBUG ("glphymemory setup buffer format %s",
+      gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (info)));
+
+  if (!gst_is_gl_physical_memory_supported_fmt (info)) {
+    GST_DEBUG ("Not support format.");
+    return FALSE;
+  }
+
+  if (!pTexDirectVIVMap || !pTexDirectInvalidateVIV) {
+    pTexDirectVIVMap =
+        gst_gl_context_get_proc_address (params->parent.context,
+        "glTexDirectVIVMap");
+    pTexDirectInvalidateVIV =
+        gst_gl_context_get_proc_address (params->parent.context,
+        "glTexDirectInvalidateVIV");
+  }
+
+  if (!pTexDirectVIVMap || !pTexDirectInvalidateVIV) {
+    GST_DEBUG ("Load directviv functions failed.");
+    return FALSE;
+  }
+
+  size = gst_gl_get_plane_data_size (info, valign, 0);
+  mem = gst_allocator_alloc (allocator, size, params->parent.alloc_params);
+  if (!mem) {
+    GST_DEBUG ("Can't allocate physical memory size %" G_GSIZE_FORMAT, size);
+    return FALSE;
+  }
+
+  memblk = gst_memory_query_phymem_block (mem);
+  if (!memblk) {
+    GST_ERROR ("Can't find physic memory block.");
+    return FALSE;
+  }
+
+  gl_alloc =
+      GST_GL_BASE_MEMORY_ALLOCATOR (gst_gl_memory_allocator_get_default
+      (params->parent.context));
+
+  params->plane = 0;
+  params->parent.user_data = mem;
+  params->parent.notify = (GDestroyNotify) gst_gl_phy_mem_destroy;
+  params->tex_format =
+      gst_gl_format_from_video_info (params->parent.context, info, 0);
+
+  glmem =
+      (GstGLMemory *) gst_gl_base_memory_alloc (gl_alloc,
+      (GstGLAllocationParams *) params);
+  gst_object_unref (gl_alloc);
+  if (!glmem) {
+    GST_ERROR ("Can't get gl memory.");
+    return FALSE;
+  }
+
+  gst_buffer_append_memory (buffer, (GstMemory *) glmem);
+
+  gst_buffer_add_video_meta_full (buffer, 0,
+      GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info),
+      GST_VIDEO_INFO_HEIGHT (info), 1, info->offset, info->stride);
+
+  guint viv_fmt =
+      _directviv_video_format_to_gl_format (GST_VIDEO_INFO_FORMAT (info));
+
+  DirectVIVData directvivdata = {
+    glmem->tex_id,
+    GST_VIDEO_INFO_WIDTH (info),
+    GST_VIDEO_INFO_HEIGHT (info),
+    viv_fmt,
+    memblk->vaddr,
+    (uintptr_t) memblk->paddr,
+    FALSE
+  };
+
+  gst_gl_context_thread_add (params->parent.context,
+      (GstGLContextThreadFunc) _do_viv_direct_tex_bind_mem, &directvivdata);
+
+  return directvivdata.ret;
+}
+
+GstBuffer *
+gst_gl_phymem_buffer_to_gstbuffer (GstGLContext * ctx,
+    GstVideoInfo * info, GstBuffer * glbuf)
+{
+  GstBuffer *buf;
+  GstGLBaseMemory *glmem;
+
+  gst_gl_context_thread_add (ctx, (GstGLContextThreadFunc) _finish_texture,
+      NULL);
+
+  glmem = (GstGLBaseMemory *)gst_buffer_peek_memory (glbuf, 0);
+
+  buf = gst_buffer_new ();
+  gst_buffer_append_memory (buf, (GstMemory *) glmem->user_data);
+  gst_memory_ref ((GstMemory *) glmem->user_data);
+
+  gst_buffer_add_video_meta_full (buf, 0,
+      GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info),
+      GST_VIDEO_INFO_HEIGHT (info), 1, info->offset, info->stride);
+  GST_BUFFER_FLAGS (buf) = GST_BUFFER_FLAGS (glbuf);
+  GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (glbuf);
+  GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (glbuf);
+  GST_BUFFER_DURATION (buf) = GST_BUFFER_DURATION (glbuf);
+
+  return buf;
+}
diff --git a/gst-libs/gst/gl/gstglphymemory.h b/gst-libs/gst/gl/gstglphymemory.h
new file mode 100644
index 000000000..b1354d69e
--- /dev/null
+++ b/gst-libs/gst/gl/gstglphymemory.h
@@ -0,0 +1,49 @@
+/*
+ * GStreamer
+ * Copyright (c) 2015, Freescale Semiconductor, Inc.
+ * Copyright 2017 NXP
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _GST_GL_PHY_MEMORY_H_
+#define _GST_GL_PHY_MEMORY_H_
+
+#include <gst/gst.h>
+#include <gst/gstmemory.h>
+#include <gst/video/video.h>
+#include <gst/allocators/gstallocatorphymem.h>
+
+#include <gst/gl/gl.h>
+
+G_BEGIN_DECLS
+
+#define GST_GL_PHY_MEM_ALLOCATOR "GLPhyMemory"
+
+GST_GL_API
+GstAllocator *gst_phy_mem_allocator_obtain (void);
+GST_GL_API
+gboolean gst_is_gl_physical_memory (GstMemory * mem);
+GST_GL_API
+gboolean gst_is_gl_physical_memory_supported_fmt (GstVideoInfo * info);
+GST_GL_API
+gboolean gst_gl_physical_memory_setup_buffer (GstAllocator * allocator, GstBuffer *buffer, GstGLVideoAllocationParams * params);
+GST_GL_API
+GstBuffer * gst_gl_phymem_buffer_to_gstbuffer (GstGLContext * ctx, GstVideoInfo * info, GstBuffer *glbuf);
+
+G_END_DECLS
+
+#endif /* _GST_GL_PHY_MEMORY_H_ */
diff --git a/gst-libs/gst/gl/meson.build b/gst-libs/gst/gl/meson.build
index bbc5c8f21..06ae1e739 100644
--- a/gst-libs/gst/gl/meson.build
+++ b/gst-libs/gst/gl/meson.build
@@ -145,6 +145,7 @@ glconf_options = [
 
     'GST_GL_HAVE_DMABUF',
     'GST_GL_HAVE_VIV_DIRECTVIV',
+    'GST_GL_HAVE_PHYMEM',
 
     'GST_GL_HAVE_GLEGLIMAGEOES',
     'GST_GL_HAVE_GLCHAR',
@@ -952,7 +953,8 @@ if need_win_gbm != 'no'
   endif
 endif
 
-if cc.has_function ('glTexDirectVIV', dependencies : gles2_dep)
+gles2_lib_dep = cc.find_library('GLESv2', required : false)
+if cc.has_function ('glTexDirectVIV', dependencies : gles2_lib_dep)
   glconf.set('GST_GL_HAVE_VIV_DIRECTVIV', 1)
 endif
 
@@ -976,6 +978,18 @@ if need_platform_egl != 'no' and need_win_viv_fb != 'no'
   endif
 endif
 
+g2d_dep = unneeded_dep
+if cc.has_function ('glTexDirectVIV', dependencies : gles2_lib_dep) and cc.check_header('g2d.h')
+  gl_sources += [
+    'gstglphymemory.c',
+  ]
+  gl_headers += [
+    'gstglphymemory.h',
+  ]
+  glconf.set10('GST_GL_HAVE_PHYMEM', 1)
+  g2d_dep = cc.find_library('g2d', required : false)
+endif
+
 if need_win_android == 'yes'
   if need_platform_egl == 'no'
     error('Impossible situation requested: Cannot build for Android without EGL')
@@ -1104,7 +1118,8 @@ if build_gstgl
     darwin_versions : osxversion,
     install : true,
     dependencies : [gst_base_dep, video_dep, allocators_dep, gmodule_dep,
-                    gl_lib_deps, gl_platform_deps, gl_winsys_deps, gl_misc_deps],
+                    gl_lib_deps, gl_platform_deps, gl_winsys_deps, gl_misc_deps,
+                    g2d_dep],
     # don't confuse EGL/egl.h with gst-libs/gl/egl/egl.h on case-insensitive file systems
     implicit_include_directories : false)
 
-- 
2.34.1


From 81f9f69fc5317ef8cc094d5462212aaa71c0b060 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Fri, 17 May 2019 13:58:09 +0800
Subject: [PATCH 26/93] MMFMWK-8456 gldownload: set allocator to buffer pool

make sure upstream can use our buffer pool with
right allocator that we proposed

upstream status: imx specific

(cherry picked from commit 94ee2d90fb4de442adc2556861aabc745b58096a)
---
 ext/gl/gstgldownloadelement.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ext/gl/gstgldownloadelement.c b/ext/gl/gstgldownloadelement.c
index fb545bd17..d0d7bcf72 100644
--- a/ext/gl/gstgldownloadelement.c
+++ b/ext/gl/gstgldownloadelement.c
@@ -1428,7 +1428,6 @@ gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
   }
 
   gst_query_add_allocation_param (query, allocator, &params);
-  gst_object_unref (allocator);
 
   n_pools = gst_query_get_n_allocation_pools (query);
   for (i = 0; i < n_pools; i++) {
@@ -1448,6 +1447,12 @@ gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
   gst_buffer_pool_config_add_option (config,
       GST_BUFFER_POOL_OPTION_GL_SYNC_META);
 
+  //set allocator to buffer pool
+  if (allocator){
+    gst_buffer_pool_config_set_allocator (config, allocator, &params);
+    gst_object_unref (allocator);
+  }
+
   if (!gst_buffer_pool_set_config (pool, config)) {
     gst_object_unref (pool);
     GST_WARNING_OBJECT (bt, "failed setting config");
-- 
2.34.1


From 32e7b9c48d9e0231e795e8da21f43e41cf50899f Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Tue, 21 Aug 2018 13:20:29 +0800
Subject: [PATCH 27/93] MMFMWK-8190 glupload: refine transform caps of
 directviv uploader

if downstream only accept YUV format (eg, add capsfilter), then
we should passthrough color format and don't extend to directviv
support format

upstream status: Pending
https://bugzilla.gnome.org/show_bug.cgi?id=797002

Conflicts:
	gst-libs/gst/gl/gstglupload.c
(cherry picked from commit 839528dabae8ac5ec07a21d0607284d1de05c864)
---
 gst-libs/gst/gl/gstglupload.c | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index ee41a5cfb..c60a85bd7 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -2462,8 +2462,25 @@ _directviv_upload_transform_caps (gpointer impl, GstGLContext * context,
     gst_caps_unref (ret);
     ret = tmp;
   } else {
-    ret = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
+    GstCaps *caps_copy, *caps_intersect;
+    caps_copy = gst_caps_copy (caps);
+    gst_caps_set_simple (caps_copy, "format", G_TYPE_STRING, "RGBA", NULL);
+    caps_intersect = gst_caps_intersect (caps_copy, caps);
+    if (gst_caps_is_empty (caps_intersect)) {
+      ret =
+          _set_caps_features_with_passthrough (caps,
+          GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, passthrough);
+    } else {
+      GstCaps *tmp;
+      tmp = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
         (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, GST_GL_DIRECTVIV_FORMAT));
+      ret =
+          _set_caps_features_with_passthrough (tmp,
+          GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, passthrough);
+      gst_caps_unref (tmp);
+    }
+    gst_caps_unref (caps_copy);
+    gst_caps_unref (caps_intersect);
   }
 
   gst_caps_features_free (passthrough);
-- 
2.34.1


From 05a639955504bd7cdd9ab3f2af7e1ce64e871e3e Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Thu, 25 Oct 2018 10:09:00 +0800
Subject: [PATCH 28/93] MMFMWK-8235 glupload: add internal physical buffer pool

For legacy platform with G2D, add internal buffer pool
and copy non physical memory to internal buffer to enable
directviv upload. Only when output format is RGBA need this,
otherwise use default glmemory to do upload.

upstream status: imx specific
---
 gst-libs/gst/gl/gstglupload.c | 224 +++++++++++++++++++++++++++++++++-
 1 file changed, 222 insertions(+), 2 deletions(-)

diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index c60a85bd7..86be32b34 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -24,6 +24,7 @@
 #endif
 
 #include <stdio.h>
+#include <string.h>
 
 #include "gl.h"
 #include "gstglupload.h"
@@ -49,6 +50,10 @@
 #include <gst/gl/gstglfuncs.h>
 #endif
 
+#if GST_GL_HAVE_PHYMEM
+#include "gstglphymemory.h"
+#endif
+
 /**
  * SECTION:gstglupload
  * @title: GstGLUpload
@@ -2414,6 +2419,7 @@ struct DirectVIVUpload
       GLenum Format, GLvoid ** Logical, const GLuint * Physical);
   void (*TexDirectInvalidateVIV) (GLenum Target);
   gboolean loaded_functions;
+  GstBufferPool *pool;
 };
 
 #define GST_GL_DIRECTVIV_FORMAT "{RGBA, I420, YV12, NV12, NV21, YUY2, UYVY, BGRA, RGB16}"
@@ -2431,6 +2437,71 @@ _directviv_upload_new (GstGLUpload * upload)
   return directviv;
 }
 
+static gboolean
+_directviv_upload_setup_buffer_pool (GstBufferPool **pool, GstAllocator *allocator,
+    GstCaps *caps, GstVideoInfo *info)
+{
+  GstAllocationParams params;
+  GstStructure *config;
+  gsize size;
+  guint width, height;
+  GstVideoAlignment alignment;
+
+  g_return_val_if_fail (caps != NULL && info != NULL, FALSE);
+
+  width = GST_VIDEO_INFO_WIDTH (info);
+  height = GST_VIDEO_INFO_HEIGHT (info);
+
+  gst_allocation_params_init (&params);
+
+  /* if user not provide an allocator, then use default physical allocator*/
+  if (!allocator) {
+#if GST_GL_HAVE_PHYMEM
+    allocator = gst_phy_mem_allocator_obtain ();
+#endif
+  }
+
+  if (!allocator) {
+    GST_WARNING ("Cannot get available allocator");
+    return FALSE;
+  }
+  GST_DEBUG ("got allocator(%p).", allocator);
+
+  if (*pool)
+    gst_object_unref(*pool);
+
+  *pool = gst_video_buffer_pool_new ();
+  if (!*pool) {
+    GST_WARNING ("New video buffer pool failed.");
+    return FALSE;
+  }
+  GST_DEBUG ("create buffer pool(%p).", *pool);
+
+  config = gst_buffer_pool_get_config (*pool);
+
+  /* configure alignment for eglimage to import this dma-fd buffer */
+  memset (&alignment, 0, sizeof (GstVideoAlignment));
+  alignment.padding_right = GST_ROUND_UP_N(width, DEFAULT_ALIGN) - width;
+  GST_DEBUG ("align buffer pool, w(%d) h(%d), padding_right (%d), padding_bottom (%d)",
+      width, height, alignment.padding_right, alignment.padding_bottom);
+
+  /* the normal size of a frame */
+  size = info->size;
+  gst_buffer_pool_config_set_params (config, caps, size, 0, 30);
+  gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
+  gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
+  gst_buffer_pool_config_set_video_alignment (config, &alignment);
+  gst_buffer_pool_config_set_allocator (config, allocator, &params);
+
+  if (!gst_buffer_pool_set_config (*pool, config)) {
+    GST_WARNING ("buffer pool config failed.");
+    gst_object_unref (*pool);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
 static GstCaps *
 _directviv_upload_transform_caps (gpointer impl, GstGLContext * context,
     GstPadDirection direction, GstCaps * caps)
@@ -2487,6 +2558,22 @@ _directviv_upload_transform_caps (gpointer impl, GstGLContext * context,
   return ret;
 }
 
+static gboolean
+_directviv_upload_buffer_pool_is_ok (GstBufferPool * pool, GstCaps * newcaps,
+    gint size)
+{
+  GstCaps *oldcaps;
+  GstStructure *config;
+  guint bsize;
+  gboolean ret;
+
+  config = gst_buffer_pool_get_config (pool);
+  gst_buffer_pool_config_get_params (config, &oldcaps, &bsize, NULL, NULL);
+  ret = (size <= bsize) && gst_caps_is_equal (newcaps, oldcaps);
+  gst_structure_free (config);
+
+  return ret;
+}
 
 static void
 _directviv_upload_load_functions_gl_thread (GstGLContext * context,
@@ -2547,13 +2634,137 @@ _directviv_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
     mem = NULL;
   }
 
+#if GST_GL_HAVE_PHYMEM
+  GstVideoInfo *in_info = &directviv->upload->priv->in_info;
+  GstVideoFormat fmt = GST_VIDEO_INFO_FORMAT (&directviv->upload->priv->out_info);
+  if (fmt != GST_VIDEO_FORMAT_RGBA)
+    return FALSE;
+
+  if (n_mem != 1 || !mem || !gst_is_phys_memory (mem)) {
+    GstVideoFrame frame1, frame2;
+    GstCaps *new_caps;
+    GstVideoInfo info;
+
+    gst_video_frame_map (&frame1, in_info, buffer, GST_MAP_READ);
+    new_caps = gst_video_info_to_caps(&frame1.info);
+    gst_video_info_from_caps(&info, new_caps);
+
+    if (!directviv->pool || !_directviv_upload_buffer_pool_is_ok (directviv->pool,new_caps,info.size)) {
+      gboolean ret;
+      if (directviv->pool) {
+        gst_object_unref(directviv->pool);
+        directviv->pool = NULL;
+      }
+
+      ret = _directviv_upload_setup_buffer_pool (&directviv->pool, NULL, new_caps, in_info);
+      if (!ret) {
+        gst_video_frame_unmap (&frame1);
+        gst_caps_unref (new_caps);
+        GST_WARNING_OBJECT (directviv->upload, "no available buffer pool");
+        return FALSE;
+      }
+    }
+
+    if (!gst_buffer_pool_is_active (directviv->pool)
+        && gst_buffer_pool_set_active (directviv->pool, TRUE) != TRUE) {
+      gst_video_frame_unmap (&frame1);
+      GST_WARNING_OBJECT (directviv->upload, "buffer pool is not ok");
+      return FALSE;
+    }
+
+    if (directviv->inbuf)
+      gst_buffer_unref(directviv->inbuf);
+    directviv->inbuf = NULL;
+
+    gst_buffer_pool_acquire_buffer (directviv->pool, &directviv->inbuf, NULL);
+    if (!directviv->inbuf) {
+      gst_video_frame_unmap (&frame1);
+      GST_WARNING_OBJECT (directviv->upload, "acquire_buffer failed");
+      return FALSE;
+    }
+
+    GST_DEBUG_OBJECT (directviv->upload, "copy plane resolution (%d)x(%d)\n", in_info->width, in_info->height);
+    gst_video_frame_map (&frame2, in_info, directviv->inbuf, GST_MAP_WRITE);
+    gst_video_frame_copy (&frame2, &frame1);
+    gst_video_frame_unmap (&frame1);
+    gst_video_frame_unmap (&frame2);
+  }
+
+  return TRUE;
+#else
   return n_mem == 1 && mem && gst_is_phys_memory (mem);
+#endif
 }
 
 static void
 _directviv_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
     GstQuery * query)
 {
+#if GST_GL_HAVE_PHYMEM
+  struct DirectVIVUpload *directviv = impl;
+  GstBufferPool *pool = NULL;
+  GstAllocator *allocator = NULL;
+  GstCaps *caps;
+  GstVideoInfo info;
+  GstVideoFormat fmt = GST_VIDEO_FORMAT_UNKNOWN;
+  GstCapsFeatures * caps_features;
+
+  if (directviv->upload->priv->out_info.finfo)
+    fmt = GST_VIDEO_INFO_FORMAT (&directviv->upload->priv->out_info);
+
+  if (fmt != GST_VIDEO_FORMAT_RGBA)
+    return;
+
+#if GST_GL_HAVE_IONDMA
+  /* physical memory buffer pool was only proposed
+   * when ion is not available to avoid allocator
+   * overwrite in allocation query, we need keep use
+   * ion when it is available */
+  allocator = gst_ion_allocator_obtain ();
+  if (allocator)
+    return;
+#endif
+
+  gst_query_parse_allocation (query, &caps, NULL);
+
+  if (!gst_video_info_from_caps (&info, caps))
+    goto invalid_caps;
+
+  caps_features = gst_caps_get_features (caps, 0);
+  if (gst_caps_features_contains (caps_features, "memory:GLMemory")) {
+    GST_DEBUG ("upstream has memory:GLMemory feature");
+    return;
+  }
+
+  allocator = gst_phy_mem_allocator_obtain ();
+  if (!allocator) {
+    GST_WARNING ("New physical memory allocator failed.");
+    return;
+  }
+  GST_DEBUG ("create physical memory allocator(%p).", allocator);
+
+  gst_query_set_nth_allocation_param(query, 0, allocator, NULL);
+
+  if (!_directviv_upload_setup_buffer_pool (&pool, allocator, caps, &info))
+    goto setup_failed;
+
+  gst_query_set_nth_allocation_pool (query, 0, pool, info.size, 1, 30);
+
+  if (pool)
+    gst_object_unref (pool);
+
+  return;
+invalid_caps:
+  {
+    GST_WARNING_OBJECT (directviv->upload, "invalid caps specified");
+    return;
+  }
+setup_failed:
+  {
+    GST_WARNING_OBJECT (directviv->upload, "failed to setup buffer pool");
+    return;
+  }
+#endif
 }
 
 static GLenum
@@ -2693,11 +2904,14 @@ _directviv_upload_perform (gpointer impl, GstBuffer * buffer,
 {
   struct DirectVIVUpload *directviv = impl;
 
-  directviv->inbuf = buffer;
+  if (!directviv->inbuf)
+    directviv->inbuf = buffer;
   directviv->outbuf = NULL;
   gst_gl_context_thread_add (directviv->upload->context,
       (GstGLContextThreadFunc) _directviv_upload_perform_gl_thread, directviv);
-  directviv->inbuf = NULL;
+
+  if (directviv->inbuf == buffer)
+    directviv->inbuf = NULL;
 
   if (!directviv->outbuf)
     return GST_GL_UPLOAD_ERROR;
@@ -2716,6 +2930,12 @@ _directviv_upload_free (gpointer impl)
   if (directviv->params)
     gst_gl_allocation_params_free ((GstGLAllocationParams *) directviv->params);
 
+  if (directviv->inbuf)
+    gst_buffer_unref (directviv->inbuf);
+
+  if (directviv->pool)
+    gst_object_unref(directviv->pool);
+
   g_free (impl);
 }
 
-- 
2.34.1


From 23758fd72052cb802a3705f15c6a42974fdba8fb Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@nxp.com>
Date: Mon, 2 Apr 2018 15:51:48 +0800
Subject: [PATCH 29/93] ionmemory: dmabuf memory allocator based on ion driver.

merge below 4 commits from imx-1.16.x
59ccb9183 Fix build break for ion on 4.14 kernel [YOCIMX-2861]
6d020d790 MMFMWK-8113 [ion] Enable ion allocator based on Kernel 4.14
ad0160d35 ionmemory: support get phys memory
1ac453e86 ionmemory: dmabuf memory allocator based on ion driver.

merge below 1 commits from imx-1.22.x
eef04f0e21a gstionmemory.c: Convert INT pointer type to unsigned long for query.heaps
b9d3907cc allocators/ionmemory: install gstionmemory.h
---
 gst-libs/gst/allocators/gstionmemory.c | 362 +++++++++++++++++++++++++
 gst-libs/gst/allocators/gstionmemory.h |  69 +++++
 gst-libs/gst/allocators/meson.build    |  14 +
 3 files changed, 445 insertions(+)
 create mode 100644 gst-libs/gst/allocators/gstionmemory.c
 create mode 100644 gst-libs/gst/allocators/gstionmemory.h

diff --git a/gst-libs/gst/allocators/gstionmemory.c b/gst-libs/gst/allocators/gstionmemory.c
new file mode 100644
index 000000000..c29a65368
--- /dev/null
+++ b/gst-libs/gst/allocators/gstionmemory.c
@@ -0,0 +1,362 @@
+/*
+ * Copyright (c) 2016, Freescale Semiconductor, Inc. All rights reserved.
+ * Copyright 2017, 2019 NXP
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <linux/ion.h>
+#include <linux/dma-buf.h>
+#include <linux/version.h>
+
+
+#include <gst/allocators/gstdmabuf.h>
+#include "gstphysmemory.h"
+#include "gstionmemory.h"
+
+GST_DEBUG_CATEGORY_STATIC (ion_allocator_debug);
+#define GST_CAT_DEFAULT ion_allocator_debug
+
+#define gst_ion_allocator_parent_class parent_class
+
+#define DEFAULT_HEAP_ID  0
+#define DEFAULT_FLAG     0
+
+enum
+{
+  PROP_0,
+  PROP_HEAP_ID,
+  PROP_FLAG,
+  PROP_LAST
+};
+
+static guintptr
+gst_ion_allocator_get_phys_addr (GstPhysMemoryAllocator * allocator,
+    GstMemory * mem)
+{
+  GstIONAllocator *self = GST_ION_ALLOCATOR (allocator);
+  gint ret, fd;
+
+  if (self->fd < 0 || !mem) {
+    GST_ERROR ("ion get phys param wrong");
+    return 0;
+  }
+
+  if (!gst_is_dmabuf_memory (mem)) {
+    GST_ERROR ("isn't dmabuf memory");
+    return 0;
+  }
+
+  fd = gst_dmabuf_memory_get_fd (mem);
+  if (fd < 0) {
+    GST_ERROR ("dmabuf memory get fd failed");
+    return 0;
+  }
+
+  GST_DEBUG ("ion DMA FD: %d", fd);
+
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
+  struct ion_phys_dma_data data = {
+    .phys = 0,
+    .size = 0,
+    .dmafd = fd,
+  };
+  struct ion_custom_data custom = {
+    .cmd = ION_IOC_PHYS_DMA,
+    .arg = (unsigned long) &data,
+  };
+
+  ret = ioctl (self->fd, ION_IOC_CUSTOM, &custom);
+  if (ret < 0)
+    return 0;
+
+  return data.phys;
+#else
+  struct dma_buf_phys dma_phys;
+
+  ret = ioctl (fd, DMA_BUF_IOCTL_PHYS, &dma_phys);
+  if (ret < 0)
+    return 0;
+
+  return dma_phys.phys;
+#endif
+}
+
+static void
+gst_ion_allocator_iface_init (gpointer g_iface)
+{
+  GstPhysMemoryAllocatorInterface *iface = g_iface;
+  iface->get_phys_addr = gst_ion_allocator_get_phys_addr;
+}
+
+G_DEFINE_TYPE_WITH_CODE (GstIONAllocator, gst_ion_allocator,
+    GST_TYPE_DMABUF_ALLOCATOR,
+    G_IMPLEMENT_INTERFACE (GST_TYPE_PHYS_MEMORY_ALLOCATOR,
+        gst_ion_allocator_iface_init));
+
+static gint
+gst_ion_ioctl (gint fd, gint req, void *arg)
+{
+  gint ret = ioctl (fd, req, arg);
+  if (ret < 0) {
+    GST_ERROR ("ioctl %x failed with code %d: %s\n", req, ret,
+        strerror (errno));
+  }
+  return ret;
+}
+
+static void
+gst_ion_mem_init (void)
+{
+  GstAllocator *allocator = g_object_new (gst_ion_allocator_get_type (), NULL);
+  GstIONAllocator *self = GST_ION_ALLOCATOR (allocator);
+  gint fd;
+
+  fd = open ("/dev/ion", O_RDWR);
+  if (fd < 0) {
+    GST_WARNING ("Could not open ion driver");
+    g_object_unref (self);
+    return;
+  }
+
+  self->fd = fd;
+
+  gst_allocator_register (GST_ALLOCATOR_ION, allocator);
+}
+
+GstAllocator *
+gst_ion_allocator_obtain (void)
+{
+  static GOnce ion_allocator_once = G_ONCE_INIT;
+  GstAllocator *allocator;
+
+  g_once (&ion_allocator_once, (GThreadFunc) gst_ion_mem_init, NULL);
+
+  allocator = gst_allocator_find (GST_ALLOCATOR_ION);
+  if (allocator == NULL)
+    GST_WARNING ("No allocator named %s found", GST_ALLOCATOR_ION);
+
+  return allocator;
+}
+
+static GstMemory *
+gst_ion_alloc_alloc (GstAllocator * allocator, gsize size,
+    GstAllocationParams * params)
+{
+  GstIONAllocator *self = GST_ION_ALLOCATOR (allocator);
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
+  struct ion_allocation_data allocation_data = { 0 };
+  struct ion_fd_data fd_data = { 0 };
+  struct ion_handle_data handle_data = { 0 };
+  ion_user_handle_t ion_handle;
+  GstMemory *mem;
+  gsize ion_size;
+  gint dma_fd = -1;
+  gint ret;
+
+  if (self->fd < 0) {
+    GST_ERROR ("ion allocate param wrong");
+    return NULL;
+  }
+
+  ion_size = size + params->prefix + params->padding;
+  allocation_data.len = ion_size;
+  allocation_data.align = params->align;
+  allocation_data.heap_id_mask = 1 << self->heap_id;
+  allocation_data.flags = self->flags;
+  if (gst_ion_ioctl (self->fd, ION_IOC_ALLOC, &allocation_data) < 0) {
+    GST_ERROR ("ion allocate failed.");
+    return NULL;
+  }
+  ion_handle = allocation_data.handle;
+
+  fd_data.handle = ion_handle;
+  ret = gst_ion_ioctl (self->fd, ION_IOC_MAP, &fd_data);
+  if (ret < 0 || fd_data.fd < 0) {
+    GST_ERROR ("map ioctl failed or returned negative fd");
+    goto bail;
+  }
+  dma_fd = fd_data.fd;
+
+  handle_data.handle = ion_handle;
+  gst_ion_ioctl (self->fd, ION_IOC_FREE, &handle_data);
+
+#else
+  gint heapCnt = 0;
+  gint heap_mask = 0;
+  GstMemory *mem;
+  gsize ion_size;
+  gint dma_fd = -1;
+  gint ret;
+
+  struct ion_heap_query query;
+  memset (&query, 0, sizeof (query));
+  ret = gst_ion_ioctl (self->fd, ION_IOC_HEAP_QUERY, &query);
+  if (ret != 0 || query.cnt == 0) {
+    GST_ERROR ("can't query heap count");
+    return NULL;
+  }
+  heapCnt = query.cnt;
+
+  struct ion_heap_data ihd[heapCnt];
+  memset (&ihd, 0, sizeof (ihd));
+  query.cnt = heapCnt;
+  query.heaps = (uintptr_t)&ihd;
+  ret = gst_ion_ioctl (self->fd, ION_IOC_HEAP_QUERY, &query);
+  if (ret != 0) {
+    GST_ERROR ("can't get ion heaps");
+    return NULL;
+  }
+
+  for (gint i = 0; i < heapCnt; i++) {
+    if (ihd[i].type == ION_HEAP_TYPE_DMA) {
+      heap_mask |= 1 << ihd[i].heap_id;
+    }
+  }
+
+  ion_size = size + params->prefix + params->padding;
+  struct ion_allocation_data data = {
+    .len = ion_size,
+    .heap_id_mask = heap_mask,
+    .flags = self->flags,
+  };
+  ret = gst_ion_ioctl (self->fd, ION_IOC_ALLOC, &data);
+  if (ret < 0) {
+    GST_ERROR ("ion allocate failed.");
+    return NULL;
+  }
+  dma_fd = data.fd;
+#endif
+
+  mem = gst_dmabuf_allocator_alloc (allocator, dma_fd, size);
+
+  GST_DEBUG ("ion allocated size: %" G_GSIZE_FORMAT "DMA FD: %d", ion_size,
+      dma_fd);
+
+  return mem;
+
+bail:
+  if (dma_fd >= 0) {
+    close (dma_fd);
+  }
+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 14, 0)
+  handle_data.handle = ion_handle;
+  gst_ion_ioctl (self->fd, ION_IOC_FREE, &handle_data);
+#endif
+
+  return NULL;
+}
+
+static void
+gst_ion_allocator_dispose (GObject * object)
+{
+  GstIONAllocator *self = GST_ION_ALLOCATOR (object);
+
+  if (self->fd > 0) {
+    close (self->fd);
+    self->fd = -1;
+  }
+
+  G_OBJECT_CLASS (parent_class)->dispose (object);
+}
+
+static void
+gst_ion_allocator_set_property (GObject * object, guint prop_id,
+    const GValue * value, GParamSpec * pspec)
+{
+  GstIONAllocator *self = GST_ION_ALLOCATOR (object);
+
+  switch (prop_id) {
+    case PROP_HEAP_ID:
+      self->heap_id = g_value_get_uint (value);
+      break;
+    case PROP_FLAG:
+      self->flags = g_value_get_uint (value);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+gst_ion_allocator_get_property (GObject * object, guint prop_id, GValue * value,
+    GParamSpec * pspec)
+{
+  GstIONAllocator *self = GST_ION_ALLOCATOR (object);
+
+  switch (prop_id) {
+    case PROP_HEAP_ID:
+      g_value_set_uint (value, self->heap_id);
+      break;
+    case PROP_FLAG:
+      g_value_set_uint (value, self->flags);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+gst_ion_allocator_class_init (GstIONAllocatorClass * klass)
+{
+  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
+  GstAllocatorClass *allocator_class = GST_ALLOCATOR_CLASS (klass);
+
+  gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_ion_allocator_dispose);
+  gobject_class->set_property = gst_ion_allocator_set_property;
+  gobject_class->get_property = gst_ion_allocator_get_property;
+
+  g_object_class_install_property (gobject_class, PROP_HEAP_ID,
+      g_param_spec_uint ("heap-id", "Heap ID",
+          "ION heap id", 0, G_MAXUINT32, DEFAULT_HEAP_ID,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  g_object_class_install_property (gobject_class, PROP_FLAG,
+      g_param_spec_uint ("flags", "Flags",
+          "ION memory flags", 0, G_MAXUINT32, DEFAULT_FLAG,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  allocator_class->alloc = GST_DEBUG_FUNCPTR (gst_ion_alloc_alloc);
+
+  GST_DEBUG_CATEGORY_INIT (ion_allocator_debug, "ionmemory", 0,
+      "DMA FD memory allocator based on ion");
+}
+
+static void
+gst_ion_allocator_init (GstIONAllocator * self)
+{
+  GstAllocator *allocator = GST_ALLOCATOR (self);
+
+  allocator->mem_type = GST_ALLOCATOR_ION;
+
+  self->heap_id = DEFAULT_HEAP_ID;
+  self->flags = DEFAULT_FLAG;
+}
diff --git a/gst-libs/gst/allocators/gstionmemory.h b/gst-libs/gst/allocators/gstionmemory.h
new file mode 100644
index 000000000..272db409c
--- /dev/null
+++ b/gst-libs/gst/allocators/gstionmemory.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2016, Freescale Semiconductor, Inc. All rights reserved.
+ * Copyright 2017 NXP
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __GST_IONMEMORY_H__
+#define __GST_IONMEMORY_H__
+
+#include <gst/gst.h>
+#include <gst/allocators/gstdmabuf.h>
+#include <gst/allocators/allocators-prelude.h>
+
+G_BEGIN_DECLS
+
+typedef struct _GstIONAllocator GstIONAllocator;
+typedef struct _GstIONAllocatorClass GstIONAllocatorClass;
+typedef struct _GstIONMemory GstIONMemory;
+
+#define GST_ALLOCATOR_ION "ionmem"
+
+#define GST_TYPE_ION_ALLOCATOR gst_ion_allocator_get_type ()
+#define GST_IS_ION_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), \
+    GST_TYPE_ION_ALLOCATOR))
+#define GST_ION_ALLOCATOR(obj) \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_ION_ALLOCATOR, GstIONAllocator))
+#define GST_ION_ALLOCATOR_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_ION_ALLOCATOR, GstIONAllocatorClass))
+#define GST_ION_ALLOCATOR_CAST(obj) ((GstIONAllocator *)(obj))
+
+#define GST_ION_MEMORY_QUARK gst_ion_memory_quark ()
+
+struct _GstIONAllocator
+{
+  GstDmaBufAllocator parent;
+
+  gint fd;
+  guint heap_id;
+  guint flags;
+};
+
+struct _GstIONAllocatorClass
+{
+  GstDmaBufAllocatorClass parent;
+};
+
+GST_ALLOCATORS_API
+GType gst_ion_allocator_get_type (void);
+
+GST_ALLOCATORS_API
+GstAllocator* gst_ion_allocator_obtain (void);
+
+G_END_DECLS
+
+#endif /* __GST_IONMEMORY_H__ */
diff --git a/gst-libs/gst/allocators/meson.build b/gst-libs/gst/allocators/meson.build
index 428ba645e..31f990d24 100644
--- a/gst-libs/gst/allocators/meson.build
+++ b/gst-libs/gst/allocators/meson.build
@@ -10,6 +10,13 @@ gst_allocators_headers = files([
   'gstphymemmeta.h',
   'gstallocatorphymem.h',
 ])
+
+if cc.has_header('linux/ion.h')
+  gst_allocators_headers += [
+    'gstionmemory.h',
+  ]
+endif
+
 install_headers(gst_allocators_headers, subdir : 'gstreamer-1.0/gst/allocators/')
 
 gst_allocators_sources = files([
@@ -40,7 +47,14 @@ elif cc.has_function('shm_open')
   ]
 endif
 
+if cc.has_header('linux/ion.h')
+  gst_allocators_sources += [
+    'gstionmemory.c',
+  ]
+endif
+
 gstallocators = library('gstallocators-@0@'.format(api_version),
+
   gst_allocators_sources,
   c_args : gst_allocators_cargs,
   include_directories: [configinc, libsinc],
-- 
2.34.1


From 232c8f037f60934c4baebfd952fc2d4b1cf1f21a Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Wed, 11 Apr 2018 18:02:04 +0800
Subject: [PATCH 30/93] glupload: add ion dmabuf support in glupload

1. Support one texture for YUV format in dmabuf uploader
2. Propose ion dma-fd buffer pool to upstream to avoid memory copy
3. If upstream don't chose the proposed buffer pool, then create
   our own and do copy to avoid memory copy from CPU to GPU side
4. Add buffer alignmentw

From GST-1.16, New upload direct_dmabuf_uploader was introduced

Upstream Status: Inappropriate [i.MX specific]

Conflicts:
	gst-libs/gst/gl/gstglupload.c
(cherry picked from commit c57c5c15e9122dc426bfa21dbaec2ade4ca07fd2)
---
 gst-libs/gst/gl/gstglconfig.h.meson |   1 +
 gst-libs/gst/gl/gstglupload.c       | 247 +++++++++++++++++++++++++---
 gst-libs/gst/gl/meson.build         |   4 +
 3 files changed, 227 insertions(+), 25 deletions(-)

diff --git a/gst-libs/gst/gl/gstglconfig.h.meson b/gst-libs/gst/gl/gstglconfig.h.meson
index 9f0b40cee..272717883 100644
--- a/gst-libs/gst/gl/gstglconfig.h.meson
+++ b/gst-libs/gst/gl/gstglconfig.h.meson
@@ -34,6 +34,7 @@ G_BEGIN_DECLS
 #mesondefine GST_GL_HAVE_DMABUF
 #mesondefine GST_GL_HAVE_VIV_DIRECTVIV
 #mesondefine GST_GL_HAVE_PHYMEM
+#mesondefine GST_GL_HAVE_IONDMA
 
 #mesondefine GST_GL_HAVE_GLEGLIMAGEOES
 #mesondefine GST_GL_HAVE_GLCHAR
diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index 86be32b34..455941906 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -54,6 +54,10 @@
 #include "gstglphymemory.h"
 #endif
 
+#if GST_GL_HAVE_IONDMA
+#include <gst/allocators/gstionmemory.h>
+#endif
+
 /**
  * SECTION:gstglupload
  * @title: GstGLUpload
@@ -71,6 +75,8 @@
 #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
 #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
 
+#define DEFAULT_ALIGN 16
+
 GST_DEBUG_CATEGORY_STATIC (gst_gl_upload_debug);
 #define GST_CAT_DEFAULT gst_gl_upload_debug
 
@@ -741,7 +747,9 @@ struct DmabufUpload
   GstEGLImage *eglimage[GST_VIDEO_MAX_PLANES];
   GstEGLImageCache *eglimage_cache;
   GstGLFormat formats[GST_VIDEO_MAX_PLANES];
+  GstBuffer *inbuf;
   GstBuffer *outbuf;
+  GstBufferPool *pool;
   GstGLVideoAllocationParams *params;
   guint n_mem;
 
@@ -1398,6 +1406,7 @@ _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
       return NULL;
     }
 
+    gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL);
     tmp = _caps_intersect_texture_target (ret, 1 << GST_GL_TEXTURE_TARGET_2D);
     gst_caps_unref (ret);
     ret = tmp;
@@ -1440,6 +1449,74 @@ _dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
   return ret;
 }
 
+static gboolean
+_dma_buf_upload_setup_buffer_pool (GstBufferPool ** pool,
+    GstAllocator * allocator, GstCaps * caps, GstVideoInfo * info)
+{
+  GstAllocationParams params;
+  GstStructure *config;
+  gsize size;
+  guint width, height;
+  GstVideoAlignment alignment;
+
+  g_return_val_if_fail (caps != NULL && info != NULL, FALSE);
+
+  width = GST_VIDEO_INFO_WIDTH (info);
+  height = GST_VIDEO_INFO_HEIGHT (info);
+
+  gst_allocation_params_init (&params);
+
+  /* if user not provide an allocator, then use default ion allocator */
+  if (!allocator) {
+#if GST_GL_HAVE_IONDMA
+    allocator = gst_ion_allocator_obtain ();
+#endif
+  }
+
+  if (!allocator) {
+    GST_WARNING ("Cannot get available allocator");
+    return FALSE;
+  }
+  GST_DEBUG ("got allocator(%p).", allocator);
+
+  if (*pool)
+    gst_object_unref (*pool);
+
+  *pool = gst_video_buffer_pool_new ();
+  if (!*pool) {
+    GST_WARNING ("New video buffer pool failed.");
+    return FALSE;
+  }
+  GST_DEBUG ("create buffer pool(%p).", *pool);
+
+  config = gst_buffer_pool_get_config (*pool);
+
+  /* configure alignment for eglimage to import this dma-fd buffer */
+  memset (&alignment, 0, sizeof (GstVideoAlignment));
+  alignment.padding_right = GST_ROUND_UP_N (width, DEFAULT_ALIGN) - width;
+  alignment.padding_bottom = GST_ROUND_UP_N (height, DEFAULT_ALIGN) - height;
+  GST_DEBUG
+      ("align buffer pool, w(%d) h(%d), padding_right (%d), padding_bottom (%d)",
+      width, height, alignment.padding_right, alignment.padding_bottom);
+
+  /* the normal size of a frame */
+  size = info->size;
+  gst_buffer_pool_config_set_params (config, caps, size, 0, 30);
+  gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
+  gst_buffer_pool_config_add_option (config,
+      GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
+  gst_buffer_pool_config_set_video_alignment (config, &alignment);
+  gst_buffer_pool_config_set_allocator (config, allocator, &params);
+
+  if (!gst_buffer_pool_set_config (*pool, config)) {
+    GST_WARNING ("buffer pool config failed.");
+    gst_object_unref (*pool);
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
 static gboolean
 _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
     GstCaps * out_caps)
@@ -1461,7 +1538,7 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
 
   n_mem = gst_buffer_n_memory (buffer);
   meta = gst_buffer_get_video_meta (buffer);
-  crop = gst_buffer_get_video_crop_meta(buffer);
+  crop = gst_buffer_get_video_crop_meta (buffer);
 
   if (!dmabuf->upload->context->gl_vtable->EGLImageTargetTexture2D)
     return FALSE;
@@ -1516,8 +1593,53 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
 
   /* This will eliminate most non-dmabuf out there */
   if (!gst_is_dmabuf_memory (gst_buffer_peek_memory (buffer, 0))) {
-    GST_DEBUG_OBJECT (dmabuf->upload, "input not dmabuf");
-    return FALSE;
+    GstVideoFrame frame1, frame2;
+
+    gst_video_frame_map (&frame1, in_info, buffer, GST_MAP_READ);
+
+    if (!dmabuf->pool) {
+      gboolean ret;
+      GstCaps *new_caps = gst_video_info_to_caps (&frame1.info);
+      gst_video_info_from_caps (in_info, new_caps);
+
+      ret =
+          _dma_buf_upload_setup_buffer_pool (&dmabuf->pool, NULL, new_caps,
+          in_info);
+      if (!ret) {
+        gst_video_frame_unmap (&frame1);
+        gst_caps_unref (new_caps);
+        GST_WARNING_OBJECT (dmabuf->upload, "no available buffer pool");
+        return FALSE;
+      }
+    }
+
+    if (!gst_buffer_pool_is_active (dmabuf->pool)
+        && gst_buffer_pool_set_active (dmabuf->pool, TRUE) != TRUE) {
+      gst_video_frame_unmap (&frame1);
+      GST_WARNING_OBJECT (dmabuf->upload, "buffer pool is not ok");
+      return FALSE;
+    }
+
+    if (dmabuf->inbuf)
+      gst_buffer_unref (dmabuf->inbuf);
+    dmabuf->inbuf = NULL;
+
+    gst_buffer_pool_acquire_buffer (dmabuf->pool, &dmabuf->inbuf, NULL);
+    if (!dmabuf->inbuf) {
+      gst_video_frame_unmap (&frame1);
+      GST_WARNING_OBJECT (dmabuf->upload, "acquire_buffer failed");
+      return FALSE;
+    }
+
+    GST_DEBUG_OBJECT (dmabuf->upload, "copy plane resolution (%d)x(%d)\n",
+        in_info->width, in_info->height);
+    gst_video_frame_map (&frame2, in_info, dmabuf->inbuf, GST_MAP_WRITE);
+    gst_video_frame_copy (&frame2, &frame1);
+    gst_video_frame_unmap (&frame1);
+    gst_video_frame_unmap (&frame2);
+
+    buffer = dmabuf->inbuf;
+    meta = gst_buffer_get_video_meta (buffer);
   }
 
   n_planes = GST_VIDEO_INFO_N_PLANES (in_info);
@@ -1533,14 +1655,14 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
       in_info->stride[i] = meta->stride[i];
     }
   }
-  
+
   if (crop) {
     in_info->width = MIN (crop->width, in_info->width);
     in_info->height = MIN (crop->height, in_info->height);
 
     GST_DEBUG_OBJECT (dmabuf->upload, "got crop meta (%d)x(%d)",
         in_info->width, in_info->height);
-    gst_buffer_remove_meta (buffer, (GstMeta *)crop);
+    gst_buffer_remove_meta (buffer, (GstMeta *) crop);
   }
 
   /* We cannot have multiple dmabuf per plane */
@@ -1573,6 +1695,7 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
 
   if (dmabuf->params)
     gst_gl_allocation_params_free ((GstGLAllocationParams *) dmabuf->params);
+
   if (!(dmabuf->params = gst_gl_video_allocation_params_new_wrapped_gl_handle
           (dmabuf->upload->context, NULL, out_info, -1, NULL, dmabuf->target,
               0, NULL, NULL, NULL)))
@@ -1662,6 +1785,60 @@ _dma_buf_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
     GstQuery * query)
 {
   /* nothing to do for now. */
+  struct DmabufUpload *upload = impl;
+  GstBufferPool *pool = NULL;
+  GstAllocator *allocator = NULL;
+  GstCaps *caps;
+  GstCapsFeatures *caps_features;
+  GstVideoInfo info;
+
+  gst_query_parse_allocation (query, &caps, NULL);
+
+  if (!gst_video_info_from_caps (&info, caps))
+    goto invalid_caps;
+
+  caps_features = gst_caps_get_features (caps, 0);
+  if (gst_caps_features_contains (caps_features, "memory:GLMemory")) {
+    GST_DEBUG ("upstream has memory:GLMemory feature");
+    return;
+  }
+#if GST_GL_HAVE_IONDMA
+  allocator = gst_ion_allocator_obtain ();
+#endif
+  if (!allocator) {
+    GST_WARNING ("New ion allocator failed.");
+    return;
+  }
+  GST_DEBUG ("create ion allocator(%p).", allocator);
+
+  if (gst_query_get_n_allocation_params (query) == 0) {
+    gst_query_add_allocation_param (query, allocator, NULL);
+  } else {
+    gst_query_set_nth_allocation_param (query, 0, allocator, NULL);
+  }
+
+  if (!_dma_buf_upload_setup_buffer_pool (&pool, allocator, caps, &info))
+    goto setup_failed;
+
+  if (gst_query_get_n_allocation_pools (query) == 0)
+    gst_query_add_allocation_pool (query, pool, info.size, 1, 30);
+  else
+    gst_query_set_nth_allocation_pool (query, 0, pool, info.size, 1, 30);
+
+  if (pool)
+    gst_object_unref (pool);
+
+  return;
+invalid_caps:
+  {
+    GST_WARNING_OBJECT (upload->upload, "invalid caps specified");
+    return;
+  }
+setup_failed:
+  {
+    GST_WARNING_OBJECT (upload->upload, "failed to setup buffer pool");
+    return;
+  }
 }
 
 static void
@@ -1669,11 +1846,17 @@ _dma_buf_upload_perform_gl_thread (GstGLContext * context,
     struct DmabufUpload *dmabuf)
 {
   GstGLMemoryAllocator *allocator;
+  guint n_mem, i;
 
   allocator =
       GST_GL_MEMORY_ALLOCATOR (gst_allocator_find
       (GST_GL_MEMORY_EGL_ALLOCATOR_NAME));
 
+  n_mem = GST_VIDEO_INFO_N_PLANES (dmabuf->params->v_info);
+  for (i = 0; i < n_mem; i++) {
+    if (!dmabuf->eglimage[i])
+      return;
+  }
   /* FIXME: buffer pool */
   dmabuf->outbuf = gst_buffer_new ();
   gst_gl_memory_setup_buffer (allocator, dmabuf->outbuf, dmabuf->params,
@@ -1718,6 +1901,12 @@ _dma_buf_upload_free (gpointer impl)
     gst_gl_allocation_params_free ((GstGLAllocationParams *) dmabuf->params);
   gst_egl_image_cache_unref (dmabuf->eglimage_cache);
 
+  if (dmabuf->inbuf)
+    gst_buffer_unref (dmabuf->inbuf);
+
+  if (dmabuf->pool)
+    gst_object_unref (dmabuf->pool);
+
   g_free (impl);
 }
 
@@ -2042,11 +2231,11 @@ _upload_meta_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
   gpointer handle;
 
   gl_apis =
-      gst_gl_api_to_string (gst_gl_context_get_gl_api (upload->upload->
-          context));
-  platform =
-      gst_gl_platform_to_string (gst_gl_context_get_gl_platform (upload->
+      gst_gl_api_to_string (gst_gl_context_get_gl_api (upload->
           upload->context));
+  platform =
+      gst_gl_platform_to_string (gst_gl_context_get_gl_platform
+      (upload->upload->context));
   handle = (gpointer) gst_gl_context_get_gl_context (upload->upload->context);
 
   gl_context =
@@ -2438,8 +2627,8 @@ _directviv_upload_new (GstGLUpload * upload)
 }
 
 static gboolean
-_directviv_upload_setup_buffer_pool (GstBufferPool **pool, GstAllocator *allocator,
-    GstCaps *caps, GstVideoInfo *info)
+_directviv_upload_setup_buffer_pool (GstBufferPool ** pool,
+    GstAllocator * allocator, GstCaps * caps, GstVideoInfo * info)
 {
   GstAllocationParams params;
   GstStructure *config;
@@ -2454,7 +2643,7 @@ _directviv_upload_setup_buffer_pool (GstBufferPool **pool, GstAllocator *allocat
 
   gst_allocation_params_init (&params);
 
-  /* if user not provide an allocator, then use default physical allocator*/
+  /* if user not provide an allocator, then use default physical allocator */
   if (!allocator) {
 #if GST_GL_HAVE_PHYMEM
     allocator = gst_phy_mem_allocator_obtain ();
@@ -2468,7 +2657,7 @@ _directviv_upload_setup_buffer_pool (GstBufferPool **pool, GstAllocator *allocat
   GST_DEBUG ("got allocator(%p).", allocator);
 
   if (*pool)
-    gst_object_unref(*pool);
+    gst_object_unref (*pool);
 
   *pool = gst_video_buffer_pool_new ();
   if (!*pool) {
@@ -2481,15 +2670,17 @@ _directviv_upload_setup_buffer_pool (GstBufferPool **pool, GstAllocator *allocat
 
   /* configure alignment for eglimage to import this dma-fd buffer */
   memset (&alignment, 0, sizeof (GstVideoAlignment));
-  alignment.padding_right = GST_ROUND_UP_N(width, DEFAULT_ALIGN) - width;
-  GST_DEBUG ("align buffer pool, w(%d) h(%d), padding_right (%d), padding_bottom (%d)",
+  alignment.padding_right = GST_ROUND_UP_N (width, DEFAULT_ALIGN) - width;
+  GST_DEBUG
+      ("align buffer pool, w(%d) h(%d), padding_right (%d), padding_bottom (%d)",
       width, height, alignment.padding_right, alignment.padding_bottom);
 
   /* the normal size of a frame */
   size = info->size;
   gst_buffer_pool_config_set_params (config, caps, size, 0, 30);
   gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META);
-  gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
+  gst_buffer_pool_config_add_option (config,
+      GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
   gst_buffer_pool_config_set_video_alignment (config, &alignment);
   gst_buffer_pool_config_set_allocator (config, allocator, &params);
 
@@ -2544,7 +2735,7 @@ _directviv_upload_transform_caps (gpointer impl, GstGLContext * context,
     } else {
       GstCaps *tmp;
       tmp = gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES
-        (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, GST_GL_DIRECTVIV_FORMAT));
+          (GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, GST_GL_DIRECTVIV_FORMAT));
       ret =
           _set_caps_features_with_passthrough (tmp,
           GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, passthrough);
@@ -2636,7 +2827,8 @@ _directviv_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
 
 #if GST_GL_HAVE_PHYMEM
   GstVideoInfo *in_info = &directviv->upload->priv->in_info;
-  GstVideoFormat fmt = GST_VIDEO_INFO_FORMAT (&directviv->upload->priv->out_info);
+  GstVideoFormat fmt =
+      GST_VIDEO_INFO_FORMAT (&directviv->upload->priv->out_info);
   if (fmt != GST_VIDEO_FORMAT_RGBA)
     return FALSE;
 
@@ -2646,17 +2838,21 @@ _directviv_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
     GstVideoInfo info;
 
     gst_video_frame_map (&frame1, in_info, buffer, GST_MAP_READ);
-    new_caps = gst_video_info_to_caps(&frame1.info);
-    gst_video_info_from_caps(&info, new_caps);
+    new_caps = gst_video_info_to_caps (&frame1.info);
+    gst_video_info_from_caps (&info, new_caps);
 
-    if (!directviv->pool || !_directviv_upload_buffer_pool_is_ok (directviv->pool,new_caps,info.size)) {
+    if (!directviv->pool
+        || !_directviv_upload_buffer_pool_is_ok (directviv->pool, new_caps,
+            info.size)) {
       gboolean ret;
       if (directviv->pool) {
-        gst_object_unref(directviv->pool);
+        gst_object_unref (directviv->pool);
         directviv->pool = NULL;
       }
 
-      ret = _directviv_upload_setup_buffer_pool (&directviv->pool, NULL, new_caps, in_info);
+      ret =
+          _directviv_upload_setup_buffer_pool (&directviv->pool, NULL, new_caps,
+          in_info);
       if (!ret) {
         gst_video_frame_unmap (&frame1);
         gst_caps_unref (new_caps);
@@ -2673,7 +2869,7 @@ _directviv_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
     }
 
     if (directviv->inbuf)
-      gst_buffer_unref(directviv->inbuf);
+      gst_buffer_unref (directviv->inbuf);
     directviv->inbuf = NULL;
 
     gst_buffer_pool_acquire_buffer (directviv->pool, &directviv->inbuf, NULL);
@@ -2683,7 +2879,8 @@ _directviv_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
       return FALSE;
     }
 
-    GST_DEBUG_OBJECT (directviv->upload, "copy plane resolution (%d)x(%d)\n", in_info->width, in_info->height);
+    GST_DEBUG_OBJECT (directviv->upload, "copy plane resolution (%d)x(%d)\n",
+        in_info->width, in_info->height);
     gst_video_frame_map (&frame2, in_info, directviv->inbuf, GST_MAP_WRITE);
     gst_video_frame_copy (&frame2, &frame1);
     gst_video_frame_unmap (&frame1);
diff --git a/gst-libs/gst/gl/meson.build b/gst-libs/gst/gl/meson.build
index 06ae1e739..07f150552 100644
--- a/gst-libs/gst/gl/meson.build
+++ b/gst-libs/gst/gl/meson.build
@@ -146,6 +146,7 @@ glconf_options = [
     'GST_GL_HAVE_DMABUF',
     'GST_GL_HAVE_VIV_DIRECTVIV',
     'GST_GL_HAVE_PHYMEM',
+    'GST_GL_HAVE_IONDMA',
 
     'GST_GL_HAVE_GLEGLIMAGEOES',
     'GST_GL_HAVE_GLCHAR',
@@ -570,6 +571,9 @@ if need_platform_egl != 'no'
     if cc.has_header('libdrm/drm_fourcc.h')
       gl_misc_deps += allocators_dep
       glconf.set('GST_GL_HAVE_DMABUF', 1)
+      if cc.has_header('linux/ion.h')
+        glconf.set('GST_GL_HAVE_IONDMA', 1)
+      endif
     endif
 
     egl_includes = '''
-- 
2.34.1


From 86cb37a95ee6973c9116816acb8587233c3750ad Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Tue, 11 Jun 2019 18:34:39 +0800
Subject: [PATCH 31/93] glupload: fix direct_dma_upload perform glthread fail

if we are in direct dma mode, we only have one eglimage

(cherry picked from commit d01a0cfbfe1b5ec73c6b41489de406bb52cb2e61)
---
 gst-libs/gst/gl/gstglupload.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index 455941906..1d7ac5f67 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -1852,7 +1852,10 @@ _dma_buf_upload_perform_gl_thread (GstGLContext * context,
       GST_GL_MEMORY_ALLOCATOR (gst_allocator_find
       (GST_GL_MEMORY_EGL_ALLOCATOR_NAME));
 
-  n_mem = GST_VIDEO_INFO_N_PLANES (dmabuf->params->v_info);
+  if (dmabuf->direct)
+    n_mem = 1;
+  else
+    n_mem = GST_VIDEO_INFO_N_PLANES (dmabuf->params->v_info);
   for (i = 0; i < n_mem; i++) {
     if (!dmabuf->eglimage[i])
       return;
-- 
2.34.1


From cbd6bacdb4db7f3ddd2db7e99b6743a8ac70cc26 Mon Sep 17 00:00:00 2001
From: Bing Song <bing.song@nxp.com>
Date: Fri, 4 Dec 2020 16:31:30 +0800
Subject: [PATCH 32/93] dmabufheaps: memory allocator based on dma-buf heaps.

dma-buf heaps can allocate CMA memory. it can be used for HW.

https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/merge_requests/958

Signed-off-by: Bing Song <bing.song@nxp.com>
(cherry picked from commit 48316b86c7cca8006d64b4616a2c17e55927bfe1)
---
 gst-libs/gst/allocators/gstdmabufheaps.c | 248 +++++++++++++++++++++++
 gst-libs/gst/allocators/gstdmabufheaps.h |  68 +++++++
 gst-libs/gst/allocators/meson.build      |  12 ++
 3 files changed, 328 insertions(+)
 create mode 100644 gst-libs/gst/allocators/gstdmabufheaps.c
 create mode 100644 gst-libs/gst/allocators/gstdmabufheaps.h

diff --git a/gst-libs/gst/allocators/gstdmabufheaps.c b/gst-libs/gst/allocators/gstdmabufheaps.c
new file mode 100644
index 000000000..69cd45ef0
--- /dev/null
+++ b/gst-libs/gst/allocators/gstdmabufheaps.c
@@ -0,0 +1,248 @@
+/*
+ * Copyright 2020 NXP
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <linux/dma-buf.h>
+#include <linux/dma-heap.h>
+#include <gst/allocators/gstdmabuf.h>
+#include "gstphysmemory.h"
+#include "gstdmabufheaps.h"
+
+GST_DEBUG_CATEGORY_STATIC (dmabufheaps_allocator_debug);
+#define GST_CAT_DEFAULT dmabufheaps_allocator_debug
+
+#define gst_dmabufheaps_allocator_parent_class parent_class
+
+#define DEFAULT_FD_FLAGS  (O_RDWR | O_CLOEXEC)
+#define DEFAULT_HEAP_FLAGS     DMA_HEAP_VALID_HEAP_FLAGS
+
+enum
+{
+  PROP_0,
+  PROP_FD_FLAGS,
+  PROP_HEAP_FLAGS,
+  PROP_LAST
+};
+
+static guintptr
+gst_dmabufheaps_allocator_get_phys_addr (GstPhysMemoryAllocator * allocator,
+    GstMemory * mem)
+{
+  struct dma_buf_phys dma_phys;
+  gint ret, fd;
+
+  if (!gst_is_dmabuf_memory (mem)) {
+    GST_ERROR ("isn't dmabuf memory");
+    return 0;
+  }
+
+  fd = gst_dmabuf_memory_get_fd (mem);
+  if (fd < 0) {
+    GST_ERROR ("dmabuf memory get fd failed");
+    return 0;
+  }
+
+  GST_DEBUG ("dmabufheaps DMA FD: %d", fd);
+
+  ret = ioctl (fd, DMA_BUF_IOCTL_PHYS, &dma_phys);
+  if (ret < 0)
+    return 0;
+
+  return dma_phys.phys;
+}
+
+static void
+gst_dmabufheaps_allocator_iface_init (gpointer g_iface)
+{
+  GstPhysMemoryAllocatorInterface *iface = g_iface;
+  iface->get_phys_addr = gst_dmabufheaps_allocator_get_phys_addr;
+}
+
+G_DEFINE_TYPE_WITH_CODE (GstDMABUFHEAPSAllocator, gst_dmabufheaps_allocator,
+    GST_TYPE_DMABUF_ALLOCATOR,
+    G_IMPLEMENT_INTERFACE (GST_TYPE_PHYS_MEMORY_ALLOCATOR,
+        gst_dmabufheaps_allocator_iface_init));
+
+static void
+gst_dmabufheaps_allocator_mem_init (void)
+{
+  GstAllocator *allocator =
+      g_object_new (gst_dmabufheaps_allocator_get_type (), NULL);
+  GstDMABUFHEAPSAllocator *self = GST_DMABUFHEAPS_ALLOCATOR (allocator);
+  gint fd;
+
+  fd = open ("/dev/dma_heap/linux,cma", O_RDWR);
+  if (fd < 0) {
+    GST_WARNING ("Could not open dmabufheaps driver");
+    g_object_unref (self);
+    return;
+  }
+
+  self->fd = fd;
+
+  gst_allocator_register (GST_ALLOCATOR_DMABUFHEAPS, allocator);
+}
+
+GstAllocator *
+gst_dmabufheaps_allocator_obtain (void)
+{
+  static GOnce dmabufheaps_allocator_once = G_ONCE_INIT;
+  GstAllocator *allocator;
+
+  g_once (&dmabufheaps_allocator_once,
+      (GThreadFunc) gst_dmabufheaps_allocator_mem_init, NULL);
+
+  allocator = gst_allocator_find (GST_ALLOCATOR_DMABUFHEAPS);
+  if (allocator == NULL)
+    GST_WARNING ("No allocator named %s found", GST_ALLOCATOR_DMABUFHEAPS);
+
+  return allocator;
+}
+
+static GstMemory *
+gst_dmabufheaps_allocator_alloc (GstAllocator * allocator, gsize size,
+    GstAllocationParams * params)
+{
+  GstDMABUFHEAPSAllocator *self = GST_DMABUFHEAPS_ALLOCATOR (allocator);
+  struct dma_heap_allocation_data data = { 0 };
+  GstMemory *mem;
+  gsize dmabufheaps_size;
+  gint dma_fd = -1;
+  gint ret;
+
+  if (self->fd < 0) {
+    GST_WARNING ("don't open dmabufheaps driver");
+    return NULL;
+  }
+
+  dmabufheaps_size = size + params->prefix + params->padding;
+  data.len = dmabufheaps_size,
+      data.fd_flags = self->fd_flags,
+      data.heap_flags = self->heap_flags,
+      ret = ioctl (self->fd, DMA_HEAP_IOCTL_ALLOC, &data);
+  if (ret < 0) {
+    GST_ERROR ("dmabufheaps allocate failed.");
+    return NULL;
+  }
+  dma_fd = data.fd;
+
+  mem =
+      gst_dmabuf_allocator_alloc_with_flags (allocator, dma_fd, size,
+      GST_FD_MEMORY_FLAG_KEEP_MAPPED);
+
+  GST_DEBUG ("dmabufheaps allocated size: %" G_GSIZE_FORMAT "DMA FD: %d",
+      dmabufheaps_size, dma_fd);
+
+  return mem;
+}
+
+static void
+gst_dmabufheaps_allocator_dispose (GObject * object)
+{
+  GstDMABUFHEAPSAllocator *self = GST_DMABUFHEAPS_ALLOCATOR (object);
+
+  if (self->fd > 0) {
+    close (self->fd);
+    self->fd = -1;
+  }
+
+  G_OBJECT_CLASS (parent_class)->dispose (object);
+}
+
+static void
+gst_dmabufheaps_allocator_set_property (GObject * object, guint prop_id,
+    const GValue * value, GParamSpec * pspec)
+{
+  GstDMABUFHEAPSAllocator *self = GST_DMABUFHEAPS_ALLOCATOR (object);
+
+  switch (prop_id) {
+    case PROP_FD_FLAGS:
+      self->fd_flags = g_value_get_uint (value);
+      break;
+    case PROP_HEAP_FLAGS:
+      self->heap_flags = g_value_get_uint (value);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+gst_dmabufheaps_allocator_get_property (GObject * object, guint prop_id,
+    GValue * value, GParamSpec * pspec)
+{
+  GstDMABUFHEAPSAllocator *self = GST_DMABUFHEAPS_ALLOCATOR (object);
+
+  switch (prop_id) {
+    case PROP_FD_FLAGS:
+      g_value_set_uint (value, self->fd_flags);
+      break;
+    case PROP_HEAP_FLAGS:
+      g_value_set_uint (value, self->heap_flags);
+      break;
+    default:
+      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+      break;
+  }
+}
+
+static void
+gst_dmabufheaps_allocator_class_init (GstDMABUFHEAPSAllocatorClass * klass)
+{
+  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
+  GstAllocatorClass *allocator_class = GST_ALLOCATOR_CLASS (klass);
+
+  gobject_class->dispose =
+      GST_DEBUG_FUNCPTR (gst_dmabufheaps_allocator_dispose);
+  gobject_class->set_property = gst_dmabufheaps_allocator_set_property;
+  gobject_class->get_property = gst_dmabufheaps_allocator_get_property;
+
+  g_object_class_install_property (gobject_class, PROP_FD_FLAGS,
+      g_param_spec_uint ("fd-flags", "FD Flags",
+          "DMABUFHEAPS fd flags", 0, G_MAXUINT32, DEFAULT_FD_FLAGS,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  g_object_class_install_property (gobject_class, PROP_HEAP_FLAGS,
+      g_param_spec_uint ("heap-flags", "Heap Flags",
+          "DMABUFHEAPS heap flags", 0, G_MAXUINT32, DEFAULT_HEAP_FLAGS,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  allocator_class->alloc = GST_DEBUG_FUNCPTR (gst_dmabufheaps_allocator_alloc);
+
+  GST_DEBUG_CATEGORY_INIT (dmabufheaps_allocator_debug, "dmabufheapsmemory", 0,
+      "DMA FD memory allocator based on dma-buf heaps");
+}
+
+static void
+gst_dmabufheaps_allocator_init (GstDMABUFHEAPSAllocator * self)
+{
+  GstAllocator *allocator = GST_ALLOCATOR (self);
+
+  allocator->mem_type = GST_ALLOCATOR_DMABUFHEAPS;
+
+  self->fd_flags = DEFAULT_FD_FLAGS;
+  self->heap_flags = DEFAULT_HEAP_FLAGS;
+}
diff --git a/gst-libs/gst/allocators/gstdmabufheaps.h b/gst-libs/gst/allocators/gstdmabufheaps.h
new file mode 100644
index 000000000..e5ddba6aa
--- /dev/null
+++ b/gst-libs/gst/allocators/gstdmabufheaps.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2020 NXP
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __GST_DMABUFHEAPS_H__
+#define __GST_DMABUFHEAPS_H__
+
+#include <gst/gst.h>
+#include <gst/allocators/gstdmabuf.h>
+#include <gst/allocators/allocators-prelude.h>
+
+G_BEGIN_DECLS
+
+typedef struct _GstDMABUFHEAPSAllocator GstDMABUFHEAPSAllocator;
+typedef struct _GstDMABUFHEAPSAllocatorClass GstDMABUFHEAPSAllocatorClass;
+typedef struct _GstDMABUFHEAPSMemory GstDMABUFHEAPSMemory;
+
+#define GST_ALLOCATOR_DMABUFHEAPS "dmabufheapsmem"
+
+#define GST_TYPE_DMABUFHEAPS_ALLOCATOR gst_dmabufheaps_allocator_get_type ()
+#define GST_IS_DMABUFHEAPS_ALLOCATOR(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), \
+    GST_TYPE_DMABUFHEAPS_ALLOCATOR))
+#define GST_DMABUFHEAPS_ALLOCATOR(obj) \
+  (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_DMABUFHEAPS_ALLOCATOR, GstDMABUFHEAPSAllocator))
+#define GST_DMABUFHEAPS_ALLOCATOR_CLASS(klass) \
+  (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_DMABUFHEAPS_ALLOCATOR, GstDMABUFHEAPSAllocatorClass))
+#define GST_DMABUFHEAPS_ALLOCATOR_CAST(obj) ((GstDMABUFHEAPSAllocator *)(obj))
+
+#define GST_DMABUFHEAPS_MEMORY_QUARK gst_dmabufheaps_memory_quark ()
+
+struct _GstDMABUFHEAPSAllocator
+{
+  GstDmaBufAllocator parent;
+
+  gint fd;
+  guint fd_flags;
+  guint heap_flags;
+};
+
+struct _GstDMABUFHEAPSAllocatorClass
+{
+  GstDmaBufAllocatorClass parent;
+};
+
+GST_ALLOCATORS_API
+GType gst_dmabufheaps_allocator_get_type (void);
+
+GST_ALLOCATORS_API
+GstAllocator* gst_dmabufheaps_allocator_obtain (void);
+
+G_END_DECLS
+
+#endif /* __GST_DMABUFHEAPS_H__ */
diff --git a/gst-libs/gst/allocators/meson.build b/gst-libs/gst/allocators/meson.build
index 31f990d24..00b5cf279 100644
--- a/gst-libs/gst/allocators/meson.build
+++ b/gst-libs/gst/allocators/meson.build
@@ -17,6 +17,12 @@ if cc.has_header('linux/ion.h')
   ]
 endif
 
+if cc.has_header('linux/dma-heap.h')
+  gst_allocators_headers += [
+    'gstdmabufheaps.h',
+  ]
+endif
+
 install_headers(gst_allocators_headers, subdir : 'gstreamer-1.0/gst/allocators/')
 
 gst_allocators_sources = files([
@@ -53,6 +59,12 @@ if cc.has_header('linux/ion.h')
   ]
 endif
 
+if cc.has_header('linux/dma-heap.h')
+  gst_allocators_sources += [
+    'gstdmabufheaps.c',
+  ]
+endif
+
 gstallocators = library('gstallocators-@0@'.format(api_version),
 
   gst_allocators_sources,
-- 
2.34.1


From 1671c5a40e4fc32b8dfd01238356b1d42502ee86 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Fri, 25 Nov 2016 14:48:44 +0800
Subject: [PATCH 33/93] gldownload: Add ion dmabuf support in gldownload

Support copy into dma-fb buffer if support the buffer format

Upstream Status: Inappropriate [i.MX specific]

Signed-off-by: Haihua Hu <jared.hu@nxp.com>

Conflicts:
	ext/gl/gstgldownloadelement.c
(cherry picked from commit 11e10b991cdb516599f576bb54065a709f7e642c)
---
 ext/gl/gstgldownloadelement.c    |  40 ++++-
 gst-libs/gst/gl/gstgl_fwd.h      |   4 +
 gst-libs/gst/gl/gstglmemorydma.c | 271 +++++++++++++++++++++++++++++++
 gst-libs/gst/gl/gstglmemorydma.h |  82 ++++++++++
 gst-libs/gst/gl/meson.build      |   6 +
 5 files changed, 397 insertions(+), 6 deletions(-)
 create mode 100644 gst-libs/gst/gl/gstglmemorydma.c
 create mode 100644 gst-libs/gst/gl/gstglmemorydma.h

diff --git a/ext/gl/gstgldownloadelement.c b/ext/gl/gstgldownloadelement.c
index d0d7bcf72..c5cb12336 100644
--- a/ext/gl/gstgldownloadelement.c
+++ b/ext/gl/gstgldownloadelement.c
@@ -36,6 +36,10 @@
 #include <gst/gl/gstglphymemory.h>
 #endif
 
+#if GST_GL_HAVE_IONDMA
+#include <gst/gl/gstglmemorydma.h>
+#endif
+
 GST_DEBUG_CATEGORY_STATIC (gst_gl_download_element_debug);
 #define GST_CAT_DEFAULT gst_gl_download_element_debug
 
@@ -1236,11 +1240,26 @@ gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
   GstBaseTransformClass *bclass = GST_BASE_TRANSFORM_GET_CLASS (bt);
   GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
   GstGLSyncMeta *in_sync_meta;
+  GstCaps *src_caps = gst_pad_get_current_caps (bt->srcpad);
   gint i, n;
   GstGLMemory *glmem;
 
-#if GST_GL_HAVE_PHYMEM
   glmem = gst_buffer_peek_memory (inbuf, 0);
+#if GST_GL_HAVE_IONDMA
+  if (gst_is_gl_memory_dma (glmem)) {
+    GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
+    GstVideoInfo info;
+
+    gst_video_info_from_caps (&info, src_caps);
+    *outbuf = gst_gl_memory_dma_buffer_to_gstbuffer (context, &info, inbuf);
+
+    GST_DEBUG_OBJECT (dl, "gl download with dma buf.");
+
+    return GST_FLOW_OK;
+  }
+#endif
+
+#if GST_GL_HAVE_PHYMEM
   if (gst_is_gl_physical_memory (glmem)) {
     GstCaps *src_caps;
     GstVideoInfo info;
@@ -1400,6 +1419,7 @@ gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
   GstCaps *caps;
   GstStructure *config;
   gsize size;
+  GstVideoFormat fmt;
 
   gst_query_parse_allocation (query, &caps, NULL);
   if (!gst_video_info_from_caps (&info, caps)) {
@@ -1407,13 +1427,21 @@ gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
     return FALSE;
   }
 
-  GST_DEBUG_OBJECT (bt, "video format is %s",
-      gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&info)));
-
   gst_allocation_params_init (&params);
 
+  fmt = GST_VIDEO_INFO_FORMAT (&info);
+
+  GST_DEBUG_OBJECT (bt, "video format is %s", gst_video_format_to_string (fmt));
+
+#if GST_GL_HAVE_IONDMA
+  if (fmt == GST_VIDEO_FORMAT_RGBA || fmt == GST_VIDEO_FORMAT_RGB16) {
+    allocator = gst_gl_memory_dma_allocator_obtain ();
+    GST_DEBUG_OBJECT (bt, "obtain dma memory allocator %p.", allocator);
+  }
+#endif
+
 #if GST_GL_HAVE_PHYMEM
-  if (gst_is_gl_physical_memory_supported_fmt (&info)) {
+  if (!allocator && gst_is_gl_physical_memory_supported_fmt (&info)) {
     allocator = gst_phy_mem_allocator_obtain ();
     GST_DEBUG_OBJECT (bt, "obtain physical memory allocator %p.", allocator);
   }
@@ -1448,7 +1476,7 @@ gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
       GST_BUFFER_POOL_OPTION_GL_SYNC_META);
 
   //set allocator to buffer pool
-  if (allocator){
+  if (allocator) {
     gst_buffer_pool_config_set_allocator (config, allocator, &params);
     gst_object_unref (allocator);
   }
diff --git a/gst-libs/gst/gl/gstgl_fwd.h b/gst-libs/gst/gl/gstgl_fwd.h
index 28af5c4a4..630dea118 100644
--- a/gst-libs/gst/gl/gstgl_fwd.h
+++ b/gst-libs/gst/gl/gstgl_fwd.h
@@ -51,6 +51,10 @@ typedef struct _GstGLMemory GstGLMemory;
 typedef struct _GstGLMemoryAllocator GstGLMemoryAllocator;
 typedef struct _GstGLMemoryAllocatorClass GstGLMemoryAllocatorClass;
 
+typedef struct _GstGLMemoryDMA GstGLMemoryDMA;
+typedef struct _GstGLMemoryDMAAllocator GstGLMemoryDMAAllocator;
+typedef struct _GstGLMemoryDMAAllocatorClass GstGLMemoryDMAAllocatorClass;
+
 typedef struct _GstGLMemoryPBO GstGLMemoryPBO;
 typedef struct _GstGLMemoryPBOAllocator GstGLMemoryPBOAllocator;
 typedef struct _GstGLMemoryPBOAllocatorClass GstGLMemoryPBOAllocatorClass;
diff --git a/gst-libs/gst/gl/gstglmemorydma.c b/gst-libs/gst/gl/gstglmemorydma.c
new file mode 100644
index 000000000..b7cae21e4
--- /dev/null
+++ b/gst-libs/gst/gl/gstglmemorydma.c
@@ -0,0 +1,271 @@
+/*
+ * GStreamer
+ * Copyright (c) 2016, Freescale Semiconductor, Inc. 
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundatdma; either
+ * versdma 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundatdma, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "gstglfuncs.h"
+
+#include <string.h>
+
+#include <gst/allocators/gstdmabuf.h>
+#include <gst/gl/gstglmemorydma.h>
+
+#if GST_GL_HAVE_IONDMA
+#include <gst/allocators/gstionmemory.h>
+#endif
+
+GST_DEBUG_CATEGORY_STATIC (GST_CAT_GL_DMA_MEMORY);
+#define GST_CAT_DEFAULT GST_CAT_GL_DMA_MEMORY
+
+#define parent_class gst_gl_memory_dma_allocator_parent_class
+G_DEFINE_TYPE (GstGLMemoryDMAAllocator, gst_gl_memory_dma_allocator,
+    GST_TYPE_GL_MEMORY_ALLOCATOR);
+
+static void
+gst_gl_memory_dma_init_instance (void)
+{
+  GstAllocator *ion_allocator = NULL;
+  GstGLMemoryDMAAllocator *_gl_allocator;
+
+  GST_DEBUG_CATEGORY_INIT (GST_CAT_GL_DMA_MEMORY, "glmemorydma", 0,
+      "OpenGL dma memory");
+
+#if GST_GL_HAVE_IONDMA
+  ion_allocator = gst_ion_allocator_obtain ();
+#endif
+
+  if (!ion_allocator)
+    return;
+
+  gst_gl_memory_init_once ();
+
+  _gl_allocator = (GstGLMemoryDMAAllocator *)
+      g_object_new (GST_TYPE_GL_MEMORY_DMA_ALLOCATOR, NULL);
+  _gl_allocator->ion_allocator = ion_allocator;
+
+  gst_allocator_register (GST_GL_MEMORY_DMA_ALLOCATOR_NAME,
+      gst_object_ref (_gl_allocator));
+}
+
+GstAllocator *
+gst_gl_memory_dma_allocator_obtain (void)
+{
+
+  static GOnce once = G_ONCE_INIT;
+  GstAllocator *allocator;
+
+  g_once (&once, (GThreadFunc) gst_gl_memory_dma_init_instance, NULL);
+
+  allocator = gst_allocator_find (GST_GL_MEMORY_DMA_ALLOCATOR_NAME);
+  if (allocator == NULL)
+    GST_WARNING ("No allocator named %s found",
+        GST_GL_MEMORY_DMA_ALLOCATOR_NAME);
+
+  return allocator;
+}
+
+static void
+gst_gl_memory_dma_allocator_dispose (GObject * object)
+{
+  GstGLMemoryDMAAllocator *gl_dma_alloc = GST_GL_MEMORY_DMA_ALLOCATOR (object);
+
+  if (gl_dma_alloc->ion_allocator) {
+    GST_DEBUG ("free ion allocator");
+    gst_object_unref (gl_dma_alloc->ion_allocator);
+    gl_dma_alloc->ion_allocator = NULL;
+  }
+
+  G_OBJECT_CLASS (parent_class)->dispose (object);
+}
+
+static gboolean
+_gl_mem_create (GstGLMemoryDMA * gl_mem, GError ** error)
+{
+  GstGLContext *context = gl_mem->mem.mem.context;
+  GstGLBaseMemoryAllocatorClass *alloc_class;
+  guint dma_fd;
+
+  alloc_class = GST_GL_BASE_MEMORY_ALLOCATOR_CLASS (parent_class);
+  if (!alloc_class->create ((GstGLBaseMemory *) gl_mem, error))
+    return FALSE;
+
+  dma_fd = gst_dmabuf_memory_get_fd ((GstMemory *) gl_mem->dma);
+
+  gl_mem->eglimage =
+      gst_egl_image_from_dmabuf (context, dma_fd, &gl_mem->mem.info, 0, 0);
+
+  if (!gl_mem->eglimage) {
+    GST_CAT_ERROR (GST_CAT_GL_DMA_MEMORY, "Can't allocate eglimage memory");
+    return FALSE;
+  }
+
+  const GstGLFuncs *gl = context->gl_vtable;
+
+  gl->ActiveTexture (GL_TEXTURE0);
+  gl->BindTexture (GL_TEXTURE_2D, gl_mem->mem.tex_id);
+  gl->EGLImageTargetTexture2D (GL_TEXTURE_2D,
+      gst_egl_image_get_image (gl_mem->eglimage));
+
+  GST_CAT_DEBUG (GST_CAT_GL_DMA_MEMORY,
+      "generated dma buffer %p fd %u texid %u", gl_mem, dma_fd,
+      gl_mem->mem.tex_id);
+
+  return TRUE;
+}
+
+static GstMemory *
+_gl_mem_alloc (GstAllocator * allocator, gsize size,
+    GstAllocationParams * params)
+{
+  g_warning ("Use gst_gl_base_memory_alloc () to allocate from this "
+      "GstGLMemoryDMA allocator");
+
+  return NULL;
+}
+
+static void
+_gl_mem_destroy (GstGLMemoryDMA * gl_mem)
+{
+  GST_CAT_DEBUG (GST_CAT_GL_DMA_MEMORY, "destroy gl dma buffer %p", gl_mem);
+
+  if (gl_mem->eglimage)
+    gst_egl_image_unref (gl_mem->eglimage);
+  gl_mem->eglimage = NULL;
+  if (gl_mem->dma)
+    gst_memory_unref (GST_MEMORY_CAST (gl_mem->dma));
+  gl_mem->dma = NULL;
+
+  GST_GL_BASE_MEMORY_ALLOCATOR_CLASS (parent_class)->destroy ((GstGLBaseMemory
+          *) gl_mem);
+}
+
+static GstGLMemoryDMA *
+_gl_mem_dma_alloc (GstGLBaseMemoryAllocator * allocator,
+    GstGLVideoAllocationParams * params)
+{
+  GstGLMemoryDMA *mem;
+  guint alloc_flags;
+  gsize size;
+  GstGLMemoryDMAAllocator *gl_dma_alloc =
+      GST_GL_MEMORY_DMA_ALLOCATOR (allocator);
+
+  alloc_flags = params->parent.alloc_flags;
+
+  g_return_val_if_fail (alloc_flags & GST_GL_ALLOCATION_PARAMS_ALLOC_FLAG_VIDEO,
+      NULL);
+
+  mem = g_new0 (GstGLMemoryDMA, 1);
+
+  mem->params = params->parent.alloc_params;
+
+  size =
+      gst_gl_get_plane_data_size (params->v_info, params->valign,
+      params->plane);
+  mem->dma =
+      gst_allocator_alloc (gl_dma_alloc->ion_allocator, size, mem->params);
+
+  if (!mem->dma) {
+    GST_CAT_ERROR (GST_CAT_GL_DMA_MEMORY, "Can't allocate dma memory size %d",
+        size);
+    g_free (mem);
+    return NULL;
+  }
+
+  gst_gl_memory_init (GST_GL_MEMORY_CAST (mem), GST_ALLOCATOR_CAST (allocator),
+      NULL, params->parent.context, params->target, params->tex_format,
+      params->parent.alloc_params, params->v_info, params->plane,
+      params->valign, params->parent.user_data, params->parent.notify);
+
+  return mem;
+}
+
+static void
+gst_gl_memory_dma_allocator_class_init (GstGLMemoryDMAAllocatorClass * klass)
+{
+  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
+  GstGLBaseMemoryAllocatorClass *gl_base;
+  GstAllocatorClass *allocator_class;
+
+  gl_base = (GstGLBaseMemoryAllocatorClass *) klass;
+  allocator_class = (GstAllocatorClass *) klass;
+
+  gl_base->alloc = (GstGLBaseMemoryAllocatorAllocFunction) _gl_mem_dma_alloc;
+  gl_base->create = (GstGLBaseMemoryAllocatorCreateFunction) _gl_mem_create;
+  gl_base->destroy = (GstGLBaseMemoryAllocatorDestroyFunction) _gl_mem_destroy;
+  gobject_class->dispose =
+      GST_DEBUG_FUNCPTR (gst_gl_memory_dma_allocator_dispose);
+
+  allocator_class->alloc = _gl_mem_alloc;
+}
+
+static void
+gst_gl_memory_dma_allocator_init (GstGLMemoryDMAAllocator * allocator)
+{
+  GstAllocator *alloc = GST_ALLOCATOR_CAST (allocator);
+
+  alloc->mem_type = GST_GL_MEMORY_DMA_ALLOCATOR_NAME;
+
+  GST_OBJECT_FLAG_SET (allocator, GST_ALLOCATOR_FLAG_CUSTOM_ALLOC);
+}
+
+gboolean
+gst_is_gl_memory_dma (GstMemory * mem)
+{
+  return mem != NULL && mem->allocator != NULL
+      && g_type_is_a (G_OBJECT_TYPE (mem->allocator),
+      GST_TYPE_GL_MEMORY_DMA_ALLOCATOR);
+}
+
+static void
+_finish_texture (GstGLContext * ctx, gpointer * data)
+{
+  GstGLFuncs *gl = ctx->gl_vtable;
+
+  gl->Finish ();
+}
+
+GstBuffer *
+gst_gl_memory_dma_buffer_to_gstbuffer (GstGLContext * ctx, GstVideoInfo * info,
+    GstBuffer * glbuf)
+{
+  GstBuffer *buf;
+  GstGLMemoryDMA *glmem;
+
+  gst_gl_context_thread_add (ctx, (GstGLContextThreadFunc) _finish_texture,
+      NULL);
+
+  glmem = gst_buffer_peek_memory (glbuf, 0);
+
+  buf = gst_buffer_new ();
+  gst_buffer_append_memory (buf, (GstMemory *) glmem->dma);
+  gst_memory_ref ((GstMemory *) glmem->dma);
+
+  gst_buffer_add_video_meta_full (buf, 0,
+      GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info),
+      GST_VIDEO_INFO_HEIGHT (info), 1, info->offset, info->stride);
+  GST_BUFFER_FLAGS (buf) = GST_BUFFER_FLAGS (glbuf);
+  GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (glbuf);
+  GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (glbuf);
+  GST_BUFFER_DURATION (buf) = GST_BUFFER_DURATION (glbuf);
+
+  return buf;
+}
diff --git a/gst-libs/gst/gl/gstglmemorydma.h b/gst-libs/gst/gl/gstglmemorydma.h
new file mode 100644
index 000000000..863425ab6
--- /dev/null
+++ b/gst-libs/gst/gl/gstglmemorydma.h
@@ -0,0 +1,82 @@
+/*
+ * GStreamer
+ * Copyright (c) 2016, Freescale Semiconductor, Inc. 
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundatdma; either
+ * versdma 2 of the License, or (at your optdma) any later versdma.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundatdma, Inc., 51 Franklin St, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef _GST_GL_MEMORY_DMA_H_
+#define _GST_GL_MEMORY_DMA_H_
+
+#include <gst/gst.h>
+#include <gst/gstallocator.h>
+#include <gst/gstmemory.h>
+#include <gst/video/video.h>
+
+#include <gst/gl/gl.h>
+#include <gst/gl/egl/gstglcontext_egl.h>
+#include <gst/gl/egl/gsteglimage.h>
+
+#include <gst/gl/gstglmemory.h>
+
+G_BEGIN_DECLS
+
+#define GST_TYPE_GL_MEMORY_DMA_ALLOCATOR (gst_gl_memory_dma_allocator_get_type())
+GST_GL_API
+GType gst_gl_memory_dma_allocator_get_type(void);
+
+#define GST_IS_GL_MEMORY_DMA_ALLOCATOR(obj)              (G_TYPE_CHECK_INSTANCE_TYPE ((obj), GST_TYPE_GL_MEMORY_DMA_ALLOCATOR))
+#define GST_IS_GL_MEMORY_DMA_ALLOCATOR_CLASS(klass)      (G_TYPE_CHECK_CLASS_TYPE ((klass), GST_TYPE_GL_MEMORY_DMA_ALLOCATOR))
+#define GST_GL_MEMORY_DMA_ALLOCATOR_GET_CLASS(obj)       (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_GL_MEMORY_DMA_ALLOCATOR, GstGLMemoryDMAAllocatorClass))
+#define GST_GL_MEMORY_DMA_ALLOCATOR(obj)                 (G_TYPE_CHECK_INSTANCE_CAST ((obj), GST_TYPE_GL_MEMORY_DMA_ALLOCATOR, GstGLMemoryDMAAllocator))
+#define GST_GL_MEMORY_DMA_ALLOCATOR_CLASS(klass)         (G_TYPE_CHECK_CLASS_CAST ((klass), GST_TYPE_GL_MEMORY_DMA_ALLOCATOR, GstGLAllocatorClass))
+#define GST_GL_MEMORY_DMA_ALLOCATOR_CAST(obj)            ((GstGLMemoryDMAAllocator *)(obj))
+
+struct _GstGLMemoryDMA
+{
+  GstGLMemory   mem;
+
+  /* <private> */
+  GstEGLImage	*eglimage;
+  GstMemory     *dma;
+  GstAllocationParams *params;
+};
+
+#define GST_GL_MEMORY_DMA_ALLOCATOR_NAME   "GLMemoryDMA"
+
+struct _GstGLMemoryDMAAllocator
+{
+  GstGLMemoryAllocator parent;
+  GstAllocator  *ion_allocator;
+};
+
+struct _GstGLMemoryDMAAllocatorClass
+{
+  GstGLMemoryAllocatorClass parent_class;
+};
+
+GST_GL_API
+GstAllocator *gst_gl_memory_dma_allocator_obtain (void);
+
+GST_GL_API
+gboolean      gst_is_gl_memory_dma                      (GstMemory * mem);
+
+GST_GL_API
+GstBuffer *   gst_gl_memory_dma_buffer_to_gstbuffer     (GstGLContext * ctx, GstVideoInfo * info, GstBuffer * glbuf);
+
+G_END_DECLS
+
+#endif /* _GST_GL_MEMORY_DMA_H_ */
diff --git a/gst-libs/gst/gl/meson.build b/gst-libs/gst/gl/meson.build
index 07f150552..7734964af 100644
--- a/gst-libs/gst/gl/meson.build
+++ b/gst-libs/gst/gl/meson.build
@@ -572,6 +572,12 @@ if need_platform_egl != 'no'
       gl_misc_deps += allocators_dep
       glconf.set('GST_GL_HAVE_DMABUF', 1)
       if cc.has_header('linux/ion.h')
+          gl_sources += [
+            'gstglmemorydma.c',
+          ]
+          gl_headers += [
+            'gstglmemorydma.h',
+          ]
         glconf.set('GST_GL_HAVE_IONDMA', 1)
       endif
     endif
-- 
2.34.1


From 7683ad88d6f7ba5b5e023cf68c9f32c0cb781cec Mon Sep 17 00:00:00 2001
From: Bing Song <bing.song@nxp.com>
Date: Mon, 14 Dec 2020 10:13:46 +0800
Subject: [PATCH 34/93] glupload: add dmaheaps memory support in glupload

Enable dmabuf heaps memory allocator in glupload
add support in glmemorydma

Signed-off-by: Bing Song <bing.song@nxp.com>
(cherry picked from commit e5d0cd36a127ee9e2327edd8c29b67e1d650a5c3)
---
 gst-libs/gst/gl/gstglconfig.h.meson |  1 +
 gst-libs/gst/gl/gstglmemorydma.c    | 11 ++++++++++-
 gst-libs/gst/gl/gstglupload.c       | 29 ++++++++++++++++++++++++-----
 gst-libs/gst/gl/meson.build         |  8 +++++++-
 4 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/gst-libs/gst/gl/gstglconfig.h.meson b/gst-libs/gst/gl/gstglconfig.h.meson
index 272717883..18127f11c 100644
--- a/gst-libs/gst/gl/gstglconfig.h.meson
+++ b/gst-libs/gst/gl/gstglconfig.h.meson
@@ -35,6 +35,7 @@ G_BEGIN_DECLS
 #mesondefine GST_GL_HAVE_VIV_DIRECTVIV
 #mesondefine GST_GL_HAVE_PHYMEM
 #mesondefine GST_GL_HAVE_IONDMA
+#mesondefine GST_GL_HAVE_DMABUFHEAPS
 
 #mesondefine GST_GL_HAVE_GLEGLIMAGEOES
 #mesondefine GST_GL_HAVE_GLCHAR
diff --git a/gst-libs/gst/gl/gstglmemorydma.c b/gst-libs/gst/gl/gstglmemorydma.c
index b7cae21e4..6cc4f4b20 100644
--- a/gst-libs/gst/gl/gstglmemorydma.c
+++ b/gst-libs/gst/gl/gstglmemorydma.c
@@ -29,6 +29,10 @@
 #include <gst/allocators/gstdmabuf.h>
 #include <gst/gl/gstglmemorydma.h>
 
+#if GST_GL_HAVE_DMABUFHEAPS
+#include <gst/allocators/gstdmabufheaps.h>
+#endif
+
 #if GST_GL_HAVE_IONDMA
 #include <gst/allocators/gstionmemory.h>
 #endif
@@ -49,8 +53,13 @@ gst_gl_memory_dma_init_instance (void)
   GST_DEBUG_CATEGORY_INIT (GST_CAT_GL_DMA_MEMORY, "glmemorydma", 0,
       "OpenGL dma memory");
 
+#if GST_GL_HAVE_DMABUFHEAPS
+  ion_allocator = gst_dmabufheaps_allocator_obtain ();
+#endif
+
 #if GST_GL_HAVE_IONDMA
-  ion_allocator = gst_ion_allocator_obtain ();
+  if (!ion_allocator)
+    ion_allocator = gst_ion_allocator_obtain ();
 #endif
 
   if (!ion_allocator)
diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index 1d7ac5f67..f6689c426 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -54,6 +54,10 @@
 #include "gstglphymemory.h"
 #endif
 
+#if GST_GL_HAVE_DMABUFHEAPS
+#include <gst/allocators/gstdmabufheaps.h>
+#endif
+
 #if GST_GL_HAVE_IONDMA
 #include <gst/allocators/gstionmemory.h>
 #endif
@@ -1468,8 +1472,12 @@ _dma_buf_upload_setup_buffer_pool (GstBufferPool ** pool,
 
   /* if user not provide an allocator, then use default ion allocator */
   if (!allocator) {
+#if GST_GL_HAVE_DMABUFHEAPS
+    allocator = gst_dmabufheaps_allocator_obtain ();
+#endif
 #if GST_GL_HAVE_IONDMA
-    allocator = gst_ion_allocator_obtain ();
+    if (!allocator)
+      allocator = gst_ion_allocator_obtain ();
 #endif
   }
 
@@ -1802,9 +1810,14 @@ _dma_buf_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
     GST_DEBUG ("upstream has memory:GLMemory feature");
     return;
   }
+#if GST_GL_HAVE_DMABUFHEAPS
+  allocator = gst_dmabufheaps_allocator_obtain ();
+#endif
 #if GST_GL_HAVE_IONDMA
-  allocator = gst_ion_allocator_obtain ();
+  if (!allocator)
+    allocator = gst_ion_allocator_obtain ();
 #endif
+
   if (!allocator) {
     GST_WARNING ("New ion allocator failed.");
     return;
@@ -2915,12 +2928,18 @@ _directviv_upload_propose_allocation (gpointer impl, GstQuery * decide_query,
   if (fmt != GST_VIDEO_FORMAT_RGBA)
     return;
 
-#if GST_GL_HAVE_IONDMA
   /* physical memory buffer pool was only proposed
    * when ion is not available to avoid allocator
    * overwrite in allocation query, we need keep use
    * ion when it is available */
-  allocator = gst_ion_allocator_obtain ();
+#if GST_GL_HAVE_DMABUFHEAPS
+  allocator = gst_dmabufheaps_allocator_obtain ();
+  if (allocator)
+    return;
+#endif
+#if GST_GL_HAVE_IONDMA
+  if (!allocator)
+    allocator = gst_ion_allocator_obtain ();
   if (allocator)
     return;
 #endif
@@ -3134,7 +3153,7 @@ _directviv_upload_free (gpointer impl)
     gst_buffer_unref (directviv->inbuf);
 
   if (directviv->pool)
-    gst_object_unref(directviv->pool);
+    gst_object_unref (directviv->pool);
 
   g_free (impl);
 }
diff --git a/gst-libs/gst/gl/meson.build b/gst-libs/gst/gl/meson.build
index 7734964af..16987cac9 100644
--- a/gst-libs/gst/gl/meson.build
+++ b/gst-libs/gst/gl/meson.build
@@ -147,6 +147,7 @@ glconf_options = [
     'GST_GL_HAVE_VIV_DIRECTVIV',
     'GST_GL_HAVE_PHYMEM',
     'GST_GL_HAVE_IONDMA',
+    'GST_GL_HAVE_DMABUFHEAPS',
 
     'GST_GL_HAVE_GLEGLIMAGEOES',
     'GST_GL_HAVE_GLCHAR',
@@ -572,13 +573,18 @@ if need_platform_egl != 'no'
       gl_misc_deps += allocators_dep
       glconf.set('GST_GL_HAVE_DMABUF', 1)
       if cc.has_header('linux/ion.h')
+        glconf.set('GST_GL_HAVE_IONDMA', 1)
+      endif
+      if cc.has_header('linux/dma-heap.h')
+        glconf.set('GST_GL_HAVE_DMABUFHEAPS', 1)
+      endif
+      if cc.has_header('linux/ion.h') or cc.has_header('linux/dma-heap.h')
           gl_sources += [
             'gstglmemorydma.c',
           ]
           gl_headers += [
             'gstglmemorydma.h',
           ]
-        glconf.set('GST_GL_HAVE_IONDMA', 1)
       endif
     endif
 
-- 
2.34.1


From 53872f66c6ec0545cf6dfea92054814de5674bf4 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Mon, 21 Feb 2022 13:41:34 +0800
Subject: [PATCH 35/93] glmemorydma: fix gldownload cannot use dmabuf on imx8

also need enable when we have dmabufheaps enabled

Signed-off-by: Haihua Hu <jared.hu@nxp.com>
---
 ext/gl/gstgldownloadelement.c    | 19 +++++++++++--------
 gst-libs/gst/gl/gstglmemorydma.c |  9 ++++-----
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/ext/gl/gstgldownloadelement.c b/ext/gl/gstgldownloadelement.c
index c5cb12336..7ad9efcaf 100644
--- a/ext/gl/gstgldownloadelement.c
+++ b/ext/gl/gstgldownloadelement.c
@@ -36,7 +36,7 @@
 #include <gst/gl/gstglphymemory.h>
 #endif
 
-#if GST_GL_HAVE_IONDMA
+#if GST_GL_HAVE_IONDMA || GST_GL_HAVE_DMABUFHEAPS
 #include <gst/gl/gstglmemorydma.h>
 #endif
 
@@ -1240,13 +1240,17 @@ gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
   GstBaseTransformClass *bclass = GST_BASE_TRANSFORM_GET_CLASS (bt);
   GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
   GstGLSyncMeta *in_sync_meta;
-  GstCaps *src_caps = gst_pad_get_current_caps (bt->srcpad);
   gint i, n;
+
+#if GST_GL_HAVE_IONDMA || GST_GL_HAVE_PHYMEM || GST_GL_HAVE_DMABUFHEAPS
+  GstCaps *src_caps = gst_pad_get_current_caps (bt->srcpad);
   GstGLMemory *glmem;
 
-  glmem = gst_buffer_peek_memory (inbuf, 0);
-#if GST_GL_HAVE_IONDMA
-  if (gst_is_gl_memory_dma (glmem)) {
+  glmem = (GstGLMemory *) gst_buffer_peek_memory (inbuf, 0);
+#endif
+
+#if GST_GL_HAVE_IONDMA || GST_GL_HAVE_DMABUFHEAPS
+  if (gst_is_gl_memory_dma ((GstMemory *) glmem)) {
     GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
     GstVideoInfo info;
 
@@ -1260,8 +1264,7 @@ gst_gl_download_element_prepare_output_buffer (GstBaseTransform * bt,
 #endif
 
 #if GST_GL_HAVE_PHYMEM
-  if (gst_is_gl_physical_memory (glmem)) {
-    GstCaps *src_caps;
+  if (gst_is_gl_physical_memory ((GstMemory *) glmem)) {
     GstVideoInfo info;
     GstGLContext *context = GST_GL_BASE_FILTER (bt)->context;
 
@@ -1433,7 +1436,7 @@ gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
 
   GST_DEBUG_OBJECT (bt, "video format is %s", gst_video_format_to_string (fmt));
 
-#if GST_GL_HAVE_IONDMA
+#if GST_GL_HAVE_IONDMA || GST_GL_HAVE_DMABUFHEAPS
   if (fmt == GST_VIDEO_FORMAT_RGBA || fmt == GST_VIDEO_FORMAT_RGB16) {
     allocator = gst_gl_memory_dma_allocator_obtain ();
     GST_DEBUG_OBJECT (bt, "obtain dma memory allocator %p.", allocator);
diff --git a/gst-libs/gst/gl/gstglmemorydma.c b/gst-libs/gst/gl/gstglmemorydma.c
index 6cc4f4b20..3160cb2fd 100644
--- a/gst-libs/gst/gl/gstglmemorydma.c
+++ b/gst-libs/gst/gl/gstglmemorydma.c
@@ -80,7 +80,7 @@ gst_gl_memory_dma_allocator_obtain (void)
 {
 
   static GOnce once = G_ONCE_INIT;
-  GstAllocator *allocator;
+  GstAllocator *allocator = NULL;
 
   g_once (&once, (GThreadFunc) gst_gl_memory_dma_init_instance, NULL);
 
@@ -110,6 +110,7 @@ static gboolean
 _gl_mem_create (GstGLMemoryDMA * gl_mem, GError ** error)
 {
   GstGLContext *context = gl_mem->mem.mem.context;
+  const GstGLFuncs *gl = context->gl_vtable;
   GstGLBaseMemoryAllocatorClass *alloc_class;
   guint dma_fd;
 
@@ -127,8 +128,6 @@ _gl_mem_create (GstGLMemoryDMA * gl_mem, GError ** error)
     return FALSE;
   }
 
-  const GstGLFuncs *gl = context->gl_vtable;
-
   gl->ActiveTexture (GL_TEXTURE0);
   gl->BindTexture (GL_TEXTURE_2D, gl_mem->mem.tex_id);
   gl->EGLImageTargetTexture2D (GL_TEXTURE_2D,
@@ -193,7 +192,7 @@ _gl_mem_dma_alloc (GstGLBaseMemoryAllocator * allocator,
       gst_allocator_alloc (gl_dma_alloc->ion_allocator, size, mem->params);
 
   if (!mem->dma) {
-    GST_CAT_ERROR (GST_CAT_GL_DMA_MEMORY, "Can't allocate dma memory size %d",
+    GST_CAT_ERROR (GST_CAT_GL_DMA_MEMORY, "Can't allocate dma memory size %" G_GSIZE_FORMAT,
         size);
     g_free (mem);
     return NULL;
@@ -262,7 +261,7 @@ gst_gl_memory_dma_buffer_to_gstbuffer (GstGLContext * ctx, GstVideoInfo * info,
   gst_gl_context_thread_add (ctx, (GstGLContextThreadFunc) _finish_texture,
       NULL);
 
-  glmem = gst_buffer_peek_memory (glbuf, 0);
+  glmem = (GstGLMemoryDMA *) gst_buffer_peek_memory (glbuf, 0);
 
   buf = gst_buffer_new ();
   gst_buffer_append_memory (buf, (GstMemory *) glmem->dma);
-- 
2.34.1


From d509da1b157126d5c69268dfcf12141104d78799 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Thu, 12 Apr 2018 16:22:25 +0800
Subject: [PATCH 36/93] gloverlaycompositor: add subtitle rotate support follow
 video

upstream status: Pending
https://bugzilla.gnome.org/show_bug.cgi?id=790470

(cherry picked from commit 48513da2bacabaaf3dad92f6cc96c20d387c3356)
---
 ext/gl/gstglimagesink.c                  | 33 ++++++++++++------------
 ext/gl/gstgloverlaycompositorelement.c   |  2 +-
 gst-libs/gst/gl/gstgloverlaycompositor.c | 11 ++++++--
 gst-libs/gst/gl/gstgloverlaycompositor.h |  2 +-
 4 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/ext/gl/gstglimagesink.c b/ext/gl/gstglimagesink.c
index 400026949..4d6b56234 100644
--- a/ext/gl/gstglimagesink.c
+++ b/ext/gl/gstglimagesink.c
@@ -2383,6 +2383,9 @@ gst_glimage_sink_on_draw (GstGLImageSink * gl_sink)
   GstGLWindow *window = NULL;
   gboolean do_redisplay = FALSE;
   GstSample *sample = NULL;
+  GstVideoAffineTransformationMeta *af_meta;
+  gfloat matrix[16];
+
   guint gl_target = gst_gl_texture_target_to_gl (gl_sink->texture_target);
 
   g_return_if_fail (GST_IS_GLIMAGE_SINK (gl_sink));
@@ -2512,27 +2515,23 @@ gst_glimage_sink_on_draw (GstGLImageSink * gl_sink)
     gl->ActiveTexture (GL_TEXTURE0);
     gl->BindTexture (gl_target, gl_sink->redisplay_texture);
     gst_gl_shader_set_uniform_1i (gl_sink->redisplay_shader, "tex", 0);
-    {
-      GstVideoAffineTransformationMeta *af_meta;
-      gfloat matrix[16];
 
-      af_meta =
-          gst_buffer_get_video_affine_transformation_meta
-          (gl_sink->stored_buffer[0]);
+    af_meta =
+        gst_buffer_get_video_affine_transformation_meta
+        (gl_sink->stored_buffer[0]);
 
-      if (gl_sink->transform_matrix) {
-        gfloat tmp[16];
+    if (gl_sink->transform_matrix) {
+      gfloat tmp[16];
 
-        gst_gl_get_affine_transformation_meta_as_ndc (af_meta, tmp);
-        gst_gl_multiply_matrix4 (tmp, gl_sink->transform_matrix, matrix);
-      } else {
-        gst_gl_get_affine_transformation_meta_as_ndc (af_meta, matrix);
-      }
-
-      gst_gl_shader_set_uniform_matrix_4fv (gl_sink->redisplay_shader,
-          "u_transformation", 1, FALSE, matrix);
+      gst_gl_get_affine_transformation_meta_as_ndc (af_meta, tmp);
+      gst_gl_multiply_matrix4 (tmp, gl_sink->transform_matrix, matrix);
+    } else {
+      gst_gl_get_affine_transformation_meta_as_ndc (af_meta, matrix);
     }
 
+    gst_gl_shader_set_uniform_matrix_4fv (gl_sink->redisplay_shader,
+        "u_transformation", 1, FALSE, matrix);
+
     gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0);
 
     gl->BindTexture (gl_target, 0);
@@ -2546,7 +2545,7 @@ gst_glimage_sink_on_draw (GstGLImageSink * gl_sink)
     if (gl_sink->ignore_alpha)
       gl->Disable (GL_BLEND);
 
-    gst_gl_overlay_compositor_draw_overlays (gl_sink->overlay_compositor);
+    gst_gl_overlay_compositor_draw_overlays (gl_sink->overlay_compositor, matrix);
   }
   /* end default opengl scene */
   window->is_drawing = FALSE;
diff --git a/ext/gl/gstgloverlaycompositorelement.c b/ext/gl/gstgloverlaycompositorelement.c
index 80ba9f386..ef4929827 100644
--- a/ext/gl/gstgloverlaycompositorelement.c
+++ b/ext/gl/gstgloverlaycompositorelement.c
@@ -341,7 +341,7 @@ gst_gl_overlay_compositor_element_callback (GstGLFilter * filter,
 
   GST_LOG_OBJECT (self, "drawing overlays");
 
-  gst_gl_overlay_compositor_draw_overlays (self->overlay_compositor);
+  gst_gl_overlay_compositor_draw_overlays (self->overlay_compositor, NULL);
 
   return TRUE;
 }
diff --git a/gst-libs/gst/gl/gstgloverlaycompositor.c b/gst-libs/gst/gl/gstgloverlaycompositor.c
index cc1111e7c..6231a36ec 100644
--- a/gst-libs/gst/gl/gstgloverlaycompositor.c
+++ b/gst-libs/gst/gl/gstgloverlaycompositor.c
@@ -554,7 +554,10 @@ gst_gl_overlay_compositor_init_gl (GstGLContext * context,
 
   if (!(compositor->shader =
           gst_gl_shader_new_link_with_stages (context, &error,
-              gst_glsl_stage_new_default_vertex (context),
+              gst_glsl_stage_new_with_string (context,
+                  GL_VERTEX_SHADER, GST_GLSL_VERSION_NONE,
+                  GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY,
+                  gst_gl_shader_string_vertex_mat4_vertex_transform),
               gst_glsl_stage_new_with_strings (context, GL_FRAGMENT_SHADER,
                   GST_GLSL_VERSION_NONE,
                   GST_GLSL_PROFILE_ES | GST_GLSL_PROFILE_COMPATIBILITY, 2,
@@ -711,7 +714,7 @@ gst_gl_overlay_compositor_upload_overlays (GstGLOverlayCompositor * compositor,
 }
 
 void
-gst_gl_overlay_compositor_draw_overlays (GstGLOverlayCompositor * compositor)
+gst_gl_overlay_compositor_draw_overlays (GstGLOverlayCompositor * compositor, gfloat *matrix)
 {
   const GstGLFuncs *gl = compositor->context->gl_vtable;
   if (compositor->overlays != NULL) {
@@ -723,6 +726,10 @@ gst_gl_overlay_compositor_draw_overlays (GstGLOverlayCompositor * compositor)
     gl->ActiveTexture (GL_TEXTURE0);
     gst_gl_shader_set_uniform_1i (compositor->shader, "tex", 0);
 
+    if (matrix)
+      gst_gl_shader_set_uniform_matrix_4fv (compositor->shader,
+          "u_transformation", 1, FALSE, matrix);
+
     for (l = compositor->overlays; l != NULL; l = l->next) {
       GstGLCompositionOverlay *overlay = (GstGLCompositionOverlay *) l->data;
       GstVideoOverlayFormatFlags flags;
diff --git a/gst-libs/gst/gl/gstgloverlaycompositor.h b/gst-libs/gst/gl/gstgloverlaycompositor.h
index 705963ee9..1c6e72f5c 100644
--- a/gst-libs/gst/gl/gstgloverlaycompositor.h
+++ b/gst-libs/gst/gl/gstgloverlaycompositor.h
@@ -83,7 +83,7 @@ void gst_gl_overlay_compositor_upload_overlays (GstGLOverlayCompositor * composi
         GstBuffer * buf);
 
 GST_GL_API
-void gst_gl_overlay_compositor_draw_overlays (GstGLOverlayCompositor * compositor);
+void gst_gl_overlay_compositor_draw_overlays (GstGLOverlayCompositor * compositor, gfloat * matrix);
 
 GST_GL_API
 GstCaps * gst_gl_overlay_compositor_add_caps(GstCaps * caps);
-- 
2.34.1


From 8959c42a75809ee9dbf03d8aad9813b054ca2680 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Thu, 28 Dec 2017 16:37:20 +0800
Subject: [PATCH 37/93] glimagesink: need add glFinish after eglswapbuffer

when video playback using glimagesink, video tearing occurs. eglswapbuffer
is an async call that is to say the buffer maybe still used by GPU
but we are already prepare next buffer.

upstream status: pending
https://bugzilla.gnome.org/show_bug.cgi?id=791937

(cherry picked from commit 58a3fdd0ba2a1c20ad6b35a6b2dc574930c4c93d)
---
 gst-libs/gst/gl/egl/gstglcontext_egl.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/gst-libs/gst/gl/egl/gstglcontext_egl.c b/gst-libs/gst/gl/egl/gstglcontext_egl.c
index a919014cc..30d38ddd6 100644
--- a/gst-libs/gst/gl/egl/gstglcontext_egl.c
+++ b/gst-libs/gst/gl/egl/gstglcontext_egl.c
@@ -36,6 +36,7 @@
 #include "gstegl.h"
 #include "../utils/opengl_versions.h"
 #include "../utils/gles_versions.h"
+#include "../gstglfuncs.h"
 
 #if GST_GL_HAVE_WINDOW_X11
 #include "../x11/gstglwindow_x11.h"
@@ -1303,10 +1304,13 @@ static void
 gst_gl_context_egl_swap_buffers (GstGLContext * context)
 {
   GstGLContextEGL *egl;
+  const GstGLFuncs *gl;
 
   egl = GST_GL_CONTEXT_EGL (context);
+  gl = context->gl_vtable;
 
   eglSwapBuffers (egl->egl_display, egl->egl_surface);
+  gl->Finish();
 }
 
 static GstGLAPI
-- 
2.34.1


From 26e6e8da92c3b7f3d0f1abc2ce4af54bc38a75cc Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Thu, 15 Oct 2020 16:47:21 +0800
Subject: [PATCH 38/93] gl/viv-fb: install viv-fb header into include dir

previous patch is in below link, but this change is
missing on meson build

https://bugzilla.gnome.org/show_bug.cgi?id=795499

Conflicts:
	gst-libs/gst/gl/meson.build
(cherry picked from commit 8b5a51210302a0ff09bf112efe3ca4555fe25a3b)
---
 gst-libs/gst/gl/meson.build | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gst-libs/gst/gl/meson.build b/gst-libs/gst/gl/meson.build
index 16987cac9..3d0782da1 100644
--- a/gst-libs/gst/gl/meson.build
+++ b/gst-libs/gst/gl/meson.build
@@ -991,6 +991,7 @@ if need_platform_egl != 'no' and need_win_viv_fb != 'no'
       'viv-fb/gstglviv-fb.h',
       'viv-fb/gstgldisplay_viv_fb.h',
     ])
+    install_headers(gl_viv_fb_headers, subdir : 'gstreamer-1.0/gst/gl/viv-fb')
   endif
 endif
 
-- 
2.34.1


From ff9067e15d2b1a753a2f29e22cc0bed416b81464 Mon Sep 17 00:00:00 2001
From: Carlos Rafael Giani <crg7475@mailbox.org>
Date: Tue, 21 May 2019 14:01:11 +0200
Subject: [PATCH 39/93] viv-fb: Make sure config.h is included

This prevents build errors due to missing GST_API_* symbols

Upstream-Status: Pending

Signed-off-by: Carlos Rafael Giani <crg7475@mailbox.org>
---
 gst-libs/gst/gl/gl-prelude.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/gst-libs/gst/gl/gl-prelude.h b/gst-libs/gst/gl/gl-prelude.h
index 85fca5a0f..946c72913 100644
--- a/gst-libs/gst/gl/gl-prelude.h
+++ b/gst-libs/gst/gl/gl-prelude.h
@@ -22,6 +22,10 @@
 #ifndef __GST_GL_PRELUDE_H__
 #define __GST_GL_PRELUDE_H__
 
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
 #include <gst/gst.h>
 
 #ifdef BUILDING_GST_GL
-- 
2.34.1


From e31070441a11498817284352b4e545d8341dd379 Mon Sep 17 00:00:00 2001
From: Bing Song <bing.song@nxp.com>
Date: Mon, 21 Dec 2020 17:10:56 +0800
Subject: [PATCH 40/93] MMFMWK-8918 [dmabufheaps] enable dmabuf heaps memory
 allocator.

Add one workaround for cache flush by get physical address as
the buffer get physical address can't be flush rightly.

Signed-off-by: Bing Song <bing.song@nxp.com>
(cherry picked from commit d6ad337837085f8dc1ef29eae6844edbf3a3915f)
---
 gst-libs/gst/allocators/gstdmabuf.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/gst-libs/gst/allocators/gstdmabuf.c b/gst-libs/gst/allocators/gstdmabuf.c
index 15de4c83e..e163b7280 100644
--- a/gst-libs/gst/allocators/gstdmabuf.c
+++ b/gst-libs/gst/allocators/gstdmabuf.c
@@ -69,9 +69,11 @@ gst_dmabuf_mem_map (GstMemory * gmem, GstMapInfo * info, gsize maxsize)
 
 #ifdef HAVE_LINUX_DMA_BUF_H
   if (ret) {
+    struct dma_buf_phys dma_phys;
     if (ioctl (gst_fd_memory_get_fd (gmem), DMA_BUF_IOCTL_SYNC, &sync) < 0)
       GST_WARNING_OBJECT (allocator, "Failed to synchronize DMABuf: %s (%i)",
           g_strerror (errno), errno);
+    ioctl (gst_fd_memory_get_fd (gmem), DMA_BUF_IOCTL_PHYS, &dma_phys);
   }
 #endif
 
@@ -84,6 +86,7 @@ gst_dmabuf_mem_unmap (GstMemory * gmem, GstMapInfo * info)
   GstAllocator *allocator = gmem->allocator;
 #ifdef HAVE_LINUX_DMA_BUF_H
   struct dma_buf_sync sync = { DMA_BUF_SYNC_END };
+  struct dma_buf_phys dma_phys;
 
   if (info->flags & GST_MAP_READ)
     sync.flags |= DMA_BUF_SYNC_READ;
@@ -94,6 +97,7 @@ gst_dmabuf_mem_unmap (GstMemory * gmem, GstMapInfo * info)
   if (ioctl (gst_fd_memory_get_fd (gmem), DMA_BUF_IOCTL_SYNC, &sync) < 0)
     GST_WARNING_OBJECT (allocator, "Failed to synchronize DMABuf: %s (%i)",
         g_strerror (errno), errno);
+  ioctl (gst_fd_memory_get_fd (gmem), DMA_BUF_IOCTL_PHYS, &dma_phys);
 #else
   GST_WARNING_OBJECT (allocator, "Using DMABuf without synchronization.");
 #endif
@@ -156,7 +160,8 @@ gst_dmabuf_allocator_alloc (GstAllocator * allocator, gint fd, gsize size)
 {
   g_return_val_if_fail (GST_IS_DMABUF_ALLOCATOR (allocator), NULL);
 
-  return gst_fd_allocator_alloc (allocator, fd, size, GST_FD_MEMORY_FLAG_KEEP_MAPPED);
+  return gst_fd_allocator_alloc (allocator, fd, size,
+      GST_FD_MEMORY_FLAG_KEEP_MAPPED);
 }
 
 /**
-- 
2.34.1


From 9c7b4075bc41fafb33e6b66788a6bf2bc974e54c Mon Sep 17 00:00:00 2001
From: Bing Song <bing.song@nxp.com>
Date: Wed, 23 Dec 2020 13:19:24 +0800
Subject: [PATCH 41/93] MMFMWK-8918 [dmabufheaps] enable dmabuf heaps memory
 allocator.

Remove DMABUF SYNC temporarily as it will cause issue.

Signed-off-by: Bing Song <bing.song@nxp.com>
(cherry picked from commit 6dd8dc7566a76354e3283dc4e2dcecae817bf20e)
---
 gst-libs/gst/allocators/gstdmabuf.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/gst-libs/gst/allocators/gstdmabuf.c b/gst-libs/gst/allocators/gstdmabuf.c
index e163b7280..7e42b06db 100644
--- a/gst-libs/gst/allocators/gstdmabuf.c
+++ b/gst-libs/gst/allocators/gstdmabuf.c
@@ -70,9 +70,9 @@ gst_dmabuf_mem_map (GstMemory * gmem, GstMapInfo * info, gsize maxsize)
 #ifdef HAVE_LINUX_DMA_BUF_H
   if (ret) {
     struct dma_buf_phys dma_phys;
-    if (ioctl (gst_fd_memory_get_fd (gmem), DMA_BUF_IOCTL_SYNC, &sync) < 0)
-      GST_WARNING_OBJECT (allocator, "Failed to synchronize DMABuf: %s (%i)",
-          g_strerror (errno), errno);
+    /* if (ioctl (gst_fd_memory_get_fd (gmem), DMA_BUF_IOCTL_SYNC, &sync) < 0)
+       GST_WARNING_OBJECT (allocator, "Failed to synchronize DMABuf: %s (%i)",
+       g_strerror (errno), errno); */
     ioctl (gst_fd_memory_get_fd (gmem), DMA_BUF_IOCTL_PHYS, &dma_phys);
   }
 #endif
@@ -94,9 +94,9 @@ gst_dmabuf_mem_unmap (GstMemory * gmem, GstMapInfo * info)
   if (info->flags & GST_MAP_WRITE)
     sync.flags |= DMA_BUF_SYNC_WRITE;
 
-  if (ioctl (gst_fd_memory_get_fd (gmem), DMA_BUF_IOCTL_SYNC, &sync) < 0)
-    GST_WARNING_OBJECT (allocator, "Failed to synchronize DMABuf: %s (%i)",
-        g_strerror (errno), errno);
+  /* if (ioctl (gst_fd_memory_get_fd (gmem), DMA_BUF_IOCTL_SYNC, &sync) < 0)
+     GST_WARNING_OBJECT (allocator, "Failed to synchronize DMABuf: %s (%i)",
+     g_strerror (errno), errno); */
   ioctl (gst_fd_memory_get_fd (gmem), DMA_BUF_IOCTL_PHYS, &dma_phys);
 #else
   GST_WARNING_OBJECT (allocator, "Using DMABuf without synchronization.");
-- 
2.34.1


From 07a7b68ab0cb5c6650b3c7b89c3661e5677afa49 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Tue, 24 Jul 2018 14:05:17 +0800
Subject: [PATCH 42/93] MMFMWK-8077 glupload: respect to downstream format
 priority

Make upstream caps format respect to downstream's priority
to try hard to choose RGBA format if possible

upstream status: imx specific

(cherry picked from commit 76fc25cf894e065c303cbb7684e2a3e3f8fd468e)
---
 gst-libs/gst/gl/gstglupload.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index f6689c426..6da247892 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -3849,7 +3849,7 @@ gst_gl_upload_transform_caps (GstGLUpload * upload, GstGLContext * context,
   }
 
   if (filter) {
-    result = gst_caps_intersect_full (filter, tmp, GST_CAPS_INTERSECT_FIRST);
+    result = gst_caps_intersect_full (tmp, filter, GST_CAPS_INTERSECT_FIRST);
     gst_caps_unref (tmp);
   } else {
     result = tmp;
-- 
2.34.1


From c6dee055af6e91af89d2aa1e3f0214d63603982f Mon Sep 17 00:00:00 2001
From: Arnaud Pouliquen <arnaud.pouliquen@st.com>
Date: Fri, 30 Oct 2015 09:45:27 +0100
Subject: [PATCH 43/93] alsasink: endianess fix for iec61937

if format supported by driver is S16_LE, data should be swapped only for big endian systems
https://bugzilla.gnome.org/show_bug.cgi?id=757258

Signed-off-by: Arnaud Pouliquen <arnaud.pouliquen@st.com>
(cherry picked from commit 004e321b8ee8029a26620bc2868d28341472d161)
---
 ext/alsa/gstalsasink.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ext/alsa/gstalsasink.c b/ext/alsa/gstalsasink.c
index 5d73a1a12..259f0bee2 100644
--- a/ext/alsa/gstalsasink.c
+++ b/ext/alsa/gstalsasink.c
@@ -451,10 +451,12 @@ set_hwparams (GstAlsaSink * alsa)
     /* Try to use big endian first else fallback to le and swap bytes */
     if (snd_pcm_hw_params_set_format (alsa->handle, params, alsa->format) < 0) {
       alsa->format = SND_PCM_FORMAT_S16_LE;
-      alsa->need_swap = TRUE;
+      /* swap needed if system is big endian */
+      alsa->need_swap = (G_BYTE_ORDER == G_BIG_ENDIAN);
       GST_DEBUG_OBJECT (alsa, "falling back to little endian with swapping");
     } else {
-      alsa->need_swap = FALSE;
+      /* swap needed if system is little endian */
+      alsa->need_swap = (G_BYTE_ORDER != G_BIG_ENDIAN);
     }
   }
   CHECK (snd_pcm_hw_params_set_format (alsa->handle, params, alsa->format),
-- 
2.34.1


From 5605a9ea119b6c60a6babbbaa8c2d56c4c73191b Mon Sep 17 00:00:00 2001
From: Arnaud Pouliquen <arnaud.pouliquen@st.com>
Date: Fri, 30 Oct 2015 09:45:46 +0100
Subject: [PATCH 44/93] iec61937: incoherent endianness in payload

Order header based on specified endianness and not system endianness
Fix following bugzilla
https://bugzilla.gnome.org/show_bug.cgi?id=678021
https://bugzilla.gnome.org/show_bug.cgi?id=757258

Signed-off-by: Arnaud Pouliquen <arnaud.pouliquen@st.com>
(cherry picked from commit 7591f5f3daf437d3ebdbe8fa71be2c01558af378)
---
 gst-libs/gst/audio/gstaudioiec61937.c | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/gst-libs/gst/audio/gstaudioiec61937.c b/gst-libs/gst/audio/gstaudioiec61937.c
index 948325e84..a19291331 100644
--- a/gst-libs/gst/audio/gstaudioiec61937.c
+++ b/gst-libs/gst/audio/gstaudioiec61937.c
@@ -161,14 +161,16 @@ gst_audio_iec61937_payload (const guint8 * src, guint src_n, guint8 * dst,
     guint dst_n, const GstAudioRingBufferSpec * spec, gint endianness)
 {
   guint i, tmp;
-#if G_BYTE_ORDER == G_BIG_ENDIAN
-  guint8 zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5, six = 6,
-      seven = 7;
-#else
-  /* We need to send the data byte-swapped */
-  guint8 zero = 1, one = 0, two = 3, three = 2, four = 5, five = 4, six = 7,
-      seven = 6;
-#endif
+  guint8 zero = 0, one = 1, two = 2, three = 3, four = 4, five = 5, six = 6;
+  guint8 seven = 7;
+
+  if (G_BYTE_ORDER != endianness) {
+    /* We need to send the data byte-swapped */
+    zero = one--;
+    two = three--;
+    four = five--;
+    six = seven--;
+  }
 
   g_return_val_if_fail (src != NULL, FALSE);
   g_return_val_if_fail (dst != NULL, FALSE);
-- 
2.34.1


From 06a701f76d3b6d93bc48f45a0b26aa90e26d71a9 Mon Sep 17 00:00:00 2001
From: Arnaud Pouliquen <arnaud.pouliquen@st.com>
Date: Tue, 3 Nov 2015 14:13:10 +0100
Subject: [PATCH 45/93] alsasink: fix iec958 format detection

Add specific function that first close the current pcm device
before trying to re-open it with AES parameters.
https://bugzilla.gnome.org/show_bug.cgi?id=757258

Signed-off-by: Arnaud Pouliquen <arnaud.pouliquen@st.com>
(cherry picked from commit a2177475f7ea2d1170785e11971b1da8c510367f)
---
 ext/alsa/gstalsa.c     | 61 ++++++++++++++++++++++++++++++++----------
 ext/alsa/gstalsa.h     |  4 +++
 ext/alsa/gstalsasink.c | 11 +++++++-
 3 files changed, 61 insertions(+), 15 deletions(-)

diff --git a/ext/alsa/gstalsa.c b/ext/alsa/gstalsa.c
index e22eec369..c83516857 100644
--- a/ext/alsa/gstalsa.c
+++ b/ext/alsa/gstalsa.c
@@ -700,7 +700,6 @@ gst_alsa_probe_supported_formats (GstObject * obj, gchar * device,
     snd_pcm_t * handle, const GstCaps * template_caps)
 {
   snd_pcm_hw_params_t *hw_params;
-  snd_pcm_stream_t stream_type;
   GstCaps *caps;
   GstCaps *dsd_caps;
   gint err;
@@ -709,8 +708,6 @@ gst_alsa_probe_supported_formats (GstObject * obj, gchar * device,
   if ((err = snd_pcm_hw_params_any (handle, hw_params)) < 0)
     goto error;
 
-  stream_type = snd_pcm_stream (handle);
-
   /* Try detecting PCM */
 
   caps = gst_alsa_detect_formats (obj, hw_params,
@@ -761,17 +758,6 @@ gst_alsa_probe_supported_formats (GstObject * obj, gchar * device,
     GST_INFO_OBJECT (obj, "DSD support not detected");
   }
 
-  /* Try opening IEC958 device to see if we can support that format (playback
-   * only for now but we could add SPDIF capture later) */
-  if (stream_type == SND_PCM_STREAM_PLAYBACK) {
-    snd_pcm_t *pcm = gst_alsa_open_iec958_pcm (obj, device);
-
-    if (G_LIKELY (pcm)) {
-      gst_caps_append (caps, gst_caps_from_string (PASSTHROUGH_CAPS));
-      snd_pcm_close (pcm);
-    }
-  }
-
   snd_pcm_hw_params_free (hw_params);
   return caps;
 
@@ -791,6 +777,53 @@ subroutine_error:
   }
 }
 
+/*
+ * gst_alsa_probe_supported_formats:
+ *
+ * Takes the template caps and returns the subset which is actually
+ * supported by this device.
+ */
+gboolean gst_alsa_iec958_formats_supported (GstObject * obj, gchar * device,
+    snd_pcm_t ** handle)
+{
+  snd_pcm_stream_t stream_type;
+  gboolean supported = FALSE;
+  snd_pcm_t *pcm;
+  gint err;
+
+  /* Try opening IEC958 device to see if we can support that format (playback
+   * only for now but we could add SPDIF capture later) */
+  stream_type = snd_pcm_stream (*handle);
+  if (stream_type == SND_PCM_STREAM_PLAYBACK) {
+    /* Close the exiting device to re-open it as iec958 device*/
+    err = snd_pcm_close(*handle);
+    if (G_UNLIKELY (err < 0)) {
+      GST_DEBUG_OBJECT (obj, " failed closing alsa device: %s",
+        snd_strerror (err));
+      return FALSE;
+    }
+
+    pcm = gst_alsa_open_iec958_pcm(GST_OBJECT (obj), device);
+    if (G_LIKELY (pcm)) {
+      supported = TRUE;
+      snd_pcm_close (pcm);
+    }
+
+    /* Reopen device in raw audio mode*/
+    err = snd_pcm_open (handle, device, SND_PCM_STREAM_PLAYBACK,
+        SND_PCM_NONBLOCK);
+    if (G_UNLIKELY (err < 0)) {
+       GST_DEBUG_OBJECT (obj, "failed re-opening alsa device: %s",
+           snd_strerror (err));
+    }
+    return supported;
+  }
+
+  GST_DEBUG_OBJECT (obj, "only supported for playback");
+  return FALSE;
+
+}
+
 /* returns the card name when the device number is unknown or -1 */
 static gchar *
 gst_alsa_find_device_name_no_handle (GstObject * obj, const gchar * devcard,
diff --git a/ext/alsa/gstalsa.h b/ext/alsa/gstalsa.h
index 46324a3a8..d9e62845c 100644
--- a/ext/alsa/gstalsa.h
+++ b/ext/alsa/gstalsa.h
@@ -58,6 +58,10 @@ GstCaps * gst_alsa_probe_supported_formats (GstObject      * obj,
                                             snd_pcm_t      * handle,
                                             const GstCaps  * template_caps);
 
+gboolean gst_alsa_iec958_formats_supported (GstObject * obj,
+                                            gchar * device,
+                                            snd_pcm_t ** handle);
+
 gchar   * gst_alsa_find_device_name (GstObject        * obj,
                                      const gchar      * device,
                                      snd_pcm_t        * handle,
diff --git a/ext/alsa/gstalsasink.c b/ext/alsa/gstalsasink.c
index 259f0bee2..9a356e0e5 100644
--- a/ext/alsa/gstalsasink.c
+++ b/ext/alsa/gstalsasink.c
@@ -329,10 +329,19 @@ gst_alsasink_getcaps (GstBaseSink * bsink, GstCaps * filter)
       sink->handle, templ_caps);
   gst_caps_unref (templ_caps);
 
+  /* Try opening IEC958 device to see if we can support that format (playback
+   * only for now but we could add SPDIF capture later)
+   */
+
+  if (gst_alsa_iec958_formats_supported(GST_OBJECT (sink), sink->device,
+    &sink->handle) == TRUE) {
+    GST_LOG_OBJECT (sink, "Add pass-through capabilities");
+    gst_caps_append (caps, gst_caps_from_string (PASSTHROUGH_CAPS));
+  }
+
   if (caps) {
     sink->cached_caps = gst_caps_ref (caps);
   }
-
   GST_OBJECT_UNLOCK (sink);
 
   GST_INFO_OBJECT (sink, "returning caps %" GST_PTR_FORMAT, caps);
-- 
2.34.1


From 66d732b0d7a092f6d83c55dc07cacebed16bfafd Mon Sep 17 00:00:00 2001
From: Arnaud Pouliquen <arnaud.pouliquen@st.com>
Date: Fri, 6 Nov 2015 10:01:50 +0100
Subject: [PATCH 46/93] iec61937: force hw_param channels to stereo

In case of pass-through, iec frames are sent over a stereo PCM
stream. So alsa device should be configured with channels = 2.
This is needed to be compatible with SPDIF electrical/optical output
that supports only stereo in PCM mode and up to 8 channels in iec61937 mode.

Signed-off-by: Arnaud Pouliquen <arnaud.pouliquen@st.com>
(cherry picked from commit a6c5c17c00d03f1a11c6cd44c678064740b2fd1e)
---
 ext/alsa/gstalsasink.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/ext/alsa/gstalsasink.c b/ext/alsa/gstalsasink.c
index 9a356e0e5..12752ebb8 100644
--- a/ext/alsa/gstalsasink.c
+++ b/ext/alsa/gstalsasink.c
@@ -841,6 +841,7 @@ alsasink_parse_spec (GstAlsaSink * alsa, GstAudioRingBufferSpec * spec)
         default:
           goto error;
       }
+      alsa->channels = GST_AUDIO_INFO_CHANNELS (&spec->info);
       break;
     case GST_AUDIO_RING_BUFFER_FORMAT_TYPE_DSD:
       switch (GST_AUDIO_RING_BUFFER_SPEC_DSD_FORMAT (spec)) {
@@ -865,9 +866,11 @@ alsasink_parse_spec (GstAlsaSink * alsa, GstAudioRingBufferSpec * spec)
       break;
     case GST_AUDIO_RING_BUFFER_FORMAT_TYPE_A_LAW:
       alsa->format = SND_PCM_FORMAT_A_LAW;
+      alsa->channels = GST_AUDIO_INFO_CHANNELS (&spec->info);
       break;
     case GST_AUDIO_RING_BUFFER_FORMAT_TYPE_MU_LAW:
       alsa->format = SND_PCM_FORMAT_MU_LAW;
+      alsa->channels = GST_AUDIO_INFO_CHANNELS (&spec->info);
       break;
     case GST_AUDIO_RING_BUFFER_FORMAT_TYPE_AC3:
     case GST_AUDIO_RING_BUFFER_FORMAT_TYPE_EAC3:
@@ -875,13 +878,14 @@ alsasink_parse_spec (GstAlsaSink * alsa, GstAudioRingBufferSpec * spec)
     case GST_AUDIO_RING_BUFFER_FORMAT_TYPE_MPEG:
       alsa->format = SND_PCM_FORMAT_S16_BE;
       alsa->iec958 = TRUE;
+      /* Set channels to stereo as iec playload are rendered on a stereo stream */
+      alsa->channels = 2;
       break;
     default:
       goto error;
 
   }
   alsa->rate = GST_AUDIO_INFO_RATE (&spec->info);
-  alsa->channels = GST_AUDIO_INFO_CHANNELS (&spec->info);
   alsa->buffer_time = spec->buffer_time;
   alsa->period_time = spec->latency_time;
   alsa->access = SND_PCM_ACCESS_RW_INTERLEAVED;
-- 
2.34.1


From 02ea7fbe684735b53a1973a3cbd65f0bd8c62b87 Mon Sep 17 00:00:00 2001
From: Arnaud Pouliquen <arnaud.pouliquen@st.com>
Date: Thu, 5 Nov 2015 18:47:19 +0100
Subject: [PATCH 47/93] iec61937: set iec958 boolean before use it

During prepare alsa->iec958  is tested before set.
Effect is that alsa device is not opened in iec958 mode,
so AES value are not transmitted to alsa driver.

https://bugzilla.gnome.org/show_bug.cgi?id=757258

Signed-off-by: Arnaud Pouliquen <arnaud.pouliquen@st.com>
(cherry picked from commit 80c282b3b918fab28dd75dc14126efcce1674a64)
---
 ext/alsa/gstalsasink.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ext/alsa/gstalsasink.c b/ext/alsa/gstalsasink.c
index 12752ebb8..ab1d8d635 100644
--- a/ext/alsa/gstalsasink.c
+++ b/ext/alsa/gstalsasink.c
@@ -947,6 +947,9 @@ gst_alsasink_prepare (GstAudioSink * asink, GstAudioRingBufferSpec * spec)
 
   alsa = GST_ALSA_SINK (asink);
 
+  if (!alsasink_parse_spec (alsa, spec))
+    goto spec_parse;
+
   if (alsa->iec958) {
     snd_pcm_close (alsa->handle);
     alsa->handle = gst_alsa_open_iec958_pcm (GST_OBJECT (alsa), alsa->device);
@@ -955,9 +958,6 @@ gst_alsasink_prepare (GstAudioSink * asink, GstAudioRingBufferSpec * spec)
     }
   }
 
-  if (!alsasink_parse_spec (alsa, spec))
-    goto spec_parse;
-
   CHECK (set_hwparams (alsa), hw_params_failed);
   CHECK (set_swparams (alsa), sw_params_failed);
 
-- 
2.34.1


From 07feb31588b89e9bd187a48d763b0dedcec16bbb Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@nxp.com>
Date: Thu, 21 Jun 2018 15:32:04 +0800
Subject: [PATCH 48/93] alsasink: Open iec958 audio device

- Our audio driver not support open device
with AES parameter, so just use hw: device name
- i.mx specific for audio passthrough

https://bugzilla.gnome.org/show_bug.cgi?id=757258

Signed-off-by: Lyon Wang <lyon.wang@nxp.com>
(cherry picked from commit 91a02413e3e90c2c325f8b2470597ebb8659174f)
---
 ext/alsa/gstalsa.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ext/alsa/gstalsa.c b/ext/alsa/gstalsa.c
index c83516857..4d9b422dd 100644
--- a/ext/alsa/gstalsa.c
+++ b/ext/alsa/gstalsa.c
@@ -666,12 +666,14 @@ gst_alsa_open_iec958_pcm (GstObject * obj, gchar * device)
    * SPDIF_CON: Non-audio flag set:
    *    spdif:{AES0 0x2 AES1 0x82 AES2 0x0 AES3 0x2}
    */
-  sprintf (devstr,
+/*  sprintf (devstr,
       "%s:{AES0 0x%02x AES1 0x%02x AES2 0x%02x AES3 0x%02x}",
       device,
       IEC958_AES0_CON_EMPHASIS_NONE | IEC958_AES0_NONAUDIO,
       IEC958_AES1_CON_ORIGINAL | IEC958_AES1_CON_PCM_CODER,
-      0, IEC958_AES3_CON_FS_48000);
+      0, IEC958_AES3_CON_FS_48000);*/
+
+  sprintf (devstr, "%s", device);
 
   GST_DEBUG_OBJECT (obj, "Generated device string \"%s\"", devstr);
   iec958_pcm_name = devstr;
-- 
2.34.1


From 2db684401a2aaae413aed912babd8b71a30a92e8 Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@nxp.com>
Date: Fri, 6 Jul 2018 13:18:54 +0800
Subject: [PATCH 49/93] alsasink: Add passthrough property

- Add property pass-through to control
  if enable pass-through mode

Signed-off-by: Lyon Wang <lyon.wang@nxp.com>
(cherry picked from commit aba5493d19dfece10d14fa633e6c66166cb25da6)
---
 ext/alsa/gstalsasink.c | 20 +++++++++++++++++---
 ext/alsa/gstalsasink.h |  2 ++
 2 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/ext/alsa/gstalsasink.c b/ext/alsa/gstalsasink.c
index ab1d8d635..ad0a6f9be 100644
--- a/ext/alsa/gstalsasink.c
+++ b/ext/alsa/gstalsasink.c
@@ -71,6 +71,7 @@ enum
   PROP_DEVICE,
   PROP_DEVICE_NAME,
   PROP_CARD_NAME,
+  PROP_PASSTHROUGH,
   PROP_LAST
 };
 
@@ -201,6 +202,11 @@ gst_alsasink_class_init (GstAlsaSinkClass * klass)
           "Human-readable name of the sound card", DEFAULT_CARD_NAME,
           G_PARAM_READABLE | G_PARAM_STATIC_STRINGS |
           GST_PARAM_DOC_SHOW_DEFAULT));
+
+  g_object_class_install_property (gobject_class, PROP_PASSTHROUGH,
+      g_param_spec_boolean ("pass-through", "Pass through",
+          "enable pass through mode",
+          FALSE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 }
 
 static void
@@ -220,6 +226,9 @@ gst_alsasink_set_property (GObject * object, guint prop_id,
         sink->device = g_strdup (DEFAULT_DEVICE);
       }
       break;
+    case PROP_PASSTHROUGH:
+      sink->passthrough = g_value_get_boolean (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -248,6 +257,9 @@ gst_alsasink_get_property (GObject * object, guint prop_id,
           gst_alsa_find_card_name (GST_OBJECT_CAST (sink),
               sink->device, SND_PCM_STREAM_PLAYBACK));
       break;
+    case PROP_PASSTHROUGH:
+      g_value_set_boolean (value, sink->passthrough);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -265,6 +277,7 @@ gst_alsasink_init (GstAlsaSink * alsasink)
   alsasink->is_paused = FALSE;
   alsasink->after_paused = FALSE;
   alsasink->hw_support_pause = FALSE;
+  alsasink->passthrough = FALSE;
   g_mutex_init (&alsasink->alsa_lock);
   g_mutex_init (&alsasink->delay_lock);
 
@@ -333,10 +346,11 @@ gst_alsasink_getcaps (GstBaseSink * bsink, GstCaps * filter)
    * only for now but we could add SPDIF capture later)
    */
 
-  if (gst_alsa_iec958_formats_supported(GST_OBJECT (sink), sink->device,
-    &sink->handle) == TRUE) {
+  if (gst_alsa_iec958_formats_supported (GST_OBJECT (sink), sink->device,
+          &sink->handle) == TRUE) {
     GST_LOG_OBJECT (sink, "Add pass-through capabilities");
-    gst_caps_append (caps, gst_caps_from_string (PASSTHROUGH_CAPS));
+    if (TRUE == sink->passthrough)
+      gst_caps_append (caps, gst_caps_from_string (PASSTHROUGH_CAPS));
   }
 
   if (caps) {
diff --git a/ext/alsa/gstalsasink.h b/ext/alsa/gstalsasink.h
index 35efa888e..ab642ee90 100644
--- a/ext/alsa/gstalsasink.h
+++ b/ext/alsa/gstalsasink.h
@@ -81,6 +81,8 @@ struct _GstAlsaSink {
 
   GMutex alsa_lock;
   GMutex delay_lock;
+
+  gboolean passthrough;
 };
 
 struct _GstAlsaSinkClass {
-- 
2.34.1


From e0cf8b816c2eefda2e52328c068df34082dcfb47 Mon Sep 17 00:00:00 2001
From: Lyon Wang <lyon.wang@nxp.com>
Date: Mon, 3 Sep 2018 10:53:34 +0800
Subject: [PATCH 50/93] alsasink: enable eac3 pass-through mode

- For eac3 need using 192K sample rate
- and the sample count for alsa drvier need counted by 2byte per channel/sample

Signed-off-by: Lyon Wang <lyon.wang@nxp.com>
(cherry picked from commit 535a7bbcfc68fc5377b36ea67b6bef81aecbe1e0)
---
 ext/alsa/gstalsasink.c | 11 +++++++++--
 ext/alsa/gstalsasink.h |  1 +
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/ext/alsa/gstalsasink.c b/ext/alsa/gstalsasink.c
index ad0a6f9be..54cb51ffb 100644
--- a/ext/alsa/gstalsasink.c
+++ b/ext/alsa/gstalsasink.c
@@ -489,6 +489,8 @@ set_hwparams (GstAlsaSink * alsa)
       no_channels);
   /* set the stream rate */
   rrate = alsa->rate;
+  if ((TRUE== alsa->passthrough) && alsa->iec958 && (alsa->type == GST_AUDIO_RING_BUFFER_FORMAT_TYPE_EAC3))
+    rrate = 192000;
   CHECK (snd_pcm_hw_params_set_rate_near (alsa->handle, params, &rrate, NULL),
       no_rate);
 #ifndef GST_DISABLE_GST_DEBUG
@@ -903,6 +905,7 @@ alsasink_parse_spec (GstAlsaSink * alsa, GstAudioRingBufferSpec * spec)
   alsa->buffer_time = spec->buffer_time;
   alsa->period_time = spec->latency_time;
   alsa->access = SND_PCM_ACCESS_RW_INTERLEAVED;
+  alsa->type = spec->type;
 
   if ((spec->type == GST_AUDIO_RING_BUFFER_FORMAT_TYPE_RAW ||
           spec->type == GST_AUDIO_RING_BUFFER_FORMAT_TYPE_DSD) &&
@@ -1115,8 +1118,12 @@ gst_alsasink_write (GstAudioSink * asink, gpointer data, guint length)
 
   GST_LOG_OBJECT (asink, "received audio samples buffer of %u bytes", length);
 
-  cptr = length / alsa->bpf;
-
+  if (alsa->iec958 && (TRUE == alsa->passthrough) ) {
+    cptr = length / 4;  // treated as 2ch with S16 in pass-through mode 
+  }
+  else {
+    cptr = length / alsa->bpf;
+  }
   GST_ALSA_SINK_LOCK (asink);
   while (cptr > 0) {
     /* start by doing a blocking wait for free space. Set the timeout
diff --git a/ext/alsa/gstalsasink.h b/ext/alsa/gstalsasink.h
index ab642ee90..25567f683 100644
--- a/ext/alsa/gstalsasink.h
+++ b/ext/alsa/gstalsasink.h
@@ -83,6 +83,7 @@ struct _GstAlsaSink {
   GMutex delay_lock;
 
   gboolean passthrough;
+  GstAudioRingBufferFormatType  type;
 };
 
 struct _GstAlsaSinkClass {
-- 
2.34.1


From fc6480e4fc080fdd7492f30adb34ddc5eebe0046 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Fri, 10 May 2019 16:34:41 +0800
Subject: [PATCH 51/93] MMFMWK-8478 alsasink: fix eac3 passthrough playback

eac3 caps alignment set to iec61937, correct iec61937 check when payload

upstream status: pending
https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/234

(cherry picked from commit fd69bc66fc2b87d1b38aaa1dbfdeff09a161919b)
---
 ext/alsa/gstalsa.h                    | 2 +-
 gst-libs/gst/audio/gstaudioiec61937.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/ext/alsa/gstalsa.h b/ext/alsa/gstalsa.h
index d9e62845c..82a01f699 100644
--- a/ext/alsa/gstalsa.h
+++ b/ext/alsa/gstalsa.h
@@ -41,7 +41,7 @@
 
 #define PASSTHROUGH_CAPS \
     "audio/x-ac3, framed = (boolean) true;" \
-    "audio/x-eac3, framed = (boolean) true; " \
+    "audio/x-eac3, framed = (boolean) true, alignment = (string) iec61937; "\
     "audio/x-dts, framed = (boolean) true, " \
       "block-size = (int) { 512, 1024, 2048 }; " \
     "audio/mpeg, mpegversion = (int) 1, " \
diff --git a/gst-libs/gst/audio/gstaudioiec61937.c b/gst-libs/gst/audio/gstaudioiec61937.c
index a19291331..c2ee4d6d1 100644
--- a/gst-libs/gst/audio/gstaudioiec61937.c
+++ b/gst-libs/gst/audio/gstaudioiec61937.c
@@ -209,7 +209,7 @@ gst_audio_iec61937_payload (const guint8 * src, guint src_n, guint8 * dst,
 
     case GST_AUDIO_RING_BUFFER_FORMAT_TYPE_EAC3:
     {
-      if (g_str_equal (caps_get_string_field (spec->caps, "alignment"),
+      if (!g_str_equal (caps_get_string_field (spec->caps, "alignment"),
               "iec61937"))
         return FALSE;
 
-- 
2.34.1


From 2ec86c4992e54572738464595f8cfed635162878 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Tue, 2 Jul 2019 12:52:00 +0800
Subject: [PATCH 52/93] MMFMWK-8323 playsink: fix audio passthrough hang when
 set play rate

when audio passthrough is enable, audio_filter will not
put into audiochain. When do reconfigure, playsink will
remove previous audio chain as audio_filter is not match
with chain->filter (should be NULL). Then playsink will
remove audio chain and recreate. When remove, streamsynchroinizer
will release its sinkpad, but sinkpad is in waiting all pad eos
as audio is eos in fast forward 4X mode and hold stream lock,
release pad will also try to lock this stream lock and cause hang.
Need check whether chain->filter is valided first to avoid recreate
audio chain

upstream status: pending
https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/merge_requests/317

(cherry picked from commit cfa26525c865ca66d004cb84f374baec35b37d96)
---
 gst/playback/gstplaysink.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gst/playback/gstplaysink.c b/gst/playback/gstplaysink.c
index f2dc6168b..7c38e815f 100644
--- a/gst/playback/gstplaysink.c
+++ b/gst/playback/gstplaysink.c
@@ -3549,7 +3549,7 @@ gst_play_sink_do_reconfigure (GstPlaySink * playsink)
       /* try to reactivate the chain */
       if ((playsink->audio_sink
               && playsink->audio_sink != playsink->audiochain->sink)
-          || (playsink->audio_filter
+          || (playsink->audio_filter && playsink->audiochain->filter
               && playsink->audio_filter != playsink->audiochain->filter)
           || !setup_audio_chain (playsink, raw)) {
         GST_DEBUG_OBJECT (playsink, "removing current audio chain");
-- 
2.34.1


From 2f9ed2d205bdc68f8f490b0c60cae5b461917c30 Mon Sep 17 00:00:00 2001
From: Elliot Chen <elliot.chen@nxp.com>
Date: Mon, 29 Nov 2021 17:15:07 +0800
Subject: [PATCH 53/93] MMFMWK-9010 alsasrc: add new feature of reading iec958
 stream

Signed-off-by: Elliot Chen <elliot.chen@nxp.com>
(cherry picked from commit fc97d964d8db2f00dc60c34c82ce007e6fc79d9f)
---
 ext/alsa/gstalsa.c                      | 32 +++++++++++++++++++++----
 ext/alsa/gstalsasrc.c                   |  9 ++++++-
 gst-libs/gst/audio/audio-format.h       |  1 +
 gst-libs/gst/audio/gstaudioringbuffer.c |  3 ++-
 4 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/ext/alsa/gstalsa.c b/ext/alsa/gstalsa.c
index 4d9b422dd..e005c4b93 100644
--- a/ext/alsa/gstalsa.c
+++ b/ext/alsa/gstalsa.c
@@ -135,6 +135,8 @@ gst_alsa_get_pcm_format (GstAudioFormat fmt)
       return SND_PCM_FORMAT_FLOAT64_LE;
     case GST_AUDIO_FORMAT_F64BE:
       return SND_PCM_FORMAT_FLOAT64_BE;
+    case GST_AUDIO_FORMAT_IEC958_SUBFRAME_LE:
+      return SND_PCM_FORMAT_IEC958_SUBFRAME_LE;
     default:
       break;
   }
@@ -187,10 +189,16 @@ gst_alsa_detect_formats (GstObject * obj, snd_pcm_hw_params_t * hw_params,
     const GValue *format;
     GValue list = G_VALUE_INIT;
 
+    gboolean is_iec958_format = FALSE;
     s = gst_caps_get_structure (in_caps, i);
     if (!gst_structure_has_name (s, "audio/x-raw")) {
-      GST_DEBUG_OBJECT (obj, "skipping non-raw format");
-      continue;
+      if (gst_structure_has_name (s, "audio/x-iec958")) {
+        is_iec958_format = TRUE;
+        GST_DEBUG_OBJECT (obj, "audio/x-iec958");
+      } else {
+        GST_DEBUG_OBJECT (obj, "skipping non-raw format or iec958 format");
+        continue;
+      }
     }
 
     format = gst_structure_get_value (s, "format");
@@ -207,12 +215,28 @@ gst_alsa_detect_formats (GstObject * obj, snd_pcm_hw_params_t * hw_params,
         const GValue *val;
 
         val = gst_value_list_get_value (format, i);
-        if (format_supported (val, mask, endianness))
+        if (format_supported (val, mask, endianness)) {
           gst_value_list_append_value (&list, val);
+        } else {
+          if (is_iec958_format) {
+            if (snd_pcm_format_mask_test (mask,
+                    SND_PCM_FORMAT_IEC958_SUBFRAME_LE)) {
+              gst_value_list_append_value (&list, val);
+            }
+          }
+        }
       }
     } else if (G_VALUE_HOLDS_STRING (format)) {
-      if (format_supported (format, mask, endianness))
+      if (format_supported (format, mask, endianness)) {
         gst_value_list_append_value (&list, format);
+      } else {
+        if (is_iec958_format) {
+          if (snd_pcm_format_mask_test (mask,
+                  SND_PCM_FORMAT_IEC958_SUBFRAME_LE)) {
+            gst_value_list_append_value (&list, format);
+          }
+        }
+      }
     }
 
     if (gst_value_list_get_size (&list) > 1) {
diff --git a/ext/alsa/gstalsasrc.c b/ext/alsa/gstalsasrc.c
index 2e97f3927..c5ddcfc12 100644
--- a/ext/alsa/gstalsasrc.c
+++ b/ext/alsa/gstalsasrc.c
@@ -112,7 +112,11 @@ GST_STATIC_PAD_TEMPLATE ("src",
     GST_STATIC_CAPS ("audio/x-raw, "
         "format = (string) " GST_AUDIO_FORMATS_ALL ", "
         "layout = (string) interleaved, "
-        "rate = (int) [ 1, MAX ], " "channels = (int) [ 1, MAX ]")
+        "rate = (int) [ 1, MAX ], " "channels = (int) [ 1, MAX ]; "
+        "audio/x-iec958, "
+        "format = U32LE, "
+        "layout = (string) interleaved, "
+        "rate = (int) [ 1, MAX ], " "channels = (int) 2")
     );
 
 static void
@@ -742,6 +746,9 @@ alsasrc_parse_spec (GstAlsaSrc * alsa, GstAudioRingBufferSpec * spec)
     case GST_AUDIO_RING_BUFFER_FORMAT_TYPE_MU_LAW:
       alsa->format = SND_PCM_FORMAT_MU_LAW;
       break;
+    case GST_AUDIO_RING_BUFFER_FORMAT_TYPE_IEC958:
+      alsa->format = SND_PCM_FORMAT_IEC958_SUBFRAME_LE;
+      break;
     default:
       goto error;
 
diff --git a/gst-libs/gst/audio/audio-format.h b/gst-libs/gst/audio/audio-format.h
index c7aeef8de..e847fdb0a 100644
--- a/gst-libs/gst/audio/audio-format.h
+++ b/gst-libs/gst/audio/audio-format.h
@@ -128,6 +128,7 @@ typedef enum {
   GST_AUDIO_FORMAT_F32BE,
   GST_AUDIO_FORMAT_F64LE,
   GST_AUDIO_FORMAT_F64BE,
+  GST_AUDIO_FORMAT_IEC958_SUBFRAME_LE,
   /* native endianness equivalents */
   GST_AUDIO_FORMAT_S16 = _GST_AUDIO_FORMAT_NE(S16),
   GST_AUDIO_FORMAT_U16 = _GST_AUDIO_FORMAT_NE(U16),
diff --git a/gst-libs/gst/audio/gstaudioringbuffer.c b/gst-libs/gst/audio/gstaudioringbuffer.c
index 1aee68fec..af8f58e88 100644
--- a/gst-libs/gst/audio/gstaudioringbuffer.c
+++ b/gst-libs/gst/audio/gstaudioringbuffer.c
@@ -249,7 +249,8 @@ gst_audio_ring_buffer_parse_caps (GstAudioRingBufferSpec * spec, GstCaps * caps)
       goto parse_error;
 
     spec->type = GST_AUDIO_RING_BUFFER_FORMAT_TYPE_IEC958;
-    info.bpf = 4;
+    gst_structure_get_int (structure, "channels", &info.channels);
+    info.bpf = 8;
   } else if (g_str_equal (mimetype, "audio/x-ac3")) {
     /* extract the needed information from the cap */
     if (!(gst_structure_get_int (structure, "rate", &info.rate)))
-- 
2.34.1


From 42252fa859d711cb1548e0cc8d8ba14c449c4e3e Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Thu, 29 Oct 2020 10:42:38 +0800
Subject: [PATCH 54/93] disable orc build for gst-base-video

Signed-off-by: Haihua Hu <jared.hu@nxp.com>

Conflicts:
	gst-libs/gst/video/meson.build
(cherry picked from commit 6cfc332e6433a69ef04388da105b2fbc27b9b2c7)
---
 gst-libs/gst/video/meson.build | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gst-libs/gst/video/meson.build b/gst-libs/gst/video/meson.build
index 1b492fd9e..da9ff3a0e 100644
--- a/gst-libs/gst/video/meson.build
+++ b/gst-libs/gst/video/meson.build
@@ -138,7 +138,7 @@ endif
 
 gstvideo = library('gstvideo-@0@'.format(api_version),
   video_sources, gstvideo_h, gstvideo_c, orc_c, orc_h,
-  c_args : gst_plugins_base_args + ['-DBUILDING_GST_VIDEO', '-DG_LOG_DOMAIN="GStreamer-Video"'],
+  c_args : gst_plugins_base_args + ['-DBUILDING_GST_VIDEO', '-DDISABLE_ORC', '-DG_LOG_DOMAIN="GStreamer-Video"'],
   include_directories: [configinc, libsinc],
   version : libversion,
   soversion : soversion,
-- 
2.34.1


From d0724c4b605f1acf105f67c9d769cf45acb5de45 Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Fri, 11 Feb 2022 13:47:15 +0800
Subject: [PATCH 55/93] Revert "meson: Mark newly fdkaac/ogg/vorbis as allow
 fallback"

This reverts commit 38c7a7749484718205c71a236c18aca51a9ac924.

Signed-off-by: Hou Qi <qi.hou@nxp.com>
---
 ext/ogg/meson.build    | 2 +-
 ext/vorbis/meson.build | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/ext/ogg/meson.build b/ext/ogg/meson.build
index 70fff4366..825f79f5e 100644
--- a/ext/ogg/meson.build
+++ b/ext/ogg/meson.build
@@ -10,7 +10,7 @@ ogg_sources = [
   'vorbis_parse.c',
 ]
 
-ogg_dep = dependency('ogg', allow_fallback: true, version : '>=1.0', required : get_option('ogg'))
+ogg_dep = dependency('ogg', version : '>=1.0', required : get_option('ogg'))
 core_conf.set('HAVE_OGG', ogg_dep.found())
 
 if ogg_dep.found()
diff --git a/ext/vorbis/meson.build b/ext/vorbis/meson.build
index 1aff8a9bb..7d70f84df 100644
--- a/ext/vorbis/meson.build
+++ b/ext/vorbis/meson.build
@@ -17,9 +17,9 @@ vorbisidec_sources = [
   'gstvorbiscommon.c',
 ]
 
-vorbis_dep = dependency('vorbis', version : '>= 1.3.1', allow_fallback: true, required : get_option('vorbis'))
-vorbisenc_dep = dependency('vorbisenc', version : '>= 1.3.1', allow_fallback: true, required : get_option('vorbis'))
-vorbisidec_dep = dependency('vorbisidec', allow_fallback: true, required : get_option('tremor'))
+vorbis_dep = dependency('vorbis', version : '>= 1.3.1', required : get_option('vorbis'))
+vorbisenc_dep = dependency('vorbisenc', version : '>= 1.3.1', required : get_option('vorbis'))
+vorbisidec_dep = dependency('vorbisidec', required : get_option('tremor'))
 
 if vorbis_dep.found()
   vorbis_deps = [vorbis_dep]
-- 
2.34.1


From 9b46e81eca26505168e5ebc4baf0b32c61af3450 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Thu, 30 Mar 2017 13:43:05 +0800
Subject: [PATCH 56/93] MMFMWK-8218 subparse: fix critical log print out when
 set rate <0 with external subtitle

When playback with external subtitle, subparse will force its own start
and stop position when seek. When rate < 0, critial log will print out
assertion 'stop != -1', so need avoid it when rate < 0.

upstream status: Pending
https://bugzilla.gnome.org/show_bug.cgi?id=771648

(cherry picked from commit d02243efde9a71bcfc4b07b5307fe979302368ee)
---
 gst/subparse/gstsubparse.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/gst/subparse/gstsubparse.c b/gst/subparse/gstsubparse.c
index 995a416bd..675d3a976 100644
--- a/gst/subparse/gstsubparse.c
+++ b/gst/subparse/gstsubparse.c
@@ -282,6 +282,11 @@ gst_sub_parse_src_event (GstPad * pad, GstObject * parent, GstEvent * event)
 
       /* Convert that seek to a seeking in bytes at position 0,
          FIXME: could use an index */
+      if (rate < 0) {
+        rate = 1.0;
+        GST_WARNING_OBJECT (self, "Only can push positive rate upstream");
+      }
+
       ret = gst_pad_push_event (self->sinkpad,
           gst_event_new_seek (rate, GST_FORMAT_BYTES, flags,
               GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_NONE, 0));
-- 
2.34.1


From 8f95e459717873d8b824825115b5813d7bca9bc5 Mon Sep 17 00:00:00 2001
From: Song Bing <bing.song@nxp.com>
Date: Mon, 1 Apr 2019 11:50:04 -0700
Subject: [PATCH 57/93] MMFMWK-8438 [iMX8QXP_MEK]Grecorder: Error log printed
 if set resolution to 640x480 or 752x480 then start recording, 100%

v4l2src will use default colorimetry. But the default will change
based on resolution. All other moduler don't change colorimetry
based on resolution. Change SD colorimetry to HD colorimetry.

upstream status: imx specific
Signed-off-by: Song Bing <bing.song@nxp.com>
(cherry picked from commit b51ab6f8c3d04deda97e3709daaaa9b3fab526e4)
---
 gst-libs/gst/video/video-info.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gst-libs/gst/video/video-info.c b/gst-libs/gst/video/video-info.c
index 35fa2a2f2..746d43f53 100644
--- a/gst-libs/gst/video/video-info.c
+++ b/gst-libs/gst/video/video-info.c
@@ -170,7 +170,7 @@ set_default_colorimetry (GstVideoInfo * info)
       info->colorimetry = default_color[DEFAULT_YUV_HD];
     } else {
       info->chroma_site = GST_VIDEO_CHROMA_SITE_NONE;
-      info->colorimetry = default_color[DEFAULT_YUV_SD];
+      info->colorimetry = default_color[DEFAULT_YUV_HD];
     }
   } else if (GST_VIDEO_FORMAT_INFO_IS_GRAY (finfo)) {
     info->colorimetry = default_color[DEFAULT_GRAY];
-- 
2.34.1


From 895aed3b5ee54c3b9085525dac72b43a06a784a8 Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Fri, 5 Apr 2019 17:35:09 +0800
Subject: [PATCH 58/93] MMFMWK-8329 [8mq]pulsesink: Video can't seek twice
 within short time

The clip's interleave depth is rather big which made audio queue of multiqueue is empty while
video queue is full. So enlarge the max-size-bytes from 2M to 10M to avoid preroll fail.

Signed-off-by: Hou Qi <qi.hou@nxp.com>

Conflicts:
	gst/playback/gstdecodebin2.c
(cherry picked from commit 64ac77aa3d4a2b58764bcc10596c7bdb69d470e8)
---
 gst/playback/gstdecodebin2.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/gst/playback/gstdecodebin2.c b/gst/playback/gstdecodebin2.c
index 9247625a3..8f3bf0055 100644
--- a/gst/playback/gstdecodebin2.c
+++ b/gst/playback/gstdecodebin2.c
@@ -248,7 +248,10 @@ enum
 
 /* when playing, keep a max of 8MB of data but try to keep the number of buffers
  * as low as possible (try to aim for 5 buffers) */
-#define AUTO_PLAY_SIZE_BYTES        8 * 1024 * 1024
+
+/* enlarge the max-size-bytes to 10M to avoid file-based mode and
+ * long-depth interleaved stream preroll fail. */
+#define AUTO_PLAY_SIZE_BYTES        10 * 1024 * 1024
 #define AUTO_PLAY_SIZE_BUFFERS      5
 #define AUTO_PLAY_SIZE_TIME         0
 
-- 
2.34.1


From 73fff64ce3e0986e5c5c3dcd4b07e9d1458e0b21 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Thu, 17 Dec 2020 12:11:45 +0800
Subject: [PATCH 59/93] gst/playback/: support runtime change connection-speed
 of adaptivedemux

update connection-speed at runtime in playbin, uridecodebin and decodebin
also do the same thing in playbin3/uridecodebin3/urisourcebin

upstream status: pending
https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/merge_requests/981

Conflict:
	gst/playback/gstplaybin3.c
	gst/playback/gsturidecodebin3.c
(cherry picked from commit 7d1879c796ca007d2d2757f3b0060e0dc95bc302)
---
 gst/playback/gstplaybin2.c      |  2 +-
 gst/playback/gsturidecodebin.c  |  2 --
 gst/playback/gsturidecodebin3.c | 26 +++++++++++++++++++++++++-
 gst/playback/gsturisourcebin.c  | 28 ++++++++++++----------------
 4 files changed, 38 insertions(+), 20 deletions(-)

diff --git a/gst/playback/gstplaybin2.c b/gst/playback/gstplaybin2.c
index 7cef7d51a..575892ca9 100644
--- a/gst/playback/gstplaybin2.c
+++ b/gst/playback/gstplaybin2.c
@@ -2473,6 +2473,7 @@ gst_play_bin_set_property (GObject * object, guint prop_id,
       GST_PLAY_BIN_LOCK (playbin);
       playbin->connection_speed = g_value_get_uint64 (value) * 1000;
       connection_speed = playbin->connection_speed;
+      GST_PLAY_BIN_UNLOCK (playbin);
       if (playbin->curr_group) {
         GST_SOURCE_GROUP_LOCK (playbin->curr_group);
         if (playbin->curr_group->uridecodebin) {
@@ -2489,7 +2490,6 @@ gst_play_bin_set_property (GObject * object, guint prop_id,
         }
         GST_SOURCE_GROUP_UNLOCK (playbin->next_group);
       }
-      GST_PLAY_BIN_UNLOCK (playbin);
       break;
     }
     case PROP_BUFFER_SIZE:
diff --git a/gst/playback/gsturidecodebin.c b/gst/playback/gsturidecodebin.c
index 86e20a9be..2b358bd1d 100644
--- a/gst/playback/gsturidecodebin.c
+++ b/gst/playback/gsturidecodebin.c
@@ -828,13 +828,11 @@ gst_uri_decode_bin_set_connection_speed (GstURIDecodeBin * dec)
   GST_OBJECT_UNLOCK (dec);
 
   /* set the property on all decodebins now */
-  GST_URI_DECODE_BIN_LOCK (dec);
   for (walk = dec->decodebins; walk; walk = g_slist_next (walk)) {
     GObject *decodebin = G_OBJECT (walk->data);
 
     g_object_set (decodebin, "connection-speed", connection_speed / 1000, NULL);
   }
-  GST_URI_DECODE_BIN_UNLOCK (dec);
 }
 
 static void
diff --git a/gst/playback/gsturidecodebin3.c b/gst/playback/gsturidecodebin3.c
index 85474bbcd..09e1b9b5e 100644
--- a/gst/playback/gsturidecodebin3.c
+++ b/gst/playback/gsturidecodebin3.c
@@ -260,6 +260,8 @@ struct _GstURIDecodeBin3
   gint shutdown;
 
   GList *output_pads;           /* List of OutputPad */
+
+  GList *source_handlers;       /* List of SourceHandler */
 };
 
 static GstStateChangeReturn activate_play_item (GstPlayItem * item);
@@ -1560,6 +1562,9 @@ new_source_handler (GstURIDecodeBin3 * uridecodebin, GstPlayItem * item,
       g_signal_connect (handler->urisourcebin, "about-to-finish",
       (GCallback) src_about_to_finish_cb, handler);
 
+  uridecodebin->source_handlers =
+      g_list_append (uridecodebin->source_handlers, handler);
+
   handler->expected_pads = 1;
 
   return handler;
@@ -1592,6 +1597,20 @@ source_handler_set_eos (GstSourceHandler * handler)
   }
 }
 
+static void
+gst_uri_decode_bin3_set_connection_speed (GstURIDecodeBin3 * dec)
+{
+  GList *walk;
+
+  /* set the property on all decodebins now */
+  for (walk = dec->source_handlers; walk; walk = g_list_next (walk)) {
+    GstSourceHandler *handler = (GstSourceHandler *) (walk->data);
+
+    g_object_set (handler->urisourcebin, "connection-speed",
+        dec->connection_speed / 1000, NULL);
+  }
+}
+
 static void
 gst_uri_decode_bin3_set_property (GObject * object, guint prop_id,
     const GValue * value, GParamSpec * pspec)
@@ -1608,6 +1627,7 @@ gst_uri_decode_bin3_set_property (GObject * object, guint prop_id,
     case PROP_CONNECTION_SPEED:
       GST_OBJECT_LOCK (dec);
       dec->connection_speed = g_value_get_uint64 (value) * 1000;
+      gst_uri_decode_bin3_set_connection_speed (dec);
       GST_OBJECT_UNLOCK (dec);
       break;
     case PROP_BUFFER_SIZE:
@@ -1767,7 +1787,11 @@ free_source_handler (GstURIDecodeBin3 * uridecodebin,
   }
   if (handler->pending_buffering_msg)
     gst_message_unref (handler->pending_buffering_msg);
-  g_free (handler);
+
+  uridecodebin->source_handlers =
+      g_list_remove (uridecodebin->source_handlers, handler);
+  g_slice_free (GstSourceHandler, handler);
+
 }
 
 static GstSourceItem *
diff --git a/gst/playback/gsturisourcebin.c b/gst/playback/gsturisourcebin.c
index 3c9a71c9c..8951defb6 100644
--- a/gst/playback/gsturisourcebin.c
+++ b/gst/playback/gsturisourcebin.c
@@ -573,29 +573,25 @@ static void
 gst_uri_source_bin_update_connection_speed (GstURISourceBin * urisrc)
 {
   guint64 speed = 0;
-  GList *iter;
 
-  if (!urisrc->is_adaptive) {
-    return;
-  }
+  GParamSpec *pspec = NULL;
+  GList *tmp;
 
   GST_OBJECT_LOCK (urisrc);
   speed = urisrc->connection_speed / 1000;
   GST_OBJECT_UNLOCK (urisrc);
 
-  GST_URI_SOURCE_BIN_LOCK (urisrc);
-  for (iter = urisrc->src_infos; iter; iter = iter->next) {
-    ChildSrcPadInfo *info = iter->data;
-    GParamSpec *pspec = NULL;
-    if (!info->demuxer)
-      continue;
-
-    pspec = g_object_class_find_property (G_OBJECT_GET_CLASS (info->demuxer),
-        "connection-speed");
-    if (pspec != NULL)
-      g_object_set (info->demuxer, "connection-speed", speed, NULL);
+  if (urisrc->is_adaptive) {
+    for (tmp = urisrc->src_infos; tmp; tmp = tmp->next) {
+      ChildSrcPadInfo *info = tmp->data;
+      if (info->demuxer) {
+        pspec = g_object_class_find_property (G_OBJECT_GET_CLASS (info->demuxer),
+            "connection-speed");
+        if (pspec != NULL)
+          g_object_set (info->demuxer, "connection-speed", speed, NULL);
+      }
+    }
   }
-  GST_URI_SOURCE_BIN_UNLOCK (urisrc);
 }
 
 static void
-- 
2.34.1


From 4de3e7aeb2b08430bc3edd0989a49ada5822f3a5 Mon Sep 17 00:00:00 2001
From: Ming Qian <ming.qian@nxp.com>
Date: Thu, 22 Dec 2022 17:24:59 +0800
Subject: [PATCH 60/93] MMFMWK-9186-1 video: Add support for Y012_LE

Y012_LE is 12-bit grayscale, least significant byte first.
expanded to 16 bits, little endian,
data in the 12 high bits, zeros in the 4 low bits
---
 gst-libs/gst/video/video-converter.c |  1 +
 gst-libs/gst/video/video-format.c    |  3 +++
 gst-libs/gst/video/video-format.h    | 16 ++++++++++++++--
 gst-libs/gst/video/video-info.c      |  1 +
 4 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/gst-libs/gst/video/video-converter.c b/gst-libs/gst/video/video-converter.c
index 435c7a2b5..8f56c1367 100644
--- a/gst-libs/gst/video/video-converter.c
+++ b/gst-libs/gst/video/video-converter.c
@@ -7365,6 +7365,7 @@ get_scale_format (GstVideoFormat format, gint plane)
     case GST_VIDEO_FORMAT_P016_LE:
     case GST_VIDEO_FORMAT_P012_BE:
     case GST_VIDEO_FORMAT_P012_LE:
+    case GST_VIDEO_FORMAT_Y012_LE:
     case GST_VIDEO_FORMAT_Y212_BE:
     case GST_VIDEO_FORMAT_Y212_LE:
     case GST_VIDEO_FORMAT_Y412_BE:
diff --git a/gst-libs/gst/video/video-format.c b/gst-libs/gst/video/video-format.c
index a8b1a47e8..e51dd6acf 100644
--- a/gst-libs/gst/video/video-format.c
+++ b/gst-libs/gst/video/video-format.c
@@ -7353,6 +7353,7 @@ typedef struct
 #define DPTH10_10_10_10  10, 4, { 0, 0, 0, 0 }, { 10, 10, 10, 10 }
 #define DPTH10_10_10_HI  16, 3, { 6, 6, 6, 0 }, { 10, 10, 10, 0 }
 #define DPTH10_10_10_2   10, 4, { 0, 0, 0, 0 }, { 10, 10, 10, 2}
+#define DPTH12           12, 1, { 0, 0, 0, 0 }, { 12, 0, 0, 0 }
 #define DPTH12_12_12     12, 3, { 0, 0, 0, 0 }, { 12, 12, 12, 0 }
 #define DPTH12_12_12_HI  16, 3, { 4, 4, 4, 0 }, { 12, 12, 12, 0 }
 #define DPTH12_12_12_12  12, 4, { 0, 0, 0, 0 }, { 12, 12, 12, 12 }
@@ -7790,6 +7791,8 @@ static const VideoFormat formats[] = {
       OFFS0, SUB444, PACK_GBR_16BE),
   MAKE_RGBA_FORMAT (RBGA, "raw video", DPTH8888, PSTR4444, PLANE0, OFFS0213,
       SUB4444, PACK_RBGA),
+  MAKE_GRAY_LE_FORMAT (Y012_LE, "raw video", DPTH12, PSTR2, PLANE0, OFFS0,
+      SUB4, PACK_GRAY16_LE),
 };
 
 G_GNUC_END_IGNORE_DEPRECATIONS;
diff --git a/gst-libs/gst/video/video-format.h b/gst-libs/gst/video/video-format.h
index b85348532..361f62817 100644
--- a/gst-libs/gst/video/video-format.h
+++ b/gst-libs/gst/video/video-format.h
@@ -169,6 +169,7 @@ G_BEGIN_DECLS
  * @GST_VIDEO_FORMAT_GBR_16LE: planar 4:4:4 RGB, 16 bits per channel (Since: 1.24)
  * @GST_VIDEO_FORMAT_GBR_16BE: planar 4:4:4 RGB, 16 bits per channel (Since: 1.24)
  * @GST_VIDEO_FORMAT_RBGA: packed RGB with alpha, 8 bits per channel (Since: 1.24)
+ * @GST_VIDEO_FORMAT_Y012_LE: 12-bit grayscale, least significant byte first, expanded to 16bits, zeros in the 4 low bits,  (Since: 1.24)
  *
  * Enum value describing the most common video formats.
  *
@@ -619,6 +620,17 @@ typedef enum {
    * Since: 1.24
    */
   GST_VIDEO_FORMAT_RBGA,
+
+  /**
+   * GST_VIDEO_FORMAT_Y012_LE:
+   *
+   * 12-bit grayscale, least significant byte first
+   * expanded to 16 bits, little endian, data in the 12 high bits, zeros in the 4 low bits
+   *
+   * Since: 1.24
+   */
+  GST_VIDEO_FORMAT_Y012_LE,
+
 } GstVideoFormat;
 
 #define GST_VIDEO_MAX_PLANES 4
@@ -1096,7 +1108,7 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "RGB10A2_LE, A444, GBRA, AYUV, VUYA, RGBA, RBGA, ARGB, BGRA, ABGR, A422, " \
     "A420, AV12, Y444_16BE, GBR_16BE, Y444_16LE, GBR_16LE, v216, P016_BE, " \
     "P016_LE, Y444_12BE, GBR_12BE, Y444_12LE, GBR_12LE, I422_12BE, I422_12LE, " \
-    "Y212_BE, Y212_LE, I420_12BE, I420_12LE, P012_BE, P012_LE, Y444_10BE, " \
+    "Y212_BE, Y012_LE, Y212_LE, I420_12BE, I420_12LE, P012_BE, P012_LE, Y444_10BE, " \
     "GBR_10BE, Y444_10LE, GBR_10LE, r210, I422_10BE, I422_10LE, NV16_10LE32, " \
     "Y210, UYVP, v210, I420_10BE, I420_10LE, P010_10BE, MT2110R, MT2110T, " \
     "NV12_10BE_8L128, NV12_10LE40_4L4, P010_10LE, NV12_10LE40, NV12_10LE32, " \
@@ -1114,7 +1126,7 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "A422_10LE, A422_10BE, A420_10LE, A420_10BE, BGR10A2_LE, RGB10A2_LE, Y410, " \
     "A444, GBRA, AYUV, VUYA, RGBA, RBGA, ARGB, BGRA, ABGR, A422, A420, AV12, " \
     "Y444_16LE, GBR_16LE, Y444_16BE, GBR_16BE, v216, P016_LE, P016_BE, " \
-    "Y444_12LE, GBR_12LE, Y444_12BE, GBR_12BE, I422_12LE, I422_12BE, Y212_LE, " \
+    "Y444_12LE, GBR_12LE, Y444_12BE, GBR_12BE, I422_12LE, I422_12BE, Y012_LE, Y212_LE, " \
     "Y212_BE, I420_12LE, I420_12BE, P012_LE, P012_BE, Y444_10LE, GBR_10LE, " \
     "Y444_10BE, GBR_10BE, r210, I422_10LE, I422_10BE, NV16_10LE32, Y210, UYVP, " \
     "v210, I420_10LE, I420_10BE, P010_10LE, NV12_10LE40, NV12_10LE32, " \
diff --git a/gst-libs/gst/video/video-info.c b/gst-libs/gst/video/video-info.c
index 746d43f53..099e55c1f 100644
--- a/gst-libs/gst/video/video-info.c
+++ b/gst-libs/gst/video/video-info.c
@@ -870,6 +870,7 @@ fill_planes (GstVideoInfo * info, gsize plane_size[GST_VIDEO_MAX_PLANES])
       break;
     case GST_VIDEO_FORMAT_GRAY16_BE:
     case GST_VIDEO_FORMAT_GRAY16_LE:
+    case GST_VIDEO_FORMAT_Y012_LE:
       info->stride[0] = GST_ROUND_UP_4 (width * 2);
       info->offset[0] = 0;
       info->size = info->stride[0] * height;
-- 
2.34.1


From e6734c5159ee0166e0fbdb4e14727861bf34cc6d Mon Sep 17 00:00:00 2001
From: Ming Qian <ming.qian@nxp.com>
Date: Fri, 10 Jun 2022 15:31:45 +0800
Subject: [PATCH 61/93] MMFMWK-9186-2 video: Add support for Y312_LE

Y312_LE is a YUV format with 12-bits per component like YUV24,
expanded to 16bits.
Data in the 12 high bits, zeros in the 4 low bits,
arranged in little endian order.
---
 gst-libs/gst/video/video-converter.c |  1 +
 gst-libs/gst/video/video-format.c    | 55 ++++++++++++++++++++++++++++
 gst-libs/gst/video/video-format.h    | 14 ++++++-
 gst-libs/gst/video/video-info.c      |  5 +++
 4 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/gst-libs/gst/video/video-converter.c b/gst-libs/gst/video/video-converter.c
index 8f56c1367..fe68a89d1 100644
--- a/gst-libs/gst/video/video-converter.c
+++ b/gst-libs/gst/video/video-converter.c
@@ -7368,6 +7368,7 @@ get_scale_format (GstVideoFormat format, gint plane)
     case GST_VIDEO_FORMAT_Y012_LE:
     case GST_VIDEO_FORMAT_Y212_BE:
     case GST_VIDEO_FORMAT_Y212_LE:
+    case GST_VIDEO_FORMAT_Y312_LE:
     case GST_VIDEO_FORMAT_Y412_BE:
     case GST_VIDEO_FORMAT_Y412_LE:
     case GST_VIDEO_FORMAT_NV12_8L128:
diff --git a/gst-libs/gst/video/video-format.c b/gst-libs/gst/video/video-format.c
index e51dd6acf..0d10d6143 100644
--- a/gst-libs/gst/video/video-format.c
+++ b/gst-libs/gst/video/video-format.c
@@ -6670,6 +6670,59 @@ pack_Y212_LE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
   }
 }
 
+#define PACK_Y312_LE GST_VIDEO_FORMAT_AYUV64, unpack_Y312_LE, 1, pack_Y312_LE
+static void
+unpack_Y312_LE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
+    gpointer dest, const gpointer data[GST_VIDEO_MAX_PLANES],
+    const gint stride[GST_VIDEO_MAX_PLANES], gint x, gint y, gint width)
+{
+  int i;
+  const guint16 *restrict s = GET_LINE (y);
+  guint16 *restrict d = dest;
+  guint16 Y, U, V;
+
+  s += x * 3;
+
+  for (i = 0; i < width; i++) {
+    Y = GST_READ_UINT16_LE (s + 3 * i + 0) & 0xfff0;
+    U = GST_READ_UINT16_LE (s + 3 * i + 1) & 0xfff0;
+    V = GST_READ_UINT16_LE (s + 3 * i + 2) & 0xfff0;
+
+    if (!(flags & GST_VIDEO_PACK_FLAG_TRUNCATE_RANGE)) {
+      U |= (U >> 12);
+      Y |= (Y >> 12);
+      V |= (V >> 12);
+    }
+
+    d[4 * i + 0] = 0xffff;
+    d[4 * i + 1] = Y;
+    d[4 * i + 2] = U;
+    d[4 * i + 3] = V;
+  }
+}
+
+static void
+pack_Y312_LE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
+    const gpointer src, gint sstride, gpointer data[GST_VIDEO_MAX_PLANES],
+    const gint stride[GST_VIDEO_MAX_PLANES], GstVideoChromaSite chroma_site,
+    gint y, gint width)
+{
+  int i;
+  guint16 *restrict d = GET_LINE (y);
+  const guint16 *restrict s = src;
+  guint16 Y, U, V;
+
+  for (i = 0; i < width; i++) {
+    Y = s[4 * i + 1] & 0xfff0;
+    U = s[4 * i + 2] & 0xfff0;
+    V = s[4 * i + 3] & 0xfff0;
+
+    GST_WRITE_UINT16_LE (d + 3 * i + 0, Y);
+    GST_WRITE_UINT16_LE (d + 3 * i + 1, U);
+    GST_WRITE_UINT16_LE (d + 3 * i + 2, V);
+  }
+}
+
 #define PACK_Y412_BE GST_VIDEO_FORMAT_AYUV64, unpack_Y412_BE, 1, pack_Y412_BE
 static void
 unpack_Y412_BE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
@@ -7793,6 +7846,8 @@ static const VideoFormat formats[] = {
       SUB4444, PACK_RBGA),
   MAKE_GRAY_LE_FORMAT (Y012_LE, "raw video", DPTH12, PSTR2, PLANE0, OFFS0,
       SUB4, PACK_GRAY16_LE),
+  MAKE_YUV_LE_FORMAT (Y312_LE, "raw video", 0x00000000, DPTH12_12_12_HI,
+      PSTR222, PLANE0, OFFS0, SUB444, PACK_Y312_LE),
 };
 
 G_GNUC_END_IGNORE_DEPRECATIONS;
diff --git a/gst-libs/gst/video/video-format.h b/gst-libs/gst/video/video-format.h
index 361f62817..804cccb78 100644
--- a/gst-libs/gst/video/video-format.h
+++ b/gst-libs/gst/video/video-format.h
@@ -170,6 +170,7 @@ G_BEGIN_DECLS
  * @GST_VIDEO_FORMAT_GBR_16BE: planar 4:4:4 RGB, 16 bits per channel (Since: 1.24)
  * @GST_VIDEO_FORMAT_RBGA: packed RGB with alpha, 8 bits per channel (Since: 1.24)
  * @GST_VIDEO_FORMAT_Y012_LE: 12-bit grayscale, least significant byte first, expanded to 16bits, zeros in the 4 low bits,  (Since: 1.24)
+ * @GST_VIDEO_FORMAT_Y312_LE: packed 4:4:4 YUV, 12 bits per channel (Y-U-V) (Since: 1.24)
  *
  * Enum value describing the most common video formats.
  *
@@ -631,6 +632,15 @@ typedef enum {
    */
   GST_VIDEO_FORMAT_Y012_LE,
 
+  /**
+   * GST_VIDEO_FORMAT_Y312_LE:
+   *
+   * YUV 4:4:4 12 bits little endian.
+   * expanded to 16 bits, data in the 12 high bits, zeros in the 4 low bits
+   *
+   * Since: 1.24
+   */
+  GST_VIDEO_FORMAT_Y312_LE,
 } GstVideoFormat;
 
 #define GST_VIDEO_MAX_PLANES 4
@@ -1108,7 +1118,7 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "RGB10A2_LE, A444, GBRA, AYUV, VUYA, RGBA, RBGA, ARGB, BGRA, ABGR, A422, " \
     "A420, AV12, Y444_16BE, GBR_16BE, Y444_16LE, GBR_16LE, v216, P016_BE, " \
     "P016_LE, Y444_12BE, GBR_12BE, Y444_12LE, GBR_12LE, I422_12BE, I422_12LE, " \
-    "Y212_BE, Y012_LE, Y212_LE, I420_12BE, I420_12LE, P012_BE, P012_LE, Y444_10BE, " \
+    "Y212_BE, Y012_LE, Y212_LE, Y312_LE, I420_12BE, I420_12LE, P012_BE, P012_LE, Y444_10BE, " \
     "GBR_10BE, Y444_10LE, GBR_10LE, r210, I422_10BE, I422_10LE, NV16_10LE32, " \
     "Y210, UYVP, v210, I420_10BE, I420_10LE, P010_10BE, MT2110R, MT2110T, " \
     "NV12_10BE_8L128, NV12_10LE40_4L4, P010_10LE, NV12_10LE40, NV12_10LE32, " \
@@ -1127,7 +1137,7 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "A444, GBRA, AYUV, VUYA, RGBA, RBGA, ARGB, BGRA, ABGR, A422, A420, AV12, " \
     "Y444_16LE, GBR_16LE, Y444_16BE, GBR_16BE, v216, P016_LE, P016_BE, " \
     "Y444_12LE, GBR_12LE, Y444_12BE, GBR_12BE, I422_12LE, I422_12BE, Y012_LE, Y212_LE, " \
-    "Y212_BE, I420_12LE, I420_12BE, P012_LE, P012_BE, Y444_10LE, GBR_10LE, " \
+    "Y212_BE, Y312_LE, I420_12LE, I420_12BE, P012_LE, P012_BE, Y444_10LE, GBR_10LE, " \
     "Y444_10BE, GBR_10BE, r210, I422_10LE, I422_10BE, NV16_10LE32, Y210, UYVP, " \
     "v210, I420_10LE, I420_10BE, P010_10LE, NV12_10LE40, NV12_10LE32, " \
     "P010_10BE, MT2110R, MT2110T, NV12_10BE_8L128, NV12_10LE40_4L4, Y444, " \
diff --git a/gst-libs/gst/video/video-info.c b/gst-libs/gst/video/video-info.c
index 099e55c1f..c28d4a270 100644
--- a/gst-libs/gst/video/video-info.c
+++ b/gst-libs/gst/video/video-info.c
@@ -863,6 +863,11 @@ fill_planes (GstVideoInfo * info, gsize plane_size[GST_VIDEO_MAX_PLANES])
       info->offset[0] = 0;
       info->size = info->stride[0] * height;
       break;
+    case GST_VIDEO_FORMAT_Y312_LE:
+      info->stride[0] = width * 6;
+      info->offset[0] = 0;
+      info->size = info->stride[0] * height;
+      break;
     case GST_VIDEO_FORMAT_GRAY8:
       info->stride[0] = GST_ROUND_UP_4 (width);
       info->offset[0] = 0;
-- 
2.34.1


From 52656a604acbaf7550e242bc00d343ea8b2f89ac Mon Sep 17 00:00:00 2001
From: Ming Qian <ming.qian@nxp.com>
Date: Fri, 10 Jun 2022 16:15:41 +0800
Subject: [PATCH 62/93] MMFMWK-9186-3 video: Add support for BGR_12LE

BGR_12LE is reversed RGB format with 12 bits per component,
expanded to 16bits.
Data in the 12 high bits, zeros in the 4 low bits,
arranged in little endian order.

Conflicts:
	gst-libs/gst/video/video-format.h
---
 gst-libs/gst/video/video-converter.c |  1 +
 gst-libs/gst/video/video-format.c    | 54 ++++++++++++++++++++++++++++
 gst-libs/gst/video/video-format.h    | 15 ++++++--
 gst-libs/gst/video/video-info.c      |  1 +
 4 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/gst-libs/gst/video/video-converter.c b/gst-libs/gst/video/video-converter.c
index fe68a89d1..20d4c0191 100644
--- a/gst-libs/gst/video/video-converter.c
+++ b/gst-libs/gst/video/video-converter.c
@@ -7329,6 +7329,7 @@ get_scale_format (GstVideoFormat format, gint plane)
     case GST_VIDEO_FORMAT_GBRA_12LE:
     case GST_VIDEO_FORMAT_GBR_16BE:
     case GST_VIDEO_FORMAT_GBR_16LE:
+    case GST_VIDEO_FORMAT_BGR_12LE:
     case GST_VIDEO_FORMAT_NV12_64Z32:
     case GST_VIDEO_FORMAT_NV12_4L4:
     case GST_VIDEO_FORMAT_NV12_32L32:
diff --git a/gst-libs/gst/video/video-format.c b/gst-libs/gst/video/video-format.c
index 0d10d6143..16cc1024c 100644
--- a/gst-libs/gst/video/video-format.c
+++ b/gst-libs/gst/video/video-format.c
@@ -3486,6 +3486,58 @@ pack_GBRA_12BE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
   }
 }
 
+#define PACK_BGR_12LE GST_VIDEO_FORMAT_ARGB64, unpack_BGR_12LE, 1, pack_BGR_12LE
+static void
+unpack_BGR_12LE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
+    gpointer dest, const gpointer data[GST_VIDEO_MAX_PLANES],
+    const gint stride[GST_VIDEO_MAX_PLANES], gint x, gint y, gint width)
+{
+  int i;
+  const guint16 *restrict s = GET_LINE (y);
+  guint16 *d = dest, B, G, R;
+
+  s += x * 3;
+
+  for (i = 0; i < width; i++) {
+    B = GST_READ_UINT16_LE (s + 3 * i + 0) & 0xfff0;
+    G = GST_READ_UINT16_LE (s + 3 * i + 1) & 0xfff0;
+    R = GST_READ_UINT16_LE (s + 3 * i + 2) & 0xfff0;
+
+    if (!(flags & GST_VIDEO_PACK_FLAG_TRUNCATE_RANGE)) {
+      R |= (R >> 12);
+      G |= (G >> 12);
+      B |= (B >> 12);
+    }
+
+    d[i * 4 + 0] = 0xffff;
+    d[i * 4 + 1] = R;
+    d[i * 4 + 2] = G;
+    d[i * 4 + 3] = B;
+  }
+}
+
+static void
+pack_BGR_12LE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
+    const gpointer src, gint sstride, gpointer data[GST_VIDEO_MAX_PLANES],
+    const gint stride[GST_VIDEO_MAX_PLANES], GstVideoChromaSite chroma_site,
+    gint y, gint width)
+{
+  int i;
+  guint16 *restrict d = GET_LINE (y);
+  const guint16 *restrict s = src;
+  guint16 B, G, R;
+
+  for (i = 0; i < width; i++) {
+    R = s[i * 4 + 1];
+    G = s[i * 4 + 2];
+    B = s[i * 4 + 3];
+
+    GST_WRITE_UINT16_LE (d + 3 * i + 0, B);
+    GST_WRITE_UINT16_LE (d + 3 * i + 1, G);
+    GST_WRITE_UINT16_LE (d + 3 * i + 2, R);
+  }
+}
+
 #define PACK_Y444_10LE GST_VIDEO_FORMAT_AYUV64, unpack_Y444_10LE, 1, pack_Y444_10LE
 static void
 unpack_Y444_10LE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
@@ -7848,6 +7900,8 @@ static const VideoFormat formats[] = {
       SUB4, PACK_GRAY16_LE),
   MAKE_YUV_LE_FORMAT (Y312_LE, "raw video", 0x00000000, DPTH12_12_12_HI,
       PSTR222, PLANE0, OFFS0, SUB444, PACK_Y312_LE),
+  MAKE_RGB_LE_FORMAT (BGR_12LE, "raw video", DPTH12_12_12_HI, PSTR222, PLANE0,
+      OFFS0, SUB444, PACK_BGR_12LE),
 };
 
 G_GNUC_END_IGNORE_DEPRECATIONS;
diff --git a/gst-libs/gst/video/video-format.h b/gst-libs/gst/video/video-format.h
index 804cccb78..561f08790 100644
--- a/gst-libs/gst/video/video-format.h
+++ b/gst-libs/gst/video/video-format.h
@@ -171,6 +171,7 @@ G_BEGIN_DECLS
  * @GST_VIDEO_FORMAT_RBGA: packed RGB with alpha, 8 bits per channel (Since: 1.24)
  * @GST_VIDEO_FORMAT_Y012_LE: 12-bit grayscale, least significant byte first, expanded to 16bits, zeros in the 4 low bits,  (Since: 1.24)
  * @GST_VIDEO_FORMAT_Y312_LE: packed 4:4:4 YUV, 12 bits per channel (Y-U-V) (Since: 1.24)
+ * @GST_VIDEO_FORMAT_BGR_12LE: reverse RGB, 12 bits per channel (Since: 1.24)
  *
  * Enum value describing the most common video formats.
  *
@@ -641,6 +642,16 @@ typedef enum {
    * Since: 1.24
    */
   GST_VIDEO_FORMAT_Y312_LE,
+
+  /**
+   * GST_VIDEO_FORMAT_BGR_12LE:
+   *
+   * Reverse RGB, 12 bits little endian.
+   * expanded to 16 bits, data in the 12 high bits, zeros in the 4 low bits
+   *
+   * Since: 1.24
+   */
+  GST_VIDEO_FORMAT_BGR_12LE,
 } GstVideoFormat;
 
 #define GST_VIDEO_MAX_PLANES 4
@@ -1125,7 +1136,7 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "Y444, BGRP, GBR, RGBP, NV24, v308, IYU2, RGBx, xRGB, BGRx, xBGR, RGB, " \
     "BGR, Y42B, NV16, NV61, YUY2, YVYU, UYVY, VYUY, I420, YV12, NV12, NV21, " \
     "NV12_16L32S, NV12_32L32, NV12_4L4, NV12_64Z32, NV12_8L128, Y41B, IYU1, " \
-    "YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, GRAY16_BE, GRAY16_LE, " \
+    "YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, BGR_12LE, GRAY16_BE, GRAY16_LE, " \
     "GRAY10_LE32, GRAY8"
 #elif G_BYTE_ORDER == G_LITTLE_ENDIAN
 #define GST_VIDEO_FORMATS_ALL_STR "A444_16LE, A444_16BE, AYUV64, RGBA64_LE, " \
@@ -1144,7 +1155,7 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "BGRP, GBR, RGBP, NV24, v308, IYU2, RGBx, xRGB, BGRx, xBGR, RGB, BGR, " \
     "Y42B, NV16, NV61, YUY2, YVYU, UYVY, VYUY, I420, YV12, NV12, NV21, " \
     "NV12_16L32S, NV12_32L32, NV12_4L4, NV12_64Z32, NV12_8L128, Y41B, IYU1, " \
-    "YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, GRAY16_LE, GRAY16_BE, " \
+    "YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, BGR_12LE, GRAY16_LE, GRAY16_BE, " \
     "GRAY10_LE32, GRAY8"
 #endif
 
diff --git a/gst-libs/gst/video/video-info.c b/gst-libs/gst/video/video-info.c
index c28d4a270..1b23f0487 100644
--- a/gst-libs/gst/video/video-info.c
+++ b/gst-libs/gst/video/video-info.c
@@ -863,6 +863,7 @@ fill_planes (GstVideoInfo * info, gsize plane_size[GST_VIDEO_MAX_PLANES])
       info->offset[0] = 0;
       info->size = info->stride[0] * height;
       break;
+    case GST_VIDEO_FORMAT_BGR_12LE:
     case GST_VIDEO_FORMAT_Y312_LE:
       info->stride[0] = width * 6;
       info->offset[0] = 0;
-- 
2.34.1


From aaccfd3f74e510e0c505e44de7ec61dbffa523dc Mon Sep 17 00:00:00 2001
From: Ming Qian <ming.qian@nxp.com>
Date: Fri, 10 Jun 2022 17:00:02 +0800
Subject: [PATCH 63/93] MMFMWK-9186-4 video: Add support for BGRA_12LE

BGRA_12LE is reverse RGB with alpha channel last, 12 bits per component,
expanded to 16bits.
Data in the 12 high bits, zeros in the 4 low bits,
arranged in little endian order.

Conflicts:
	gst-libs/gst/video/video-format.h
---
 gst-libs/gst/video/video-converter.c |  1 +
 gst-libs/gst/video/video-format.c    | 58 ++++++++++++++++++++++++++++
 gst-libs/gst/video/video-format.h    | 15 ++++++-
 gst-libs/gst/video/video-info.c      |  5 +++
 4 files changed, 77 insertions(+), 2 deletions(-)

diff --git a/gst-libs/gst/video/video-converter.c b/gst-libs/gst/video/video-converter.c
index 20d4c0191..f51bb58ce 100644
--- a/gst-libs/gst/video/video-converter.c
+++ b/gst-libs/gst/video/video-converter.c
@@ -7330,6 +7330,7 @@ get_scale_format (GstVideoFormat format, gint plane)
     case GST_VIDEO_FORMAT_GBR_16BE:
     case GST_VIDEO_FORMAT_GBR_16LE:
     case GST_VIDEO_FORMAT_BGR_12LE:
+    case GST_VIDEO_FORMAT_BGRA_12LE:
     case GST_VIDEO_FORMAT_NV12_64Z32:
     case GST_VIDEO_FORMAT_NV12_4L4:
     case GST_VIDEO_FORMAT_NV12_32L32:
diff --git a/gst-libs/gst/video/video-format.c b/gst-libs/gst/video/video-format.c
index 16cc1024c..457cdfe88 100644
--- a/gst-libs/gst/video/video-format.c
+++ b/gst-libs/gst/video/video-format.c
@@ -3538,6 +3538,62 @@ pack_BGR_12LE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
   }
 }
 
+#define PACK_BGRA_12LE GST_VIDEO_FORMAT_ARGB64, unpack_BGRA_12LE, 1, pack_BGRA_12LE
+static void
+unpack_BGRA_12LE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
+    gpointer dest, const gpointer data[GST_VIDEO_MAX_PLANES],
+    const gint stride[GST_VIDEO_MAX_PLANES], gint x, gint y, gint width)
+{
+  int i;
+  const guint16 *restrict s = GET_LINE (y);
+  guint16 *d = dest, B, G, R, A;
+
+  s += x * 4;
+
+  for (i = 0; i < width; i++) {
+    B = GST_READ_UINT16_LE (s + 4 * i + 0) & 0xfff0;
+    G = GST_READ_UINT16_LE (s + 4 * i + 1) & 0xfff0;
+    R = GST_READ_UINT16_LE (s + 4 * i + 2) & 0xfff0;
+    A = GST_READ_UINT16_LE (s + 4 * i + 3) & 0xfff0;
+
+    if (!(flags & GST_VIDEO_PACK_FLAG_TRUNCATE_RANGE)) {
+      R |= (R >> 12);
+      G |= (G >> 12);
+      B |= (B >> 12);
+      A |= (A >> 12);
+    }
+
+    d[i * 4 + 0] = A;
+    d[i * 4 + 1] = R;
+    d[i * 4 + 2] = G;
+    d[i * 4 + 3] = B;
+  }
+}
+
+static void
+pack_BGRA_12LE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
+    const gpointer src, gint sstride, gpointer data[GST_VIDEO_MAX_PLANES],
+    const gint stride[GST_VIDEO_MAX_PLANES], GstVideoChromaSite chroma_site,
+    gint y, gint width)
+{
+  int i;
+  guint16 *restrict d = GET_LINE (y);
+  const guint16 *restrict s = src;
+  guint16 B, G, R, A;
+
+  for (i = 0; i < width; i++) {
+    A = s[i * 4 + 0];
+    R = s[i * 4 + 1];
+    G = s[i * 4 + 2];
+    B = s[i * 4 + 3];
+
+    GST_WRITE_UINT16_LE (d + 4 * i + 0, B);
+    GST_WRITE_UINT16_LE (d + 4 * i + 1, G);
+    GST_WRITE_UINT16_LE (d + 4 * i + 2, R);
+    GST_WRITE_UINT16_LE (d + 4 * i + 3, A);
+  }
+}
+
 #define PACK_Y444_10LE GST_VIDEO_FORMAT_AYUV64, unpack_Y444_10LE, 1, pack_Y444_10LE
 static void
 unpack_Y444_10LE (const GstVideoFormatInfo * info, GstVideoPackFlags flags,
@@ -7902,6 +7958,8 @@ static const VideoFormat formats[] = {
       PSTR222, PLANE0, OFFS0, SUB444, PACK_Y312_LE),
   MAKE_RGB_LE_FORMAT (BGR_12LE, "raw video", DPTH12_12_12_HI, PSTR222, PLANE0,
       OFFS0, SUB444, PACK_BGR_12LE),
+  MAKE_RGB_LE_FORMAT (BGRA_12LE, "raw video", DPTH12_12_12_12_HI, PSTR8888, PLANE0,
+      OFFS0, SUB444, PACK_BGRA_12LE),
 };
 
 G_GNUC_END_IGNORE_DEPRECATIONS;
diff --git a/gst-libs/gst/video/video-format.h b/gst-libs/gst/video/video-format.h
index 561f08790..09512ac5d 100644
--- a/gst-libs/gst/video/video-format.h
+++ b/gst-libs/gst/video/video-format.h
@@ -172,6 +172,7 @@ G_BEGIN_DECLS
  * @GST_VIDEO_FORMAT_Y012_LE: 12-bit grayscale, least significant byte first, expanded to 16bits, zeros in the 4 low bits,  (Since: 1.24)
  * @GST_VIDEO_FORMAT_Y312_LE: packed 4:4:4 YUV, 12 bits per channel (Y-U-V) (Since: 1.24)
  * @GST_VIDEO_FORMAT_BGR_12LE: reverse RGB, 12 bits per channel (Since: 1.24)
+ * @GST_VIDEO_FORMAT_BGRA_12LE: reverse RGB with alpha channel last, 12 bits per channel (Since: 1.24)
  *
  * Enum value describing the most common video formats.
  *
@@ -652,6 +653,16 @@ typedef enum {
    * Since: 1.24
    */
   GST_VIDEO_FORMAT_BGR_12LE,
+
+  /**
+   * GST_VIDEO_FORMAT_BGRA_12LE:
+   *
+   * Reverse RGB with alpha channel last, 12 bits little endian.
+   * expanded to 16 bits, data in the 12 high bits, zeros in the 4 low bits
+   *
+   * Since: 1.24
+   */
+  GST_VIDEO_FORMAT_BGRA_12LE,
 } GstVideoFormat;
 
 #define GST_VIDEO_MAX_PLANES 4
@@ -1136,7 +1147,7 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "Y444, BGRP, GBR, RGBP, NV24, v308, IYU2, RGBx, xRGB, BGRx, xBGR, RGB, " \
     "BGR, Y42B, NV16, NV61, YUY2, YVYU, UYVY, VYUY, I420, YV12, NV12, NV21, " \
     "NV12_16L32S, NV12_32L32, NV12_4L4, NV12_64Z32, NV12_8L128, Y41B, IYU1, " \
-    "YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, BGR_12LE, GRAY16_BE, GRAY16_LE, " \
+    "YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, BGR_12LE, BGRA_12LE, GRAY16_BE, GRAY16_LE, " \
     "GRAY10_LE32, GRAY8"
 #elif G_BYTE_ORDER == G_LITTLE_ENDIAN
 #define GST_VIDEO_FORMATS_ALL_STR "A444_16LE, A444_16BE, AYUV64, RGBA64_LE, " \
@@ -1155,7 +1166,7 @@ gconstpointer  gst_video_format_get_palette          (GstVideoFormat format, gsi
     "BGRP, GBR, RGBP, NV24, v308, IYU2, RGBx, xRGB, BGRx, xBGR, RGB, BGR, " \
     "Y42B, NV16, NV61, YUY2, YVYU, UYVY, VYUY, I420, YV12, NV12, NV21, " \
     "NV12_16L32S, NV12_32L32, NV12_4L4, NV12_64Z32, NV12_8L128, Y41B, IYU1, " \
-    "YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, BGR_12LE, GRAY16_LE, GRAY16_BE, " \
+    "YUV9, YVU9, BGR16, RGB16, BGR15, RGB15, RGB8P, BGR_12LE, BGRA_12LE, GRAY16_LE, GRAY16_BE, " \
     "GRAY10_LE32, GRAY8"
 #endif
 
diff --git a/gst-libs/gst/video/video-info.c b/gst-libs/gst/video/video-info.c
index 1b23f0487..42a74e517 100644
--- a/gst-libs/gst/video/video-info.c
+++ b/gst-libs/gst/video/video-info.c
@@ -869,6 +869,11 @@ fill_planes (GstVideoInfo * info, gsize plane_size[GST_VIDEO_MAX_PLANES])
       info->offset[0] = 0;
       info->size = info->stride[0] * height;
       break;
+    case GST_VIDEO_FORMAT_BGRA_12LE:
+      info->stride[0] = width * 8;
+      info->offset[0] = 0;
+      info->size = info->stride[0] * height;
+      break;
     case GST_VIDEO_FORMAT_GRAY8:
       info->stride[0] = GST_ROUND_UP_4 (width);
       info->offset[0] = 0;
-- 
2.34.1


From 5fc0a36bc5356698e564647beb0855231b271172 Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Thu, 16 Feb 2023 09:13:35 +0800
Subject: [PATCH 64/93] Disable HAVE_XI2 for ximagesink

Config HAVE_XI2 is added in below 1.22 MR
https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/1633,
but it causes error "X Error of failed request: XInputExtension".
So remove this config.

Signed-off-by: Hou Qi <qi.hou@nxp.com>
---
 sys/meson.build | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sys/meson.build b/sys/meson.build
index a93e07581..b7dcd00d9 100644
--- a/sys/meson.build
+++ b/sys/meson.build
@@ -4,7 +4,6 @@ if x11_dep.found()
   xshm_dep = dependency('xext', required : get_option('xshm'))
   xi_dep = dependency('xi', required : get_option('xi'))
   core_conf.set('HAVE_XSHM', xshm_dep.found())
-  core_conf.set('HAVE_XI2', xi_dep.found())
 
   subdir('ximage')
   subdir('xvimage')
-- 
2.34.1


From c2a59645a309955082257ea92e4bb7e33640d857 Mon Sep 17 00:00:00 2001
From: Elliot Chen <elliot.chen@nxp.com>
Date: Tue, 21 Feb 2023 14:06:57 +0800
Subject: [PATCH 65/93] MMFMWK-9196 uridecodebin3: fix the hang issue when
 setting connection speed

Signed-off-by: Elliot Chen <elliot.chen@nxp.com>
---
 gst/playback/gsturidecodebin3.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/gst/playback/gsturidecodebin3.c b/gst/playback/gsturidecodebin3.c
index 09e1b9b5e..3f3761a4e 100644
--- a/gst/playback/gsturidecodebin3.c
+++ b/gst/playback/gsturidecodebin3.c
@@ -1601,13 +1601,18 @@ static void
 gst_uri_decode_bin3_set_connection_speed (GstURIDecodeBin3 * dec)
 {
   GList *walk;
+  guint64 connection_speed;
+
+  GST_OBJECT_LOCK (dec);
+  connection_speed = dec->connection_speed;
+  GST_OBJECT_UNLOCK (dec);
 
   /* set the property on all decodebins now */
   for (walk = dec->source_handlers; walk; walk = g_list_next (walk)) {
     GstSourceHandler *handler = (GstSourceHandler *) (walk->data);
 
     g_object_set (handler->urisourcebin, "connection-speed",
-        dec->connection_speed / 1000, NULL);
+        connection_speed / 1000, NULL);
   }
 }
 
@@ -1627,8 +1632,8 @@ gst_uri_decode_bin3_set_property (GObject * object, guint prop_id,
     case PROP_CONNECTION_SPEED:
       GST_OBJECT_LOCK (dec);
       dec->connection_speed = g_value_get_uint64 (value) * 1000;
-      gst_uri_decode_bin3_set_connection_speed (dec);
       GST_OBJECT_UNLOCK (dec);
+      gst_uri_decode_bin3_set_connection_speed (dec);
       break;
     case PROP_BUFFER_SIZE:
       dec->buffer_size = g_value_get_int (value);
-- 
2.34.1


From 53f3d475f430cd9b912b4feaa9894f7a302c3e37 Mon Sep 17 00:00:00 2001
From: Elliot Chen <elliot.chen@nxp.com>
Date: Thu, 6 Apr 2023 10:40:25 +0800
Subject: [PATCH 66/93] MMFMWK-9219 decodebin3: can not remove input stream if
 it is eos when handling pending input

For some streams whose duration is small, should not remove
input stream if it is eos when handling pending input, otherwise
it can not guarantee that decodebin3 can send out eos event successfully

Signed-off-by: Elliot Chen <elliot.chen@nxp.com>
---
 gst/playback/gstdecodebin3.c | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/gst/playback/gstdecodebin3.c b/gst/playback/gstdecodebin3.c
index 20112721a..b56ac098e 100644
--- a/gst/playback/gstdecodebin3.c
+++ b/gst/playback/gstdecodebin3.c
@@ -1311,12 +1311,10 @@ gst_decodebin_input_unblock_streams (DecodebinInput * input,
       input_stream->buffer_probe_id = 0;
     }
 
-    if (input_stream->saw_eos) {
-      GST_DEBUG_OBJECT (dbin, "Removing EOS'd stream");
-      remove_input_stream (dbin, input_stream);
-      tmp = dbin->input_streams;
-    } else
-      tmp = next;
+    /* For some streams whose duration is small, should not remove
+     * input stream if it's EOS, otherwise it can't guarantee that decodebin3
+     * can send out eos event successfully in multiqueue_src_probe function */
+    tmp = next;
   }
 
   /* Weed out unused multiqueue slots */
-- 
2.34.1


From 51e6e8adb393ae03f21597ffcc603818cbe6c0b1 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Tue, 11 Apr 2023 15:51:13 +0800
Subject: [PATCH 67/93] LF-8635 decodebin3: set min_interleave of multiqueue to
 5s+

enlarge min_interleave to workaround the issue that pipeline will
hang when AV interleave is big

upstream status:imx specific
---
 gst/playback/gstdecodebin3.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gst/playback/gstdecodebin3.c b/gst/playback/gstdecodebin3.c
index b56ac098e..7559bb360 100644
--- a/gst/playback/gstdecodebin3.c
+++ b/gst/playback/gstdecodebin3.c
@@ -1404,7 +1404,7 @@ gst_decodebin3_update_min_interleave (GstDecodebin3 * dbin)
     return;
 
   /* Make sure we keep an extra overhead */
-  max_latency += 100 * GST_MSECOND;
+  max_latency += 5 * GST_SECOND;
   if (max_latency == dbin->current_mq_min_interleave)
     return;
 
-- 
2.34.1


From 1a312c41c6c08f348ed07c474851231657e51bb3 Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Tue, 11 Apr 2023 16:23:38 +0800
Subject: [PATCH 68/93] videodecoder: avoid duplicate timestamps for buffers
 inputted with DTS but need reorder

Some containers like AVI set DTS instead of PTS on the buffer. If the
input buffer timestamps are DTS, but the decoder reorders frames, then
needs videodecoder to calculate the PTS for output frames.

For example, the first 4 input buffer timestamps are 0,33ms,66ms,100ms.
But the decoder output frames sequence is 0->2->1->3.

In such case, videodecoder sets flag reordered_output to TRUE only when
frame's DTS is smaller that the previous output timestamp currently.
It causes output frame timestamps to be 0,66ms,66ms,100ms.

To avoid above duplicate timestamp, we need to set reordered_output to
TRUE when the frame's DTS is larger than any DTS still in the frame list.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4375>
Status: In review
---
 gst-libs/gst/video/gstvideodecoder.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gst-libs/gst/video/gstvideodecoder.c b/gst-libs/gst/video/gstvideodecoder.c
index 4fa7449d3..4793de282 100644
--- a/gst-libs/gst/video/gstvideodecoder.c
+++ b/gst-libs/gst/video/gstvideodecoder.c
@@ -3096,6 +3096,9 @@ gst_video_decoder_prepare_finish_frame (GstVideoDecoder *
         GST_TIME_ARGS (frame->pts));
   }
 
+  if (GST_CLOCK_TIME_IS_VALID (frame->pts) && frame->pts > min_pts)
+    priv->reordered_output = TRUE;
+
   /* if we detected reordered output, then PTS are void, however those were
    * obtained; bogus input, subclass etc */
   if (priv->reordered_output && !frames_without_pts) {
@@ -3142,7 +3145,6 @@ gst_video_decoder_prepare_finish_frame (GstVideoDecoder *
     GST_WARNING_OBJECT (decoder,
         "decreasing timestamp (%" GST_TIME_FORMAT " < %" GST_TIME_FORMAT ")",
         GST_TIME_ARGS (frame->pts), GST_TIME_ARGS (priv->last_timestamp_out));
-    priv->reordered_output = TRUE;
     /* make it a bit less weird downstream */
     frame->pts = priv->last_timestamp_out;
   }
-- 
2.34.1


From 32edd2366ebe045f0c4d9910848012a44f63c1da Mon Sep 17 00:00:00 2001
From: Elliot Chen <elliot.chen@nxp.com>
Date: Wed, 19 Apr 2023 16:29:07 +0800
Subject: [PATCH 69/93] MMFMWK-9229 playbin3: support audio passthrough with
 playbin3

playbin3 decode the audio or video with default fixed caps
without querying the caps of audio sink.Should update it to
realize the audio passthrough function.

Signed-off-by: Elliot Chen <elliot.chen@nxp.com>
---
 gst/playback/gstdecodebin3.c    | 25 +++++++++
 gst/playback/gstplaybin3.c      | 93 +++++++++++++++++++++++++++++++++
 gst/playback/gstrawcaps.h       | 26 +++++++++
 gst/playback/gsturidecodebin3.c |  2 +
 4 files changed, 146 insertions(+)

diff --git a/gst/playback/gstdecodebin3.c b/gst/playback/gstdecodebin3.c
index 7559bb360..a0195189a 100644
--- a/gst/playback/gstdecodebin3.c
+++ b/gst/playback/gstdecodebin3.c
@@ -1954,6 +1954,31 @@ sink_query_function (GstPad * sinkpad, GstDecodebin3 * dbin, GstQuery * query)
 
   GST_DEBUG_OBJECT (sinkpad, "query %" GST_PTR_FORMAT, query);
 
+  /* report the caps if needed */
+  if (GST_QUERY_TYPE (query) == GST_QUERY_CAPS) {
+    GstCaps *select_caps = gst_caps_from_string (AUDIO_PASSTHROUGH_CAPS);
+
+    /* report if has new caps which is not in the default caps list */
+    if (gst_caps_is_subset (select_caps, dbin->caps)) {
+      GstCaps *filter;
+
+      gst_query_parse_caps (query, &filter);
+      if (filter) {
+        if (gst_caps_can_intersect (filter, dbin->caps)) {
+          GstCaps *intersection;
+
+          intersection = gst_caps_intersect_full (filter, dbin->caps,
+              GST_CAPS_INTERSECT_FIRST);
+          gst_query_set_caps_result (query, intersection);
+          gst_caps_unref (intersection);
+          gst_caps_unref (select_caps);
+          return TRUE;
+        }
+      }
+    }
+    gst_caps_unref (select_caps);
+  }
+
   /* We accept any caps, since we will reconfigure ourself internally if the new
    * stream is incompatible */
   if (GST_QUERY_TYPE (query) == GST_QUERY_ACCEPT_CAPS) {
diff --git a/gst/playback/gstplaybin3.c b/gst/playback/gstplaybin3.c
index e3d876bf0..52c3d9110 100644
--- a/gst/playback/gstplaybin3.c
+++ b/gst/playback/gstplaybin3.c
@@ -232,6 +232,7 @@
 #include "gstplaysink.h"
 #include "gstsubtitleoverlay.h"
 #include "gstplaybackutils.h"
+#include "gstrawcaps.h"
 
 GST_DEBUG_CATEGORY_STATIC (gst_play_bin3_debug);
 #define GST_CAT_DEFAULT gst_play_bin3_debug
@@ -281,6 +282,12 @@ enum
   PLAYBIN_STREAM_LAST
 };
 
+static GstStaticCaps default_raw_caps = GST_STATIC_CAPS (DEFAULT_RAW_CAPS);
+#define DEFAULT_CAPS (gst_static_caps_get (&default_raw_caps))
+
+static GstStaticCaps raw_and_passthrough_caps = GST_STATIC_CAPS (RAW_AND_PASSTHROUGH_CAPS);
+#define SELECT_CAPS (gst_static_caps_get (&raw_and_passthrough_caps))
+
 /* names matching the enum above */
 static const gchar *stream_type_names[] = {
   "audio", "video", "text"
@@ -511,6 +518,7 @@ static void gst_play_bin3_navigation_init (gpointer g_iface,
     gpointer g_iface_data);
 static void gst_play_bin3_colorbalance_init (gpointer g_iface,
     gpointer g_iface_data);
+static void reconfigure_caps (GstPlayBin3 * playbin);
 
 static void
 _do_init_type (GType type)
@@ -1460,6 +1468,7 @@ gst_play_bin3_set_property (GObject * object, guint prop_id,
     case PROP_AUDIO_SINK:
       gst_play_bin3_set_sink (playbin, GST_PLAY_SINK_TYPE_AUDIO, "audio",
           &playbin->audio_sink, g_value_get_object (value));
+      reconfigure_caps (playbin);
       break;
     case PROP_VIS_PLUGIN:
       gst_play_sink_set_vis_plugin (playbin->playsink,
@@ -2548,6 +2557,85 @@ select_stream_cb (GstElement * decodebin, GstStreamCollection * collection,
   return -1;
 }
 
+static void
+reconfigure_caps (GstPlayBin3 * playbin)
+{
+  GstElement *actual_sink = NULL;
+  GstPad *sinkpad = NULL;
+  GstCaps *sinkcaps = NULL;
+  GstCaps *select_caps = NULL;
+  GstCaps *currentcaps = NULL;
+
+  if (NULL == playbin->audio_sink) {
+    GST_LOG_OBJECT (playbin, "No audio sink found");
+    return;
+  }
+
+  /* check the sink type */
+  if (!GST_IS_BIN ((GstBin*) playbin->audio_sink)) {
+    actual_sink = playbin->audio_sink;
+  } else {
+    GstIterator *it;
+    GValue item = { 0, };
+    GstIteratorResult rc;
+
+    it = gst_bin_iterate_sinks ((GstBin *) playbin->audio_sink);
+    do {
+      rc = gst_iterator_next (it, &item);
+      if (rc == GST_ITERATOR_OK) {
+        break;
+      }
+    } while (rc != GST_ITERATOR_DONE);
+
+    actual_sink = g_value_get_object (&item);
+    g_value_unset (&item);
+    gst_iterator_free (it);
+  }
+
+  if (NULL == actual_sink) {
+    GST_LOG_OBJECT (playbin, "No sink found");
+    return;
+  }
+
+  sinkpad = gst_element_get_static_pad (actual_sink, "sink");
+  if (NULL == sinkpad) {
+    GST_LOG_OBJECT (playbin, "No pad found");
+    return;
+  }
+
+  sinkcaps = gst_pad_query_caps (sinkpad, NULL);
+  if (NULL == sinkcaps) {
+    GST_LOG_OBJECT (playbin, "No caps found");
+    return;
+  }
+
+  GST_DEBUG_OBJECT (playbin, "sink caps: %s ", gst_caps_to_string(sinkcaps));
+  /* analyze the caps and update uridecodebin caps if needed */
+  if (!gst_caps_is_any (sinkcaps)) {
+    select_caps = gst_caps_from_string (AUDIO_PASSTHROUGH_CAPS);
+
+    if (gst_caps_is_subset (select_caps, sinkcaps)) {
+      /* got the selected caps, update it */
+      g_object_set (playbin->uridecodebin, "caps",
+          SELECT_CAPS, NULL);
+      GST_LOG_OBJECT (playbin, "update and add the selected caps");
+    } else {
+      g_object_get (playbin->uridecodebin, "caps", &currentcaps, NULL);
+      if (gst_caps_is_subset (select_caps, currentcaps)) {
+        /* restore the default caps if it has select caps */
+        g_object_set (playbin->uridecodebin, "caps",
+          DEFAULT_CAPS, NULL);
+        GST_LOG_OBJECT (playbin, "update to the default caps");
+      }
+      gst_caps_unref (currentcaps);
+    }
+    gst_caps_unref (select_caps);
+  }
+
+  gst_caps_unref (sinkcaps);
+  gst_object_unref (sinkpad);
+}
+
 /* We get called when the selected stream types change and
  * reconfiguration of output (i.e. playsink and potential combiners)
  * are required.
@@ -2646,6 +2734,11 @@ reconfigure_output (GstPlayBin3 * playbin)
 
         }
       }
+
+      /* update caps for audio passthrough */
+      if (combine->stream_type == GST_STREAM_TYPE_AUDIO) {
+        reconfigure_caps (playbin);
+      }
     }
   }
 
diff --git a/gst/playback/gstrawcaps.h b/gst/playback/gstrawcaps.h
index a3f2c0003..a8aa9a171 100644
--- a/gst/playback/gstrawcaps.h
+++ b/gst/playback/gstrawcaps.h
@@ -37,6 +37,32 @@ G_BEGIN_DECLS
     "closedcaption/x-cea-708; " \
     "application/x-onvif-metadata; "
 
+#define AUDIO_PASSTHROUGH_CAPS \
+    "audio/x-ac3, framed = (boolean) true;" \
+    "audio/x-eac3, framed = (boolean) true, alignment = (string) iec61937; "\
+    "audio/x-dts, framed = (boolean) true, " \
+      "block-size = (int) { 512, 1024, 2048 }; " \
+    "audio/mpeg, mpegversion = (int) 1, " \
+      "mpegaudioversion = (int) [ 1, 3 ], parsed = (boolean) true;"
+
+#define RAW_AND_PASSTHROUGH_CAPS \
+        "audio/x-ac3, framed = (boolean) true;" \
+        "audio/x-eac3, framed = (boolean) true, alignment = (string) iec61937; "\
+        "audio/x-dts, framed = (boolean) true, " \
+          "block-size = (int) { 512, 1024, 2048 }; " \
+        "audio/mpeg, mpegversion = (int) 1, " \
+          "mpegaudioversion = (int) [ 1, 3 ], parsed = (boolean) true; "\
+        "video/x-raw(ANY); " \
+        "audio/x-raw(ANY); " \
+        "text/x-raw(ANY); " \
+        "subpicture/x-dvd; " \
+        "subpicture/x-dvb; " \
+        "subpicture/x-xsub; " \
+        "subpicture/x-pgs; " \
+        "closedcaption/x-cea-608; " \
+        "closedcaption/x-cea-708; " \
+        "application/x-onvif-metadata; "
+
 G_END_DECLS
 
 #endif /* __GST_RAW_CAPS__ */
diff --git a/gst/playback/gsturidecodebin3.c b/gst/playback/gsturidecodebin3.c
index 3f3761a4e..676d67e34 100644
--- a/gst/playback/gsturidecodebin3.c
+++ b/gst/playback/gsturidecodebin3.c
@@ -1659,6 +1659,8 @@ gst_uri_decode_bin3_set_property (GObject * object, guint prop_id,
         gst_caps_unref (dec->caps);
       dec->caps = g_value_dup_boxed (value);
       GST_OBJECT_UNLOCK (dec);
+      /* need update caps because playsink may have some caps which is not in the raw caps list */
+      g_object_set (dec->decodebin, "caps", dec->caps, NULL);
       break;
     case PROP_INSTANT_URI:
       GST_OBJECT_LOCK (dec);
-- 
2.34.1


From f897b819a05536733099836d693ddfd0f27d9f6d Mon Sep 17 00:00:00 2001
From: Elliot Chen <elliot.chen@nxp.com>
Date: Mon, 29 May 2023 15:56:40 +0800
Subject: [PATCH 70/93] LF-9192 alsasink: return actual caps if alsasink does
 not open device

The caps obtained should not include passthrough
caps if passthrough function is not enabled.

Signed-off-by: Elliot Chen <elliot.chen@nxp.com>
---
 ext/alsa/gstalsasink.c | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/ext/alsa/gstalsasink.c b/ext/alsa/gstalsasink.c
index 54cb51ffb..fcf36ec87 100644
--- a/ext/alsa/gstalsasink.c
+++ b/ext/alsa/gstalsasink.c
@@ -124,6 +124,12 @@ static GstStaticPadTemplate alsasink_sink_factory =
         PASSTHROUGH_CAPS)
     );
 
+static GstStaticCaps alsasink_default_raw_caps =
+GST_STATIC_CAPS ("audio/x-raw, "
+        "format = (string) " GST_AUDIO_FORMATS_ALL ", "
+        "layout = (string) interleaved, "
+        "rate = (int) [ 1, MAX ], " "channels = (int) [ 1, MAX ];");
+
 static void
 gst_alsasink_finalise (GObject * object)
 {
@@ -308,8 +314,13 @@ gst_alsasink_getcaps (GstBaseSink * bsink, GstCaps * filter)
   GST_OBJECT_LOCK (sink);
   if (sink->handle == NULL) {
     GST_OBJECT_UNLOCK (sink);
-    GST_DEBUG_OBJECT (sink, "device not open, using template caps");
-    return NULL;                /* base class will get template caps for us */
+    if (TRUE == sink->passthrough) {
+      GST_DEBUG_OBJECT (sink, "device not open, using template caps");
+      return NULL;                /* base class will get template caps for us */
+    } else {
+      /* should not include passthrough caps if passthrough function is not enabled */
+      return gst_static_caps_get (&alsasink_default_raw_caps);
+    }
   }
 
   if (sink->cached_caps) {
-- 
2.34.1


From 0f8d6183e484023ce4b359ac198b80384944cdb9 Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Thu, 25 May 2023 12:05:42 +0800
Subject: [PATCH 71/93] LF-9146 parsebin: skip the element if it fails to send
 sticky events

If elements in parsebin send sticky events fails, need to skip this
element.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4705>
Status: pending
---
 gst/playback/gstparsebin.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/gst/playback/gstparsebin.c b/gst/playback/gstparsebin.c
index b686f9ce5..a28b834bb 100644
--- a/gst/playback/gstparsebin.c
+++ b/gst/playback/gstparsebin.c
@@ -2165,7 +2165,8 @@ connect_pad (GstParseBin * parsebin, GstElement * src, GstParsePad * parsepad,
     GST_PAD_STREAM_LOCK (sinkpad);
 
     if ((gst_element_set_state (element,
-                GST_STATE_PAUSED)) == GST_STATE_CHANGE_FAILURE) {
+                GST_STATE_PAUSED)) == GST_STATE_CHANGE_FAILURE ||
+        !send_sticky_events (parsebin, pad)) {
       GstParseElement *dtmp = NULL;
       GstElement *tmp = NULL;
       GstMessage *error_msg;
@@ -2245,7 +2246,6 @@ connect_pad (GstParseBin * parsebin, GstElement * src, GstParsePad * parsepad,
 
       continue;
     } else {
-      send_sticky_events (parsebin, pad);
       /* Everything went well, the spice must flow now */
       GST_PAD_STREAM_UNLOCK (sinkpad);
     }
-- 
2.34.1


From fc26096b580dd07d02d3c2545f6db76e99368439 Mon Sep 17 00:00:00 2001
From: Elliot Chen <elliot.chen@nxp.com>
Date: Thu, 24 Aug 2023 10:51:37 +0800
Subject: [PATCH 72/93] MMFMWK-9280 urisourcebin: support caps query if the
 element is not linked

Whenever an autoplugged element like ac3parse that
is not linked downstream yet and not exposed does
a query, It can't get the downstream supported caps.
Add the function to support it.

Signed-off-by: Elliot Chen <elliot.chen@nxp.com>
---
 gst/playback/gsturidecodebin3.c | 17 +++++++++
 gst/playback/gsturisourcebin.c  | 64 +++++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)

diff --git a/gst/playback/gsturidecodebin3.c b/gst/playback/gsturidecodebin3.c
index 676d67e34..c425214bc 100644
--- a/gst/playback/gsturidecodebin3.c
+++ b/gst/playback/gsturidecodebin3.c
@@ -1549,6 +1549,9 @@ new_source_handler (GstURIDecodeBin3 * uridecodebin, GstPlayItem * item,
       "ring-buffer-max-size", uridecodebin->ring_buffer_max_size,
       "parse-streams", TRUE, NULL);
 
+  g_object_set (handler->urisourcebin, "caps",
+        uridecodebin->caps, NULL);
+
   handler->pad_added_id =
       g_signal_connect (handler->urisourcebin, "pad-added",
       (GCallback) src_pad_added_cb, handler);
@@ -1616,6 +1619,19 @@ gst_uri_decode_bin3_set_connection_speed (GstURIDecodeBin3 * dec)
   }
 }
 
+static void
+gst_uri_decode_bin3_set_urisourcebin_caps (GstURIDecodeBin3 * dec)
+{
+  GList *walk;
+
+  /* set the property on all decodebins now */
+  for (walk = dec->source_handlers; walk; walk = g_list_next (walk)) {
+    GstSourceHandler *handler = (GstSourceHandler *) (walk->data);
+    g_object_set (handler->urisourcebin, "caps",
+        dec->caps, NULL);
+  }
+}
+
 static void
 gst_uri_decode_bin3_set_property (GObject * object, guint prop_id,
     const GValue * value, GParamSpec * pspec)
@@ -1660,6 +1676,7 @@ gst_uri_decode_bin3_set_property (GObject * object, guint prop_id,
       dec->caps = g_value_dup_boxed (value);
       GST_OBJECT_UNLOCK (dec);
       /* need update caps because playsink may have some caps which is not in the raw caps list */
+      gst_uri_decode_bin3_set_urisourcebin_caps (dec);
       g_object_set (dec->decodebin, "caps", dec->caps, NULL);
       break;
     case PROP_INSTANT_URI:
diff --git a/gst/playback/gsturisourcebin.c b/gst/playback/gsturisourcebin.c
index 8951defb6..6bd55f218 100644
--- a/gst/playback/gsturisourcebin.c
+++ b/gst/playback/gsturisourcebin.c
@@ -194,6 +194,7 @@ struct _GstURISourceBin
   gint last_buffering_pct;      /* Avoid sending buffering over and over */
   GMutex buffering_lock;
   GMutex buffering_post_lock;
+  GstCaps *caps;
 };
 
 struct _GstURISourceBinClass
@@ -263,6 +264,7 @@ enum
   PROP_HIGH_WATERMARK,
   PROP_STATISTICS,
   PROP_PARSE_STREAMS,
+  PROP_CAPS,
 };
 
 #define CUSTOM_EOS_QUARK _custom_eos_quark_get ()
@@ -474,6 +476,11 @@ gst_uri_source_bin_class_init (GstURISourceBinClass * klass)
           "Extract the elementary streams of non-raw sources",
           DEFAULT_PARSE_STREAMS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
+  g_object_class_install_property (gobject_class, PROP_CAPS,
+      g_param_spec_boxed ("caps", "Caps",
+          "The caps on which to support audio passthrough. (NULL = default)",
+          GST_TYPE_CAPS, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
   /**
    * GstURISourceBin::drained:
    *
@@ -548,6 +555,7 @@ gst_uri_source_bin_init (GstURISourceBin * urisrc)
   urisrc->last_buffering_pct = -1;
   urisrc->low_watermark = DEFAULT_LOW_WATERMARK;
   urisrc->high_watermark = DEFAULT_HIGH_WATERMARK;
+  urisrc->caps = DEFAULT_CAPS;
 
   GST_OBJECT_FLAG_SET (urisrc,
       GST_ELEMENT_FLAG_SOURCE | GST_BIN_FLAG_STREAMS_AWARE);
@@ -645,6 +653,11 @@ gst_uri_source_bin_set_property (GObject * object, guint prop_id,
     case PROP_PARSE_STREAMS:
       urisrc->parse_streams = g_value_get_boolean (value);
       break;
+    case PROP_CAPS:
+      if (urisrc->caps)
+        gst_caps_unref (urisrc->caps);
+      urisrc->caps = g_value_dup_boxed (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -707,6 +720,9 @@ gst_uri_source_bin_get_property (GObject * object, guint prop_id,
     case PROP_PARSE_STREAMS:
       g_value_set_boolean (value, urisrc->parse_streams);
       break;
+    case PROP_CAPS:
+      g_value_set_boxed (value, urisrc->caps);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -1551,6 +1567,51 @@ demuxer_pad_removed_cb (GstElement * element, GstPad * pad,
   return;
 }
 
+static gboolean
+autoplug_query_caps (GstElement * parsebin, GstPad * pad,
+    GstElement * element, GstQuery * query, ChildSrcPadInfo * info)
+{
+  GstURISourceBin *urisrc = info->urisrc;
+
+  GST_DEBUG_OBJECT (parsebin, "query %" GST_PTR_FORMAT, query);
+
+  /* This function is called whenever an autoplugged element that is
+   * not linked downstream yet and not exposed does a query. It can
+   * be used to tell the element about the downstream supported caps */
+  GstCaps *select_caps = gst_caps_from_string (AUDIO_PASSTHROUGH_CAPS);
+  if (gst_caps_is_subset (select_caps, urisrc->caps)) {
+    GstCaps *filter;
+    gst_query_parse_caps (query, &filter);
+    if (filter) {
+      if (gst_caps_can_intersect (filter, urisrc->caps)) {
+        GstCaps *intersection;
+
+        intersection = gst_caps_intersect_full (filter, urisrc->caps,
+            GST_CAPS_INTERSECT_FIRST);
+        gst_query_set_caps_result (query, intersection);
+        GST_DEBUG_OBJECT (parsebin, "query result %" GST_PTR_FORMAT, query);
+        gst_caps_unref (intersection);
+        gst_caps_unref (select_caps);
+        return TRUE;
+      }
+    }
+  }
+  gst_caps_unref (select_caps);
+  return FALSE;
+}
+
+static gboolean
+demuxer_pad_autoplug_query_cb (GstElement * parsebin, GstPad * pad,
+    GstElement * element, GstQuery * query, ChildSrcPadInfo * info)
+{
+  switch (GST_QUERY_TYPE (query)) {
+    case GST_QUERY_CAPS:
+      return autoplug_query_caps (parsebin, pad, element, query, info);
+    default:
+      return FALSE;
+  }
+}
+
 /* helper function to lookup stuff in lists */
 static gboolean
 array_has_value (const gchar * values[], const gchar * value)
@@ -2097,6 +2158,9 @@ setup_parsebin_for_slot (ChildSrcPadInfo * info, GstPad * originating_pad)
   g_signal_connect (info->demuxer,
       "pad-removed", G_CALLBACK (demuxer_pad_removed_cb), info);
 
+  g_signal_connect (info->demuxer,
+      "autoplug-query", G_CALLBACK (demuxer_pad_autoplug_query_cb), info);
+
   if (info->pre_parse_queue) {
     gst_element_set_locked_state (info->pre_parse_queue, FALSE);
     gst_element_sync_state_with_parent (info->pre_parse_queue);
-- 
2.34.1


From bc7e9f35ec40c2e9edde28e1a2fcc4fee80cbe2f Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Fri, 22 Mar 2024 11:36:23 +0900
Subject: [PATCH 73/93] video: dma-drm: add color format mappping about
 NV12_10LE40

add mapping relationship between GST and DRM NV12_10LE40 to drm NV15
---
 gst-libs/gst/video/video-info-dma.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/gst-libs/gst/video/video-info-dma.c b/gst-libs/gst/video/video-info-dma.c
index c2d7d4848..5feb51e8b 100644
--- a/gst-libs/gst/video/video-info-dma.c
+++ b/gst-libs/gst/video/video-info-dma.c
@@ -68,6 +68,13 @@
 #define DRM_FORMAT_NV61       fourcc_code('N', 'V', '6', '1')   /* 2x1 subsampled Cb:Cr plane */
 #define DRM_FORMAT_NV24       fourcc_code('N', 'V', '2', '4')   /* non-subsampled Cr:Cb plane */
 
+/*
+ * 2 plane YCbCr
+ * index 0 = Y plane, [39:0] Y3:Y2:Y1:Y0 little endian
+ * index 1 = Cr:Cb plane, [39:0] Cr1:Cb1:Cr0:Cb0 little endian
+ */
+#define DRM_FORMAT_NV15		fourcc_code('N', 'V', '1', '5') /* 2x2 subsampled Cr:Cb plane */
+
 /*
  * 3 plane YCbCr
  * index 0: Y plane, [7:0] Y
@@ -668,6 +675,7 @@ static const struct FormatMap
   {GST_VIDEO_FORMAT_P010_10LE, DRM_FORMAT_P010},
   {GST_VIDEO_FORMAT_P012_LE, DRM_FORMAT_P012},
   {GST_VIDEO_FORMAT_BGR10A2_LE, DRM_FORMAT_ARGB2101010},
+  {GST_VIDEO_FORMAT_NV12_10LE40, DRM_FORMAT_NV15},
 };
 /* *INDENT-ON* */
 
-- 
2.34.1


From e14b1b2aad0e3fa3dcd73b5d939e21ebc5ba74de Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Sun, 7 Apr 2024 17:44:44 +0900
Subject: [PATCH 74/93] gstimxcommon: add support for i.MX91

---
 gst-libs/gst/gstimxcommon.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/gst-libs/gst/gstimxcommon.h b/gst-libs/gst/gstimxcommon.h
index 2d05ca344..5b891f4e3 100755
--- a/gst-libs/gst/gstimxcommon.h
+++ b/gst-libs/gst/gstimxcommon.h
@@ -79,6 +79,7 @@ typedef enum
   CC_MX8ULP = CHIPCODE ('M', 'X', '8', 'U'),
   CC_MX93 = CHIPCODE ('M', 'X', '9', '3'),
   CC_MX95 = CHIPCODE ('M', 'X', '9', '5'),
+  CC_MX91 = CHIPCODE ('M', 'X', '9', '1'),
   CC_UNKN = CHIPCODE ('U', 'N', 'K', 'N')
 
 } CHIP_CODE;
@@ -132,6 +133,7 @@ typedef enum {
 #define IS_IMX8ULP() (CC_MX8ULP == imx_chip_code())
 #define IS_IMX93() (CC_MX93 == imx_chip_code())
 #define IS_IMX95() (CC_MX95 == imx_chip_code())
+#define IS_IMX91() (CC_MX91 == imx_chip_code())
 #define IS_IMX8Q() ((CC_MX8QM == imx_chip_code()) || (CC_MX8QXP == imx_chip_code()))
 #define IS_IMX6Q() (CC_MX6Q == imx_chip_code())
 
@@ -248,6 +250,7 @@ static SOC_INFO soc_info[] = {
   {CC_MX8ULP, "i.MX8ULP"},
   {CC_MX93, "i.MX93"},
   {CC_MX95, "i.MX95"},
+  {CC_MX91, "i.MX91"},
 };
 
 static CHIP_CODE getChipCodeFromSocid (void)
@@ -338,6 +341,7 @@ static IMXV4l2FeatureMap g_imxv4l2feature_maps[] = {
   {CC_MX8ULP, TRUE, TRUE, FALSE, TRUE, FALSE, FALSE, FALSE},
   {CC_MX93, FALSE, FALSE, FALSE, TRUE, FALSE, FALSE, FALSE},
   {CC_MX95, TRUE, TRUE, FALSE, FALSE, TRUE, TRUE, FALSE},
+  {CC_MX91, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE},
 };
 
 
-- 
2.34.1


From b4c2a028e7a291194b4c126e22c561bb8e8d901b Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Mon, 8 Apr 2024 16:38:43 +0900
Subject: [PATCH 75/93] MGS-7584 glupload: set default alignment based on
 platform

For Mali GPU, it needs 32 bytes alignment for buffer when
import dmabuf

upstream status: imx specific
---
 gst-libs/gst/gl/gstglupload.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index 6da247892..272588216 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -79,7 +79,11 @@
 #define USING_GLES2(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 2, 0))
 #define USING_GLES3(context) (gst_gl_context_check_gl_version (context, GST_GL_API_GLES2, 3, 0))
 
+#if GST_GL_HAVE_VIV_DIRECTVIV
 #define DEFAULT_ALIGN 16
+#else
+#define DEFAULT_ALIGN 32
+#endif
 
 GST_DEBUG_CATEGORY_STATIC (gst_gl_upload_debug);
 #define GST_CAT_DEFAULT gst_gl_upload_debug
-- 
2.34.1


From a7ab3d091b8cabb9cdcbc532ead4aa8c730e29ce Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Tue, 9 Apr 2024 17:53:40 +0800
Subject: [PATCH 76/93] MMFMWK-9358 Revert "decodebin3: Don't send sticky
 events to unlinked decoder"

Community remove send_sticky_event for some reason, but it causes
regression when playback streams in LF-9135 and MMFMWK-9212.

This reverts commit 8a10ce98248694b9c625fff86719dfb069cdf0ac.
---
 gst/playback/gstdecodebin3.c | 42 +++++++++++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/gst/playback/gstdecodebin3.c b/gst/playback/gstdecodebin3.c
index a0195189a..26588bcf5 100644
--- a/gst/playback/gstdecodebin3.c
+++ b/gst/playback/gstdecodebin3.c
@@ -3888,6 +3888,37 @@ db_output_stream_expose_src_pad (DecodebinOutputStream * output)
   gst_element_add_pad (GST_ELEMENT_CAST (slot->dbin), output->src_pad);
 }
 
+typedef struct
+{
+  gboolean ret;
+  GstPad *peer;
+} SendStickyEventsData;
+
+static gboolean
+send_sticky_event (GstPad * pad, GstEvent ** event, gpointer user_data)
+{
+  SendStickyEventsData *data = user_data;
+
+  data->ret &= gst_pad_send_event (data->peer, gst_event_ref (*event));
+
+  return data->ret;
+}
+
+static gboolean
+send_sticky_events (GstDecodebin3 * dbin, GstPad * pad)
+{
+  SendStickyEventsData data;
+
+  data.ret = TRUE;
+  data.peer = gst_pad_get_peer (pad);
+
+  gst_pad_sticky_events_foreach (pad, send_sticky_event, &data);
+
+  gst_object_unref (data.peer);
+
+  return data.ret;
+}
+
 static CandidateDecoder *
 add_candidate_decoder (GstDecodebin3 * dbin, GstElement * element)
 {
@@ -3993,20 +4024,29 @@ db_output_stream_setup_decoder (DecodebinOutputStream * output,
       goto try_next;
     }
 
+    /* First lock element's sinkpad stream lock so no data reaches
+       * the possible new element added when caps are sent by element
+       * while we're still sending sticky events */
+    GST_PAD_STREAM_LOCK (output->decoder_sink);
+
     if (gst_element_set_state (output->decoder, GST_STATE_PAUSED) ==
-        GST_STATE_CHANGE_FAILURE) {
+        GST_STATE_CHANGE_FAILURE ||
+          !send_sticky_events (dbin, slot->src_pad)) {
+      GST_PAD_STREAM_UNLOCK (output->decoder_sink);
       GST_WARNING_OBJECT (dbin, "Decoder '%s' failed to reach PAUSED state",
           GST_ELEMENT_NAME (output->decoder));
       goto try_next;
     }
 
     /* Everything went well, we have a decoder */
+    GST_PAD_STREAM_UNLOCK (output->decoder_sink);
     GST_DEBUG ("created decoder %" GST_PTR_FORMAT, output->decoder);
 
     handle_stored_latency_message (dbin, output, candidate);
     remove_candidate_decoder (dbin, candidate);
     break;
 
+
   try_next:{
       db_output_stream_reset (output);
       if (candidate)
-- 
2.34.1


From 06df948fe85b7b44130c7a571698029f3ab71e99 Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Fri, 24 May 2024 10:13:19 +0800
Subject: [PATCH 77/93] LF-12447 glcolorconvert: traverse all structures in
 caps when transform_caps

This is used to avoid negotiation failure between glcolorconvert
and downstream element.

upstream status: accepted
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6908>
---
 gst-libs/gst/gl/gstglcolorconvert.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/gst-libs/gst/gl/gstglcolorconvert.c b/gst-libs/gst/gl/gstglcolorconvert.c
index 7f37c2501..e1d8b13c8 100644
--- a/gst-libs/gst/gl/gstglcolorconvert.c
+++ b/gst-libs/gst/gl/gstglcolorconvert.c
@@ -1201,11 +1201,6 @@ gst_gl_color_convert_caps_transform_format_info (GstGLContext * context,
     st = gst_caps_get_structure (caps, i);
     f = gst_caps_get_features (caps, i);
 
-    /* If this is already expressed by the existing caps
-     * skip this structure */
-    if (i > 0 && gst_caps_is_subset_structure_full (res, st, f))
-      continue;
-
     format = gst_structure_get_value (st, "format");
     st = gst_structure_copy (st);
 
-- 
2.34.1


From c1e7a01bc7d758a019df273f11c665907295b3b8 Mon Sep 17 00:00:00 2001
From: Chao Guo <chao.guo@nxp.com>
Date: Fri, 14 Jun 2024 10:53:43 +0900
Subject: [PATCH 78/93] LF-12583 glupload: Add formats supported by
 #GstGLMemory to raw caps when generating sink pad caps

When glupload generates sink caps based on src caps, src caps may only contain RGBA format.
In this case, the raw caps on the sink pad generated by glupload will only contain the RGBA
format, which will cause caps negotiation fail, because the filter caps used for negotiation
by the upstream element may only contain other formats, such as xBGR, etc.

Add the formats supported by #GstGLMemory to raw caps to ensure that caps negotiation succeeds.

upstream status: accepted
part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7061>

Signed-off-by: Chao Guo <chao.guo@nxp.com>
---
 gst-libs/gst/gl/gstglupload.c | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index 272588216..3a88ac682 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -1932,7 +1932,7 @@ _dma_buf_upload_free (gpointer impl)
 
 static const UploadMethod _dma_buf_upload = {
   "Dmabuf",
-  0,
+  METHOD_FLAG_CAN_ACCEPT_RAW,
   &_dma_buf_upload_caps,
   &_dma_buf_upload_new,
   &_dma_buf_upload_transform_caps,
@@ -2077,7 +2077,7 @@ _direct_dma_buf_upload_transform_caps (gpointer impl, GstGLContext * context,
 
 static const UploadMethod _direct_dma_buf_upload = {
   "DirectDmabuf",
-  0,
+  METHOD_FLAG_CAN_ACCEPT_RAW,
   &_dma_buf_upload_caps,
   &_direct_dma_buf_upload_new,
   &_direct_dma_buf_upload_transform_caps,
@@ -2099,7 +2099,7 @@ _direct_dma_buf_external_upload_new (GstGLUpload * upload)
 
 static const UploadMethod _direct_dma_buf_external_upload = {
   "DirectDmabufExternal",
-  0,
+  METHOD_FLAG_CAN_ACCEPT_RAW,
   &_dma_buf_upload_caps,
   &_direct_dma_buf_external_upload_new,
   &_direct_dma_buf_upload_transform_caps,
@@ -3819,8 +3819,22 @@ gst_gl_upload_transform_caps (GstGLUpload * upload, GstGLContext * context,
         GstCapsFeatures *passthrough =
             gst_caps_features_from_string
             (GST_CAPS_FEATURE_META_GST_VIDEO_OVERLAY_COMPOSITION);
-        GstCaps *raw_tmp = _set_caps_features_with_passthrough (tmp,
+        GstCaps *raw_tmp = _set_caps_features_with_passthrough (caps,
             GST_CAPS_FEATURE_MEMORY_SYSTEM_MEMORY, passthrough);
+
+        /* Add formats supported by #GstGLMemory to raw caps */
+        for (i = 0; i < gst_caps_get_size (raw_tmp); i++) {
+          GstStructure *s = gst_caps_get_structure (raw_tmp, i);
+          GValue formats = G_VALUE_INIT;
+
+          g_value_init (&formats, GST_TYPE_LIST);
+          gst_value_deserialize (&formats, GST_GL_MEMORY_VIDEO_FORMATS_STR);
+          gst_structure_take_value (s, "format", &formats);
+
+          gst_structure_remove_fields (s, "texture-target", NULL);
+          g_value_unset (&formats);
+        }
+
         gst_caps_append (tmp, raw_tmp);
         gst_caps_features_free (passthrough);
       }
-- 
2.34.1


From 46e9be8685955b3feb12476b63be316b5157fcee Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Tue, 16 Jul 2024 15:59:16 +0800
Subject: [PATCH 79/93] MMFMWK-9398 videodecoder: fix one h264 stream free run
 playback with low performance

For stream that buffer has invalid pts and needs reorder, current
videodecoder will set this invalid pts as earliest_pts_frame's pts.
The output pts of frames whose dequeue order is later than input
order owing to reorder will be set to previous frame's output pts.

When free run playback, basesink uses buffer timestamp as running time.
Basetransform will drop frames if their running time is same with
previous one. To solve such issue, videodecoder should only set valid
pts/dts to earliest_pts_frame to avoid getting duplicate output ts.
---
 gst-libs/gst/video/gstvideodecoder.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/gst-libs/gst/video/gstvideodecoder.c b/gst-libs/gst/video/gstvideodecoder.c
index 4793de282..7283f0de7 100644
--- a/gst-libs/gst/video/gstvideodecoder.c
+++ b/gst-libs/gst/video/gstvideodecoder.c
@@ -3071,11 +3071,13 @@ gst_video_decoder_prepare_finish_frame (GstVideoDecoder *
     }
   }
   /* save dts if needed */
-  if (earliest_dts_frame && earliest_dts_frame != frame) {
+  if (earliest_dts_frame && earliest_dts_frame != frame
+      && GST_CLOCK_TIME_IS_VALID (frame->abidata.ABI.ts)) {
     earliest_dts_frame->abidata.ABI.ts = frame->abidata.ABI.ts;
   }
   /* save pts if needed */
-  if (earliest_pts_frame && earliest_pts_frame != frame) {
+  if (earliest_pts_frame && earliest_pts_frame != frame
+      && GST_CLOCK_TIME_IS_VALID (frame->abidata.ABI.ts2)) {
     earliest_pts_frame->abidata.ABI.ts2 = frame->abidata.ABI.ts2;
   }
 
-- 
2.34.1


From fae453985b8196835876fb233544c5b37f1ba908 Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Mon, 8 Jul 2024 09:31:10 +0800
Subject: [PATCH 80/93] LF-12650 parsebin: fix random hang when switching
 stream

Parsepad block id may have been added by expose_pad() in chain_expose(),
and removed after pads are added to parsebin. At this time, if expose_pad
again, the block id is added and has no chance to remove, then
mpegaudioparse waits unblocking when pushing the first buffer downstream.
It causes hang. So no need to expose_pad if the parsepad has been exposed.
---
 gst/playback/gstparsebin.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/gst/playback/gstparsebin.c b/gst/playback/gstparsebin.c
index a28b834bb..c5dd97ee8 100644
--- a/gst/playback/gstparsebin.c
+++ b/gst/playback/gstparsebin.c
@@ -1939,7 +1939,10 @@ connect_pad (GstParseBin * parsebin, GstElement * src, GstParsePad * parsepad,
       case GST_AUTOPLUG_SELECT_EXPOSE:
         GST_DEBUG_OBJECT (parsebin, "autoplug select requested expose");
         /* expose the pad, we don't have the source element */
-        expose_pad (parsebin, src, parsepad, pad, caps, chain);
+        if (!parsepad->exposed)
+          expose_pad (parsebin, src, parsepad, pad, caps, chain);
+        else
+          GST_DEBUG_OBJECT (parsebin, "pad is already exposed, skip");
         res = TRUE;
         goto beach;
       case GST_AUTOPLUG_SELECT_SKIP:
-- 
2.34.1


From bee327f1a7f6608fd212a697f45a73e3e314bdf8 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Wed, 10 Jul 2024 18:34:32 +0900
Subject: [PATCH 81/93] LF-12650 playsink: fix race condition when release pad

when upstream request release pad, there is chance that
playsink do reconfiguration at the same time, if the release
pad function unlock playsink to remove pad, reconfigration
will check the pad pointer and add back the chain which
is going to be release, need add remove pad into lock area
to avoid race condition between do reconfigure

gstplaybin3.c:2697:reconfigure_output:<playbin3>m Releasing playsink pad
gstplaysink.c:4720:gst_play_sink_release_pad:<playsink>m release pad <playsink:video_sink>
gstplaysink.c:4745:gst_play_sink_release_pad:<playsink>m deactivate pad <playsink:video_sink>
gstplaysink.c:4748:gst_play_sink_release_pad:<playsink>m untargeting pad <playsink:video_sink>
gstplaysink.c:4441:sinkpad_blocked_cb:<playsink:audio_sink>m Audio pad blocked
gstplaysink.c:4448:sinkpad_blocked_cb:<playsink>m All pads blocked -- reconfiguring
gstplaysink.c:3228:gst_play_sink_do_reconfigure:<playsink>m reconfiguring
gstplaysink.c:3247:gst_play_sink_do_reconfigure:<playsink>m Video pad is raw: 1
gstplaysink.c:3253:gst_play_sink_do_reconfigure:<playsink>m Audio pad is raw: 1
gstplaysink.c:3303:gst_play_sink_do_reconfigure:<playsink>m audio:1, video:1, vis:0, text:0
gstplaysink.c:3316:gst_play_sink_do_reconfigure:<playsink>m adding video, raw 1
gstplaysink.c:1767:gen_video_chain:<playsink>m making video chain 0xffffa02c1950
gstplaysink.c:1775:gen_video_chain:<playsink>m trying autovideosink
gstplaysink.c:4751:gst_play_sink_release_pad:<playsink>m remove pad <playsink:video_sink>
---
 gst/playback/gstplaysink.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/gst/playback/gstplaysink.c b/gst/playback/gstplaysink.c
index 7c38e815f..933566175 100644
--- a/gst/playback/gstplaysink.c
+++ b/gst/playback/gstplaysink.c
@@ -4739,8 +4739,6 @@ gst_play_sink_release_pad (GstPlaySink * playsink, GstPad * pad)
     untarget = FALSE;
   }
 
-  GST_PLAY_SINK_UNLOCK (playsink);
-
   if (*res) {
     GST_DEBUG_OBJECT (playsink, "deactivate pad %" GST_PTR_FORMAT, *res);
     gst_pad_set_active (*res, FALSE);
@@ -4753,8 +4751,6 @@ gst_play_sink_release_pad (GstPlaySink * playsink, GstPad * pad)
     *res = NULL;
   }
 
-  GST_PLAY_SINK_LOCK (playsink);
-
   /* If we have a pending reconfigure, we might have met the conditions
    * to reconfigure now */
   if (gst_play_sink_ready_to_reconfigure_locked (playsink)) {
-- 
2.34.1


From 9cd8d33b92cc8a01564f85753b4abdbf9d07ed55 Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Tue, 23 Jul 2024 19:03:37 +0900
Subject: [PATCH 82/93] LF-12650 parsebin: fix segfault when pad_event_cb is
 called early

when pad_event_cb is called by stream start, there is chance that
parser chain is not realy and cause segfault, so only access the pointer
when handle eos event
---
 gst/playback/gstparsebin.c | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/gst/playback/gstparsebin.c b/gst/playback/gstparsebin.c
index c5dd97ee8..786e18947 100644
--- a/gst/playback/gstparsebin.c
+++ b/gst/playback/gstparsebin.c
@@ -2477,16 +2477,19 @@ pad_event_cb (GstPad * pad, GstPadProbeInfo * info, gpointer data)
 {
   GstEvent *event = GST_PAD_PROBE_INFO_EVENT (info);
   GstPendingPad *ppad = (GstPendingPad *) data;
-  GstParseChain *chain = ppad->chain;
-  GstParseBin *parsebin = chain->parsebin;
 
-  g_assert (ppad);
-  g_assert (chain);
-  g_assert (parsebin);
   switch (GST_EVENT_TYPE (event)) {
     case GST_EVENT_EOS:
       GST_DEBUG_OBJECT (pad, "Received EOS on a non final pad, this stream "
           "ended too early");
+
+      GstParseChain *chain = ppad->chain;
+      GstParseBin *parsebin = chain->parsebin;
+
+      g_assert (ppad);
+      g_assert (chain);
+      g_assert (parsebin);
+
       chain->deadend = TRUE;
       chain->drained = TRUE;
       gst_object_replace ((GstObject **) & chain->current_pad, NULL);
-- 
2.34.1


From b64f301c51a613671fc441f9b790db28f03eb64a Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Fri, 26 Jul 2024 09:17:25 +0800
Subject: [PATCH 83/93] LF-12650 parsebin: fix playback not-linked and warning
 "Got data flow before stream-start event"

Parsebin set typefind, demux and parser to paused, store stream-start
and segment events as pipeline is not-linked yet. Then parser starts
to send first buffer to multiqueue after starting sink task.

At the same time, parsebin is exposing and sending stream-collection
event to downstream multiqueue which will send sticky stream-start
and segment event first. If first buffer sent before these events,
warning log "Got data flow before stream-start event" is printed and
pipeline reports error not-linked.

To avoid both issues, need to make sure parsebin exposes before
setting parser to paused.
---
 gst/playback/gstparsebin.c | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/gst/playback/gstparsebin.c b/gst/playback/gstparsebin.c
index 786e18947..7201a3b52 100644
--- a/gst/playback/gstparsebin.c
+++ b/gst/playback/gstparsebin.c
@@ -2167,6 +2167,21 @@ connect_pad (GstParseBin * parsebin, GstElement * src, GstParsePad * parsepad,
      * while we're still sending sticky events */
     GST_PAD_STREAM_LOCK (sinkpad);
 
+    if (is_parser_converter) {
+      EXPOSE_LOCK (parsebin);
+      if (parsebin->parse_chain) {
+        if (gst_parse_chain_is_complete (parsebin->parse_chain)) {
+          GST_LOG_OBJECT (parsebin,
+              "That was the last dynamic object, now attempting to expose the group");
+          if (!gst_parse_bin_expose (parsebin))
+            GST_WARNING_OBJECT (parsebin, "Couldn't expose group");
+        }
+      } else {
+        GST_DEBUG_OBJECT (parsebin, "No parse chain, new pad ignored");
+      }
+      EXPOSE_UNLOCK (parsebin);
+    }
+
     if ((gst_element_set_state (element,
                 GST_STATE_PAUSED)) == GST_STATE_CHANGE_FAILURE ||
         !send_sticky_events (parsebin, pad)) {
@@ -2521,19 +2536,6 @@ pad_added_cb (GstElement * element, GstPad * pad, GstParseChain * chain)
   analyze_new_pad (parsebin, element, pad, caps, chain);
   if (caps)
     gst_caps_unref (caps);
-
-  EXPOSE_LOCK (parsebin);
-  if (parsebin->parse_chain) {
-    if (gst_parse_chain_is_complete (parsebin->parse_chain)) {
-      GST_LOG_OBJECT (parsebin,
-          "That was the last dynamic object, now attempting to expose the group");
-      if (!gst_parse_bin_expose (parsebin))
-        GST_WARNING_OBJECT (parsebin, "Couldn't expose group");
-    }
-  } else {
-    GST_DEBUG_OBJECT (parsebin, "No parse chain, new pad ignored");
-  }
-  EXPOSE_UNLOCK (parsebin);
 }
 
 static void
-- 
2.34.1


From ee126c25ef0f0166459e88672a2645355ef3fea6 Mon Sep 17 00:00:00 2001
From: Chao Guo <chao.guo@nxp.com>
Date: Tue, 13 Aug 2024 11:53:22 +0900
Subject: [PATCH 84/93] LF-12583 glimagesink: libcamera with glvideomixer no
 image output

When re-negotiation and caps is changed, resize the viewport to
the corresponding video size in changed caps.

upstream status: pending
Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7341>

Signed-off-by: Chao Guo <chao.guo@nxp.com>
---
 ext/gl/gstglimagesink.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/ext/gl/gstglimagesink.c b/ext/gl/gstglimagesink.c
index 4d6b56234..d457f3a22 100644
--- a/ext/gl/gstglimagesink.c
+++ b/ext/gl/gstglimagesink.c
@@ -1548,11 +1548,18 @@ update_output_format (GstGLImageSink * glimage_sink)
   GstVideoMultiviewMode mv_mode;
   GstGLWindow *window = NULL;
   GstGLTextureTarget previous_target;
+  gint pre_width = 0, pre_height = 0;
+  gint cur_width = 0, cur_height = 0;
   GstStructure *s;
   const gchar *target_str;
   GstCaps *out_caps;
   gboolean ret;
 
+  pre_width = GST_VIDEO_INFO_WIDTH (out_info);
+  pre_height = GST_VIDEO_INFO_HEIGHT (out_info);
+  cur_width = GST_VIDEO_INFO_WIDTH (&glimage_sink->in_info);
+  cur_height = GST_VIDEO_INFO_HEIGHT (&glimage_sink->in_info);
+
   *out_info = glimage_sink->in_info;
   previous_target = glimage_sink->texture_target;
 
@@ -1646,8 +1653,10 @@ update_output_format (GstGLImageSink * glimage_sink)
     gst_caps_unref (glimage_sink->out_caps);
   glimage_sink->out_caps = out_caps;
 
-  if (previous_target != GST_GL_TEXTURE_TARGET_NONE &&
-      glimage_sink->texture_target != previous_target) {
+  if ((previous_target != GST_GL_TEXTURE_TARGET_NONE &&
+      glimage_sink->texture_target != previous_target) ||
+      (pre_width != 0 && pre_width != cur_width) ||
+      (pre_height != 0 && pre_height != cur_height)) {
     /* regenerate the shader for the changed target */
     GstGLWindow *window = gst_gl_context_get_window (glimage_sink->context);
     gst_gl_window_send_message (window,
-- 
2.34.1


From 9fa8034e226844cfcb67bf89cc79585c2a364403 Mon Sep 17 00:00:00 2001
From: Chao Guo <chao.guo@nxp.com>
Date: Tue, 24 Sep 2024 17:05:08 +0800
Subject: [PATCH 85/93] MMFMWK-9420 gloverlaycompositor: make element draw
 overlays.

textoverlay ! glvideomixer ! glimagesink

When textoverlay is placed before glvideomixer in the pipeline,
glimagesink cannot draw the text overlay specified by textoverlay
on the output image. Because glvideomixerelement cannot pass
GstVideoOverlayCompositionMeta downstream.

Therefore, let gloverlaycompositorelement in glvideomixer bin draw
the text overlay specified by textoverlay so that the pipeline can
output the expected picture.

Upstream Status: Inappropriate [i.MX specific]

Signed-off-by: Chao Guo <chao.guo@nxp.com>
---
 ext/gl/gstgloverlaycompositorelement.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/ext/gl/gstgloverlaycompositorelement.c b/ext/gl/gstgloverlaycompositorelement.c
index ef4929827..de741c463 100644
--- a/ext/gl/gstgloverlaycompositorelement.c
+++ b/ext/gl/gstgloverlaycompositorelement.c
@@ -338,10 +338,16 @@ gst_gl_overlay_compositor_element_callback (GstGLFilter * filter,
 {
   GstGLOverlayCompositorElement *self =
       GST_GL_OVERLAY_COMPOSITOR_ELEMENT (filter);
+  gfloat matrix[16] = {
+    1.0, 0.0, 0.0, 0.0,
+    0.0, 1.0, 0.0, 0.0,
+    0.0, 0.0, 1.0, 0.0,
+    0.0, 0.0, 0.0, 1.0,
+  };
 
   GST_LOG_OBJECT (self, "drawing overlays");
 
-  gst_gl_overlay_compositor_draw_overlays (self->overlay_compositor, NULL);
+  gst_gl_overlay_compositor_draw_overlays (self->overlay_compositor, matrix);
 
   return TRUE;
 }
-- 
2.34.1


From f239a6c75c5c526b1bd7b995d31e57ec4650e22a Mon Sep 17 00:00:00 2001
From: Haihua Hu <jared.hu@nxp.com>
Date: Thu, 26 Sep 2024 17:30:33 +0900
Subject: [PATCH 86/93] Add SCR.txt and LICENSE.txt for gstreamer 1.24.7

---
 LICENSE.txt | 503 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 SCR.txt     |   9 +
 2 files changed, 512 insertions(+)
 create mode 100644 LICENSE.txt
 create mode 100644 SCR.txt

diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 000000000..efce2a87c
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,503 @@
+		  GNU LESSER GENERAL PUBLIC LICENSE
+		       Version 2.1, February 1999
+
+ Copyright (C) 1991, 1999 Free Software Foundation, Inc.
+     51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+[This is the first released version of the Lesser GPL.  It also counts
+ as the successor of the GNU Library Public License, version 2, hence
+ the version number 2.1.]
+
+			    Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+Licenses are intended to guarantee your freedom to share and change
+free software--to make sure the software is free for all its users.
+
+  This license, the Lesser General Public License, applies to some
+specially designated software packages--typically libraries--of the
+Free Software Foundation and other authors who decide to use it.  You
+can use it too, but we suggest you first think carefully about whether
+this license or the ordinary General Public License is the better
+strategy to use in any particular case, based on the explanations below.
+
+  When we speak of free software, we are referring to freedom of use,
+not price.  Our General Public Licenses are designed to make sure that
+you have the freedom to distribute copies of free software (and charge
+for this service if you wish); that you receive source code or can get
+it if you want it; that you can change the software and use pieces of
+it in new free programs; and that you are informed that you can do
+these things.
+
+  To protect your rights, we need to make restrictions that forbid
+distributors to deny you these rights or to ask you to surrender these
+rights.  These restrictions translate to certain responsibilities for
+you if you distribute copies of the library or if you modify it.
+
+  For example, if you distribute copies of the library, whether gratis
+or for a fee, you must give the recipients all the rights that we gave
+you.  You must make sure that they, too, receive or can get the source
+code.  If you link other code with the library, you must provide
+complete object files to the recipients, so that they can relink them
+with the library after making changes to the library and recompiling
+it.  And you must show them these terms so they know their rights.
+
+  We protect your rights with a two-step method: (1) we copyright the
+library, and (2) we offer you this license, which gives you legal
+permission to copy, distribute and/or modify the library.
+
+  To protect each distributor, we want to make it very clear that
+there is no warranty for the free library.  Also, if the library is
+modified by someone else and passed on, the recipients should know
+that what they have is not the original version, so that the original
+author's reputation will not be affected by problems that might be
+introduced by others.
+
+  Finally, software patents pose a constant threat to the existence of
+any free program.  We wish to make sure that a company cannot
+effectively restrict the users of a free program by obtaining a
+restrictive license from a patent holder.  Therefore, we insist that
+any patent license obtained for a version of the library must be
+consistent with the full freedom of use specified in this license.
+
+  Most GNU software, including some libraries, is covered by the
+ordinary GNU General Public License.  This license, the GNU Lesser
+General Public License, applies to certain designated libraries, and
+is quite different from the ordinary General Public License.  We use
+this license for certain libraries in order to permit linking those
+libraries into non-free programs.
+
+  When a program is linked with a library, whether statically or using
+a shared library, the combination of the two is legally speaking a
+combined work, a derivative of the original library.  The ordinary
+General Public License therefore permits such linking only if the
+entire combination fits its criteria of freedom.  The Lesser General
+Public License permits more lax criteria for linking other code with
+the library.
+
+  We call this license the "Lesser" General Public License because it
+does Less to protect the user's freedom than the ordinary General
+Public License.  It also provides other free software developers Less
+of an advantage over competing non-free programs.  These disadvantages
+are the reason we use the ordinary General Public License for many
+libraries.  However, the Lesser license provides advantages in certain
+special circumstances.
+
+  For example, on rare occasions, there may be a special need to
+encourage the widest possible use of a certain library, so that it becomes
+a de-facto standard.  To achieve this, non-free programs must be
+allowed to use the library.  A more frequent case is that a free
+library does the same job as widely used non-free libraries.  In this
+case, there is little to gain by limiting the free library to free
+software only, so we use the Lesser General Public License.
+
+  In other cases, permission to use a particular library in non-free
+programs enables a greater number of people to use a large body of
+free software.  For example, permission to use the GNU C Library in
+non-free programs enables many more people to use the whole GNU
+operating system, as well as its variant, the GNU/Linux operating
+system.
+
+  Although the Lesser General Public License is Less protective of the
+users' freedom, it does ensure that the user of a program that is
+linked with the Library has the freedom and the wherewithal to run
+that program using a modified version of the Library.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.  Pay close attention to the difference between a
+"work based on the library" and a "work that uses the library".  The
+former contains code derived from the library, whereas the latter must
+be combined with the library in order to run.
+
+		  GNU LESSER GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License Agreement applies to any software library or other
+program which contains a notice placed by the copyright holder or
+other authorized party saying it may be distributed under the terms of
+this Lesser General Public License (also called "this License").
+Each licensee is addressed as "you".
+
+  A "library" means a collection of software functions and/or data
+prepared so as to be conveniently linked with application programs
+(which use some of those functions and data) to form executables.
+
+  The "Library", below, refers to any such software library or work
+which has been distributed under these terms.  A "work based on the
+Library" means either the Library or any derivative work under
+copyright law: that is to say, a work containing the Library or a
+portion of it, either verbatim or with modifications and/or translated
+straightforwardly into another language.  (Hereinafter, translation is
+included without limitation in the term "modification".)
+
+  "Source code" for a work means the preferred form of the work for
+making modifications to it.  For a library, complete source code means
+all the source code for all modules it contains, plus any associated
+interface definition files, plus the scripts used to control compilation
+and installation of the library.
+
+  Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running a program using the Library is not restricted, and output from
+such a program is covered only if its contents constitute a work based
+on the Library (independent of the use of the Library in a tool for
+writing it).  Whether that is true depends on what the Library does
+and what the program that uses the Library does.
+  
+  1. You may copy and distribute verbatim copies of the Library's
+complete source code as you receive it, in any medium, provided that
+you conspicuously and appropriately publish on each copy an
+appropriate copyright notice and disclaimer of warranty; keep intact
+all the notices that refer to this License and to the absence of any
+warranty; and distribute a copy of this License along with the
+Library.
+
+  You may charge a fee for the physical act of transferring a copy,
+and you may at your option offer warranty protection in exchange for a
+fee.
+
+  2. You may modify your copy or copies of the Library or any portion
+of it, thus forming a work based on the Library, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) The modified work must itself be a software library.
+
+    b) You must cause the files modified to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    c) You must cause the whole of the work to be licensed at no
+    charge to all third parties under the terms of this License.
+
+    d) If a facility in the modified Library refers to a function or a
+    table of data to be supplied by an application program that uses
+    the facility, other than as an argument passed when the facility
+    is invoked, then you must make a good faith effort to ensure that,
+    in the event an application does not supply such function or
+    table, the facility still operates, and performs whatever part of
+    its purpose remains meaningful.
+
+    (For example, a function in a library to compute square roots has
+    a purpose that is entirely well-defined independent of the
+    application.  Therefore, Subsection 2d requires that any
+    application-supplied function or table used by this function must
+    be optional: if the application does not supply it, the square
+    root function must still compute square roots.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Library,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Library, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote
+it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Library.
+
+In addition, mere aggregation of another work not based on the Library
+with the Library (or with a work based on the Library) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may opt to apply the terms of the ordinary GNU General Public
+License instead of this License to a given copy of the Library.  To do
+this, you must alter all the notices that refer to this License, so
+that they refer to the ordinary GNU General Public License, version 2,
+instead of to this License.  (If a newer version than version 2 of the
+ordinary GNU General Public License has appeared, then you can specify
+that version instead if you wish.)  Do not make any other change in
+these notices.
+
+  Once this change is made in a given copy, it is irreversible for
+that copy, so the ordinary GNU General Public License applies to all
+subsequent copies and derivative works made from that copy.
+
+  This option is useful when you wish to copy part of the code of
+the Library into a program that is not a library.
+
+  4. You may copy and distribute the Library (or a portion or
+derivative of it, under Section 2) in object code or executable form
+under the terms of Sections 1 and 2 above provided that you accompany
+it with the complete corresponding machine-readable source code, which
+must be distributed under the terms of Sections 1 and 2 above on a
+medium customarily used for software interchange.
+
+  If distribution of object code is made by offering access to copy
+from a designated place, then offering equivalent access to copy the
+source code from the same place satisfies the requirement to
+distribute the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  5. A program that contains no derivative of any portion of the
+Library, but is designed to work with the Library by being compiled or
+linked with it, is called a "work that uses the Library".  Such a
+work, in isolation, is not a derivative work of the Library, and
+therefore falls outside the scope of this License.
+
+  However, linking a "work that uses the Library" with the Library
+creates an executable that is a derivative of the Library (because it
+contains portions of the Library), rather than a "work that uses the
+library".  The executable is therefore covered by this License.
+Section 6 states terms for distribution of such executables.
+
+  When a "work that uses the Library" uses material from a header file
+that is part of the Library, the object code for the work may be a
+derivative work of the Library even though the source code is not.
+Whether this is true is especially significant if the work can be
+linked without the Library, or if the work is itself a library.  The
+threshold for this to be true is not precisely defined by law.
+
+  If such an object file uses only numerical parameters, data
+structure layouts and accessors, and small macros and small inline
+functions (ten lines or less in length), then the use of the object
+file is unrestricted, regardless of whether it is legally a derivative
+work.  (Executables containing this object code plus portions of the
+Library will still fall under Section 6.)
+
+  Otherwise, if the work is a derivative of the Library, you may
+distribute the object code for the work under the terms of Section 6.
+Any executables containing that work also fall under Section 6,
+whether or not they are linked directly with the Library itself.
+
+  6. As an exception to the Sections above, you may also combine or
+link a "work that uses the Library" with the Library to produce a
+work containing portions of the Library, and distribute that work
+under terms of your choice, provided that the terms permit
+modification of the work for the customer's own use and reverse
+engineering for debugging such modifications.
+
+  You must give prominent notice with each copy of the work that the
+Library is used in it and that the Library and its use are covered by
+this License.  You must supply a copy of this License.  If the work
+during execution displays copyright notices, you must include the
+copyright notice for the Library among them, as well as a reference
+directing the user to the copy of this License.  Also, you must do one
+of these things:
+
+    a) Accompany the work with the complete corresponding
+    machine-readable source code for the Library including whatever
+    changes were used in the work (which must be distributed under
+    Sections 1 and 2 above); and, if the work is an executable linked
+    with the Library, with the complete machine-readable "work that
+    uses the Library", as object code and/or source code, so that the
+    user can modify the Library and then relink to produce a modified
+    executable containing the modified Library.  (It is understood
+    that the user who changes the contents of definitions files in the
+    Library will not necessarily be able to recompile the application
+    to use the modified definitions.)
+
+    b) Use a suitable shared library mechanism for linking with the
+    Library.  A suitable mechanism is one that (1) uses at run time a
+    copy of the library already present on the user's computer system,
+    rather than copying library functions into the executable, and (2)
+    will operate properly with a modified version of the library, if
+    the user installs one, as long as the modified version is
+    interface-compatible with the version that the work was made with.
+
+    c) Accompany the work with a written offer, valid for at
+    least three years, to give the same user the materials
+    specified in Subsection 6a, above, for a charge no more
+    than the cost of performing this distribution.
+
+    d) If distribution of the work is made by offering access to copy
+    from a designated place, offer equivalent access to copy the above
+    specified materials from the same place.
+
+    e) Verify that the user has already received a copy of these
+    materials or that you have already sent this user a copy.
+
+  For an executable, the required form of the "work that uses the
+Library" must include any data and utility programs needed for
+reproducing the executable from it.  However, as a special exception,
+the materials to be distributed need not include anything that is
+normally distributed (in either source or binary form) with the major
+components (compiler, kernel, and so on) of the operating system on
+which the executable runs, unless that component itself accompanies
+the executable.
+
+  It may happen that this requirement contradicts the license
+restrictions of other proprietary libraries that do not normally
+accompany the operating system.  Such a contradiction means you cannot
+use both them and the Library together in an executable that you
+distribute.
+
+  7. You may place library facilities that are a work based on the
+Library side-by-side in a single library together with other library
+facilities not covered by this License, and distribute such a combined
+library, provided that the separate distribution of the work based on
+the Library and of the other library facilities is otherwise
+permitted, and provided that you do these two things:
+
+    a) Accompany the combined library with a copy of the same work
+    based on the Library, uncombined with any other library
+    facilities.  This must be distributed under the terms of the
+    Sections above.
+
+    b) Give prominent notice with the combined library of the fact
+    that part of it is a work based on the Library, and explaining
+    where to find the accompanying uncombined form of the same work.
+
+  8. You may not copy, modify, sublicense, link with, or distribute
+the Library except as expressly provided under this License.  Any
+attempt otherwise to copy, modify, sublicense, link with, or
+distribute the Library is void, and will automatically terminate your
+rights under this License.  However, parties who have received copies,
+or rights, from you under this License will not have their licenses
+terminated so long as such parties remain in full compliance.
+
+  9. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Library or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Library (or any work based on the
+Library), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Library or works based on it.
+
+  10. Each time you redistribute the Library (or any work based on the
+Library), the recipient automatically receives a license from the
+original licensor to copy, distribute, link with or modify the Library
+subject to these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties with
+this License.
+
+  11. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Library at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Library by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Library.
+
+If any portion of this section is held invalid or unenforceable under any
+particular circumstance, the balance of the section is intended to apply,
+and the section as a whole is intended to apply in other circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  12. If the distribution and/or use of the Library is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Library under this License may add
+an explicit geographical distribution limitation excluding those countries,
+so that distribution is permitted only in or among countries not thus
+excluded.  In such case, this License incorporates the limitation as if
+written in the body of this License.
+
+  13. The Free Software Foundation may publish revised and/or new
+versions of the Lesser General Public License from time to time.
+Such new versions will be similar in spirit to the present version,
+but may differ in detail to address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Library
+specifies a version number of this License which applies to it and
+"any later version", you have the option of following the terms and
+conditions either of that version or of any later version published by
+the Free Software Foundation.  If the Library does not specify a
+license version number, you may choose any version ever published by
+the Free Software Foundation.
+
+  14. If you wish to incorporate parts of the Library into other free
+programs whose distribution conditions are incompatible with these,
+write to the author to ask for permission.  For software which is
+copyrighted by the Free Software Foundation, write to the Free
+Software Foundation; we sometimes make exceptions for this.  Our
+decision will be guided by the two goals of preserving the free status
+of all derivatives of our free software and of promoting the sharing
+and reuse of software generally.
+
+			    NO WARRANTY
+
+  15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO
+WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW.
+EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR
+OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY
+KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE
+LIBRARY IS WITH YOU.  SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME
+THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN
+WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY
+AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU
+FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR
+CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE
+LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING
+RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A
+FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF
+SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
+DAMAGES.
+
+		     END OF TERMS AND CONDITIONS
+
+           How to Apply These Terms to Your New Libraries
+
+  If you develop a new library, and you want it to be of the greatest
+possible use to the public, we recommend making it free software that
+everyone can redistribute and change.  You can do so by permitting
+redistribution under these terms (or, alternatively, under the terms of the
+ordinary General Public License).
+
+  To apply these terms, attach the following notices to the library.  It is
+safest to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least the
+"copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the library's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This library is free software; you can redistribute it and/or
+    modify it under the terms of the GNU Lesser General Public
+    License as published by the Free Software Foundation; either
+    version 2.1 of the License, or (at your option) any later version.
+
+    This library is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+    Lesser General Public License for more details.
+
+    You should have received a copy of the GNU Lesser General Public
+    License along with this library; if not, write to the Free Software
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+Also add information on how to contact you by electronic and paper mail.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the library, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the
+  library `Frob' (a library for tweaking knobs) written by James Random Hacker.
+
+  <signature of Ty Coon>, 1 April 1990
+  Ty Coon, President of Vice
+
+That's all there is to it!
+
diff --git a/SCR.txt b/SCR.txt
new file mode 100644
index 000000000..d14157492
--- /dev/null
+++ b/SCR.txt
@@ -0,0 +1,9 @@
+Package:                     gst-plugins-base.git
+Version:                     1.24.7.imx
+Outgoing License:            LGPL-2.1
+License File:                LICENSE.txt
+Type of Content:             source
+Description and comments:    Open Source Multimedia Farmework
+Release Location:            https://github.com/nxp-imx/gstreamer -b lf-6.6.36-2.1.0
+Origin:                      NXP (LGPL-2.1)
+                             GStreamer (LGPL-2.1+) - http://gstreamer.freedesktop.org/src/gstreamer/
-- 
2.34.1


From 1693401ea5b80f67bfb88beefb3978f7d269d8bd Mon Sep 17 00:00:00 2001
From: Elliot Chen <elliot.chen@nxp.com>
Date: Fri, 27 Sep 2024 18:09:32 +0900
Subject: [PATCH 87/93] MMFMWK-9424 decodebin3: Need check and send selected
 stream messsage even if no decoder is selected

upstream status: pending
https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/7579

Signed-off-by: Elliot Chen <elliot.chen@nxp.com>
---
 gst/playback/gstdecodebin3.c | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/gst/playback/gstdecodebin3.c b/gst/playback/gstdecodebin3.c
index 26588bcf5..87fa5142c 100644
--- a/gst/playback/gstdecodebin3.c
+++ b/gst/playback/gstdecodebin3.c
@@ -3328,20 +3328,22 @@ mq_slot_check_reconfiguration (MultiQueueSlot * slot)
     SELECTION_UNLOCK (dbin);
     if (msg)
       gst_element_post_message ((GstElement *) slot->dbin, msg);
-    if (no_more_streams)
+    if (no_more_streams) {
       GST_ELEMENT_ERROR (slot->dbin, CORE, MISSING_PLUGIN, (NULL),
           ("No suitable plugins found"));
-    else
+      return;
+    } else
       GST_ELEMENT_WARNING (slot->dbin, CORE, MISSING_PLUGIN, (NULL),
           ("Some plugins were missing"));
-  } else {
-    GstMessage *selection_msg = is_selection_done (dbin);
-    /* All good, we reconfigured the associated output. Check if we're done with
-     * the current selection */
-    SELECTION_UNLOCK (dbin);
-    if (selection_msg)
-      gst_element_post_message ((GstElement *) slot->dbin, selection_msg);
+    SELECTION_LOCK (dbin);
   }
+
+  GstMessage *selection_msg = is_selection_done (dbin);
+  /* We reconfigured the associated output. Check if we're done with
+    * the current selection */
+  SELECTION_UNLOCK (dbin);
+  if (selection_msg)
+    gst_element_post_message ((GstElement *) slot->dbin, selection_msg);
 }
 
 static void
-- 
2.34.1


From 44e9a2731d9fdfe2b3da3b03398956a83235c183 Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Tue, 8 Oct 2024 13:48:48 +0800
Subject: [PATCH 88/93] MMFMWK-9426 decodebin3: fix rtsp stream random playback
 fail with "No suitable plugins found"

1.24.7 performs stream_reconfigure in advance if all streams are
present for that collection during STREAM_START. rtsp stream has
respective collection for video and audio, if do stream_recofigure
of stream-start before receiving caps event, decodebin3 cannot find
decoder for "application/x-rtp" then report missing plugin error.

For collection that has only one stream, make a workaround to let
decodebin3 fallback to do stream_recofigure after receiving caps
event like 1.24.0.

Issue: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/3859
---
 gst/playback/gstdecodebin3.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gst/playback/gstdecodebin3.c b/gst/playback/gstdecodebin3.c
index 87fa5142c..d5640cbdf 100644
--- a/gst/playback/gstdecodebin3.c
+++ b/gst/playback/gstdecodebin3.c
@@ -3356,7 +3356,7 @@ update_stream_presence (GstDecodebin3 * dbin, DecodebinCollection * collection)
     return;
   }
 
-  if (g_list_length (dbin->slots) !=
+  if (g_list_length (dbin->slots) == 1 || g_list_length (dbin->slots) !=
       gst_stream_collection_get_size (collection->collection)) {
     collection->all_streams_present = FALSE;
     return;
-- 
2.34.1


From af8e37b8e37aac390ab83d211dcba0a0807b2275 Mon Sep 17 00:00:00 2001
From: Chao Guo <chao.guo@nxp.com>
Date: Wed, 11 Sep 2024 17:19:14 +0900
Subject: [PATCH 89/93] LF-13361 gldownload: add alignment in allocation
 bufferpool

To meet the alignment requirement of GPU driver, add alignment
for bufferpool with some special sizes when gldownload propose
allocation.

Besides, in order for downstream elements to receive the correct
buffer size and stride, prioritize using the input buffer videometa
to set the output videometa.

Upstream Status: Inappropriate [i.MX specific]

Signed-off-by: Chao Guo <chao.guo@nxp.com>
---
 ext/gl/gstgldownloadelement.c    | 28 ++++++++++++++++++++++++++++
 gst-libs/gst/gl/gstglmemorydma.c | 25 ++++++++++++++++++++++---
 2 files changed, 50 insertions(+), 3 deletions(-)

diff --git a/ext/gl/gstgldownloadelement.c b/ext/gl/gstgldownloadelement.c
index 7ad9efcaf..33a2619df 100644
--- a/ext/gl/gstgldownloadelement.c
+++ b/ext/gl/gstgldownloadelement.c
@@ -829,6 +829,12 @@ static void gst_gl_download_element_finalize (GObject * object);
 #define EXTRA_CAPS_TEMPLATE2
 #endif
 
+#if GST_GL_HAVE_VIV_DIRECTVIV
+#define DEFAULT_ALIGN 16
+#else
+#define DEFAULT_ALIGN 32
+#endif
+
 static GstStaticPadTemplate gst_gl_download_element_src_pad_template =
     GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
@@ -1423,6 +1429,7 @@ gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
   GstStructure *config;
   gsize size;
   GstVideoFormat fmt;
+  gboolean need_alignment = FALSE;
 
   gst_query_parse_allocation (query, &caps, NULL);
   if (!gst_video_info_from_caps (&info, caps)) {
@@ -1439,6 +1446,8 @@ gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
 #if GST_GL_HAVE_IONDMA || GST_GL_HAVE_DMABUFHEAPS
   if (fmt == GST_VIDEO_FORMAT_RGBA || fmt == GST_VIDEO_FORMAT_RGB16) {
     allocator = gst_gl_memory_dma_allocator_obtain ();
+    if (allocator)
+      need_alignment = TRUE;
     GST_DEBUG_OBJECT (bt, "obtain dma memory allocator %p.", allocator);
   }
 #endif
@@ -1478,6 +1487,25 @@ gst_gl_download_element_propose_allocation (GstBaseTransform * bt,
   gst_buffer_pool_config_add_option (config,
       GST_BUFFER_POOL_OPTION_GL_SYNC_META);
 
+  if (need_alignment) {
+    guint width, height;
+    GstVideoAlignment alignment;
+    width = GST_VIDEO_INFO_WIDTH (&info);
+    height = GST_VIDEO_INFO_HEIGHT (&info);
+
+    // add alignment in config when using dma allocator
+    memset (&alignment, 0, sizeof (GstVideoAlignment));
+    alignment.padding_right = GST_ROUND_UP_N (width, DEFAULT_ALIGN) - width;
+    alignment.padding_bottom = GST_ROUND_UP_N (height, DEFAULT_ALIGN) - height;
+    gst_buffer_pool_config_add_option (config,
+        GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
+    gst_buffer_pool_config_set_video_alignment (config, &alignment);
+
+    // add video meta option to get alignment from buffer.
+    gst_buffer_pool_config_add_option (config,
+        GST_BUFFER_POOL_OPTION_VIDEO_META);
+  }
+
   //set allocator to buffer pool
   if (allocator) {
     gst_buffer_pool_config_set_allocator (config, allocator, &params);
diff --git a/gst-libs/gst/gl/gstglmemorydma.c b/gst-libs/gst/gl/gstglmemorydma.c
index 3160cb2fd..ceee2af42 100644
--- a/gst-libs/gst/gl/gstglmemorydma.c
+++ b/gst-libs/gst/gl/gstglmemorydma.c
@@ -257,19 +257,38 @@ gst_gl_memory_dma_buffer_to_gstbuffer (GstGLContext * ctx, GstVideoInfo * info,
 {
   GstBuffer *buf;
   GstGLMemoryDMA *glmem;
+  GstVideoMeta *vmeta;
+  GstVideoMeta *ometa;
+  guint width = 0, height = 0;
+  const gsize *offsets = NULL;
+  const gint *strides = NULL;
 
   gst_gl_context_thread_add (ctx, (GstGLContextThreadFunc) _finish_texture,
       NULL);
 
+  vmeta = gst_buffer_get_video_meta (glbuf);
+  if (vmeta) {
+    width = vmeta->width;
+    height = vmeta->height;
+    offsets = vmeta->offset;
+    strides = vmeta->stride;
+  } else {
+    width = GST_VIDEO_INFO_WIDTH (info);
+    height = GST_VIDEO_INFO_HEIGHT (info);
+    offsets = info->offset;
+    strides = info->stride;
+  }
+
   glmem = (GstGLMemoryDMA *) gst_buffer_peek_memory (glbuf, 0);
 
   buf = gst_buffer_new ();
   gst_buffer_append_memory (buf, (GstMemory *) glmem->dma);
   gst_memory_ref ((GstMemory *) glmem->dma);
 
-  gst_buffer_add_video_meta_full (buf, 0,
-      GST_VIDEO_INFO_FORMAT (info), GST_VIDEO_INFO_WIDTH (info),
-      GST_VIDEO_INFO_HEIGHT (info), 1, info->offset, info->stride);
+  ometa = gst_buffer_add_video_meta_full (buf, 0,
+      GST_VIDEO_INFO_FORMAT (info), width, height, 1, offsets, strides);
+  if (vmeta)
+    gst_video_meta_set_alignment (ometa, vmeta->alignment);
   GST_BUFFER_FLAGS (buf) = GST_BUFFER_FLAGS (glbuf);
   GST_BUFFER_PTS (buf) = GST_BUFFER_PTS (glbuf);
   GST_BUFFER_DTS (buf) = GST_BUFFER_DTS (glbuf);
-- 
2.34.1


From 4dcd88935e9c22bd500f366b72fc7cb5b7079687 Mon Sep 17 00:00:00 2001
From: Chao Guo <chao.guo@nxp.com>
Date: Sat, 12 Oct 2024 11:13:31 +0900
Subject: [PATCH 90/93] LF-13361 glcolorconvert: Avoid copying texture to
 another texture bound to dmabuf on GLES2 platforms

In GLES2 platform, out size and mem size may be different, causing memory copy after converting.
For DMA memory, skip copying, because glCopyTexImage2D() cannot copy texture to dmabuf.

Upstream Status: Inappropriate [i.MX specific]

Signed-off-by: Chao Guo <chao.guo@nxp.com>
---
 gst-libs/gst/gl/gstglcolorconvert.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/gst-libs/gst/gl/gstglcolorconvert.c b/gst-libs/gst/gl/gstglcolorconvert.c
index e1d8b13c8..59e813211 100644
--- a/gst-libs/gst/gl/gstglcolorconvert.c
+++ b/gst-libs/gst/gl/gstglcolorconvert.c
@@ -31,6 +31,10 @@
 #include "gstglfuncs.h"
 #include "gstglsl_private.h"
 
+#if GST_GL_HAVE_IONDMA || GST_GL_HAVE_DMABUFHEAPS
+#include <gst/gl/gstglmemorydma.h>
+#endif
+
 /**
  * SECTION:gstglcolorconvert
  * @title: GstGLColorConvert
@@ -2740,12 +2744,22 @@ _do_convert_one_view (GstGLContext * context, GstGLColorConvert * convert,
   gint i, j = 0;
   const gint in_plane_offset = view_num * c_info->in_n_textures;
   const gint out_plane_offset = view_num * c_info->out_n_textures;
+  gboolean can_copy = TRUE;
 
   out_width = GST_VIDEO_INFO_WIDTH (&convert->out_info);
   out_height = GST_VIDEO_INFO_HEIGHT (&convert->out_info);
   in_width = GST_VIDEO_INFO_WIDTH (&convert->in_info);
   in_height = GST_VIDEO_INFO_HEIGHT (&convert->in_info);
 
+#if GST_GL_HAVE_IONDMA || GST_GL_HAVE_DMABUFHEAPS
+  /* In GLES2 platform, out size and mem size may be different,
+   * causing memory copy after converting. For DMA memory, skip
+   * copying, because glCopyTexImage2D() cannot copy texture to dmabuf. */
+  can_copy = USING_GLES3 (context) ||
+      !gst_is_gl_memory_dma ((GstMemory *) gst_buffer_peek_memory (convert->outbuf,
+                                                                   out_plane_offset));
+#endif
+
   for (i = 0; i < c_info->in_n_textures; i++) {
     convert->priv->in_tex[i] =
         (GstGLMemory *) gst_buffer_peek_memory (convert->inbuf,
@@ -2796,7 +2810,7 @@ _do_convert_one_view (GstGLContext * context, GstGLColorConvert * convert,
 
     if (out_tex->tex_format == GST_GL_LUMINANCE
         || out_tex->tex_format == GST_GL_LUMINANCE_ALPHA
-        || out_width != mem_width || out_height != mem_height) {
+        || ((out_width != mem_width || out_height != mem_height) && can_copy)) {
       /* Luminance formats are not color renderable */
       /* rendering to a framebuffer only renders the intersection of all
        * the attachments i.e. the smallest attachment size */
@@ -2868,7 +2882,7 @@ out:
 
     if (out_tex->tex_format == GST_GL_LUMINANCE
         || out_tex->tex_format == GST_GL_LUMINANCE_ALPHA
-        || out_width != mem_width || out_height != mem_height) {
+        || ((out_width != mem_width || out_height != mem_height) && can_copy)) {
       GstMapInfo to_info, from_info;
 
       if (!gst_memory_map ((GstMemory *) convert->priv->out_tex[j], &from_info,
-- 
2.34.1


From 572a02c736d3f9107920b89ba2aa09ebf8babc7f Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Thu, 17 Oct 2024 13:56:21 +0800
Subject: [PATCH 91/93] Update release branch in SCR.txt to lf-6.6.52-2.2.0

---
 SCR.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/SCR.txt b/SCR.txt
index d14157492..3c93f5859 100644
--- a/SCR.txt
+++ b/SCR.txt
@@ -4,6 +4,6 @@ Outgoing License:            LGPL-2.1
 License File:                LICENSE.txt
 Type of Content:             source
 Description and comments:    Open Source Multimedia Farmework
-Release Location:            https://github.com/nxp-imx/gstreamer -b lf-6.6.36-2.1.0
+Release Location:            https://github.com/nxp-imx/gstreamer -b lf-6.6.52-2.2.0
 Origin:                      NXP (LGPL-2.1)
                              GStreamer (LGPL-2.1+) - http://gstreamer.freedesktop.org/src/gstreamer/
-- 
2.34.1


From 2a74e6e77ab2f081bebbae1ab68b52c2596edddc Mon Sep 17 00:00:00 2001
From: Hou Qi <qi.hou@nxp.com>
Date: Thu, 17 Oct 2024 15:27:49 +0800
Subject: [PATCH 92/93] MMFMWK-9433 decodebin3: workaround to fix try next
 decoder fail

Owing to decodebin3 time sequence in 1.24.7, trying decoder may comes
before multiqueue probes caps event. Decoder has no chance to check
whether set_format() success or not as it's triggered by caps events.

For streams that decoder needs fallback like interleaved streams on
imx95, decodebin3 will wrongly selects v4l2 decoder, finally report
error and playback fail.

So make a workaround to setup decoder only after receiving caps event.
---
 gst/playback/gstdecodebin3.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/gst/playback/gstdecodebin3.c b/gst/playback/gstdecodebin3.c
index d5640cbdf..332e2501b 100644
--- a/gst/playback/gstdecodebin3.c
+++ b/gst/playback/gstdecodebin3.c
@@ -436,6 +436,9 @@ typedef struct _MultiQueueSlot
   gboolean is_drained;
 
   DecodebinOutputStream *output;
+
+  /* TRUE if multiqueue probes CAPS event */
+  gboolean probed_caps;
 } MultiQueueSlot;
 
 /* Streams that are exposed downstream (i.e. output) */
@@ -3535,6 +3538,7 @@ multiqueue_src_probe (GstPad * pad, GstPadProbeInfo * info,
         break;
       case GST_EVENT_CAPS:
       {
+        slot->probed_caps = TRUE;
         /* Configure the output slot if needed */
         mq_slot_check_reconfiguration (slot);
       }
@@ -3649,6 +3653,7 @@ create_new_slot (GstDecodebin3 * dbin, GstStreamType type)
       gst_stream_type_get_name (type));
   slot = g_new0 (MultiQueueSlot, 1);
   slot->dbin = dbin;
+  slot->probed_caps = FALSE;
 
   slot->id = dbin->slot_id++;
 
@@ -4171,7 +4176,8 @@ db_output_stream_reconfigure (DecodebinOutputStream * output, GstMessage ** msg)
     db_output_stream_reset (output);
 
     /* Setup the decoder */
-    ret = db_output_stream_setup_decoder (output, new_caps, msg);
+    if (slot->probed_caps)
+      ret = db_output_stream_setup_decoder (output, new_caps, msg);
   }
 
   gst_caps_unref (new_caps);
-- 
2.34.1


From 031d262e65333e58b2ce01aaf47ab1012f93b21e Mon Sep 17 00:00:00 2001
From: Chao Guo <chao.guo@nxp.com>
Date: Tue, 24 Sep 2024 15:12:05 +0800
Subject: [PATCH 93/93] MMFMWK-9419 glupload: Use info in in_caps to map video
 frames When copying input buffer to dambuf

When copying the input buffer to glupload dmabuf->inbuf, use the video info from in_caps to map video
frame. Because the data in in_info will be replaced by the meta data from dmabuf->inbuf after copying.
The in_info will not match the input buffer in next copy, causing core dump.

Upstream Status: Inappropriate [i.MX specific]

Signed-off-by: Chao Guo <chao.guo@nxp.com>
---
 gst-libs/gst/gl/gstglupload.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gst-libs/gst/gl/gstglupload.c b/gst-libs/gst/gl/gstglupload.c
index 3a88ac682..0f890be17 100644
--- a/gst-libs/gst/gl/gstglupload.c
+++ b/gst-libs/gst/gl/gstglupload.c
@@ -1606,8 +1606,10 @@ _dma_buf_upload_accept (gpointer impl, GstBuffer * buffer, GstCaps * in_caps,
   /* This will eliminate most non-dmabuf out there */
   if (!gst_is_dmabuf_memory (gst_buffer_peek_memory (buffer, 0))) {
     GstVideoFrame frame1, frame2;
+    GstVideoInfo map_in_info;
 
-    gst_video_frame_map (&frame1, in_info, buffer, GST_MAP_READ);
+    gst_video_info_from_caps (&map_in_info, in_caps);
+    gst_video_frame_map (&frame1, &map_in_info, buffer, GST_MAP_READ);
 
     if (!dmabuf->pool) {
       gboolean ret;
-- 
2.34.1