avformat/nal: remove trailing zeroes from NALUs

Based on the behaviour from cbs_h2645, which removes actual
trailing_zero_8bits bytes and possibly also work arounds issues in
ff_h2645_extract_rbsp(). In this case, the same issue could be
present in ff_nal_find_startcode().

Signed-off-by: James Almer <jamrial@gmail.com>

libavformat/nal.c

@@ -88,8 +88,11 @@ static int nal_parse_units(AVIOContext *pb, NALUList *list,
 
         nal_end = ff_nal_find_startcode(nal_start, end);
         if (pb) {
-            avio_wb32(pb, nal_end - nal_start);
-            avio_write(pb, nal_start, nal_end - nal_start);
+            ptrdiff_t nalu_size = nal_end - nal_start;
+            while (nalu_size > 0 && nal_start[nalu_size - 1] == 0)
+                --nalu_size;
+            avio_wb32(pb, nalu_size);
+            avio_write(pb, nal_start, nalu_size);
         } else if (list->nb_nalus >= nalu_limit) {
             return AVERROR(ERANGE);
         } else {

tests/ref/fate/enhanced-flv-hevc-hdr10

@@ -1,4 +1,4 @@
-#extradata 0:      111, 0x091d1885
+#extradata 0:      110, 0xf07e1884
 #tb 0: 1/1000
 #media_type 0: video
 #codec_id 0: hevc
@@ -17,7 +17,7 @@ best_effort_timestamp=0
 best_effort_timestamp_time=0.000000
 duration=N/A
 duration_time=N/A
-pkt_pos=459
+pkt_pos=458
 pkt_size=77718
 width=1280
 height=720

tests/ref/fate/enhanced-flv-vvc

@@ -1,6 +1,6 @@
-b4074094cb8c5152361a9fab71dca9e9 *tests/data/fate/enhanced-flv-vvc.flv
-90620 tests/data/fate/enhanced-flv-vvc.flv
-#extradata 0:      161, 0x98c53408
+4789b9c0248ed8ec8d63880f1a40c7b7 *tests/data/fate/enhanced-flv-vvc.flv
+90619 tests/data/fate/enhanced-flv-vvc.flv
+#extradata 0:      160, 0x64ae3407
 #tb 0: 1/1000
 #media_type 0: video
 #codec_id 0: vvc

tests/ref/fate/matroska-h264-remux

@@ -1,5 +1,5 @@
-277a08f708965112a7c2fb25d941e68a *tests/data/fate/matroska-h264-remux.matroska
-2036806 tests/data/fate/matroska-h264-remux.matroska
+c96a6298ccfd1fa0b8b1ef38d4244556 *tests/data/fate/matroska-h264-remux.matroska
+2036805 tests/data/fate/matroska-h264-remux.matroska
 #tb 0: 1/25
 #media_type 0: video
 #codec_id 0: rawvideo

tests/ref/lavf-fate/h264.mp4

@@ -1,3 +1,3 @@
-fe299ea5205b71a48281f917b1256a5d *tests/data/lavf-fate/lavf.h264.mp4
-547928 tests/data/lavf-fate/lavf.h264.mp4
+3e4893fa201d74d7b057eae8c4387436 *tests/data/lavf-fate/lavf.h264.mp4
+547927 tests/data/lavf-fate/lavf.h264.mp4
 tests/data/lavf-fate/lavf.h264.mp4 CRC=0x9da2c999

tests/ref/lavf-fate/vvc.mkv

@@ -1,3 +1,3 @@
-088fe4756b99c28031ba41d1a841a1cb *tests/data/lavf-fate/lavf.vvc.mkv
-16351 tests/data/lavf-fate/lavf.vvc.mkv
+88113d8aa98b2cac4a25c7228814ea9a *tests/data/lavf-fate/lavf.vvc.mkv
+16350 tests/data/lavf-fate/lavf.vvc.mkv
 tests/data/lavf-fate/lavf.vvc.mkv CRC=0x895790b1

tests/ref/lavf-fate/vvc.mp4

@@ -1,3 +1,3 @@
-b5d5939890d3246475f990e5ebd58dc8 *tests/data/lavf-fate/lavf.vvc.mp4
-16625 tests/data/lavf-fate/lavf.vvc.mp4
+c6813fd47c553b214fd20bda871211b4 *tests/data/lavf-fate/lavf.vvc.mp4
+16624 tests/data/lavf-fate/lavf.vvc.mp4
 tests/data/lavf-fate/lavf.vvc.mp4 CRC=0x895790b1