swscale/optimizer: promote component swizzles to plane swizzles

In some cases, we can just directly swizzle the order of input/output
planes, rather than applying a swizzle operation on the data itself.

This can eliminate some such swizzle operations entirely, for example
yuv444p -> vuya is now just a read, clear and write.

Results in a lot of simplifications like this:

 rgb24 -> gbrp:
   [ u8 XXXX -> +++X] SWS_OP_READ         : 3 elem(s) packed >> 0
-  [ u8 ...X -> +++X] SWS_OP_SWIZZLE      : 1203
-  [ u8 ...X -> +++X] SWS_OP_WRITE        : 3 elem(s) planar >> 0
+  [ u8 ...X -> +++X] SWS_OP_WRITE        : 3 elem(s) planar >> 0, via {2, 0, 1}

 rgb24 -> gbrap16le:
   [ u8 XXXX -> +++X] SWS_OP_READ         : 3 elem(s) packed >> 0
   [ u8 ...X -> +++X] SWS_OP_CONVERT      : u8 -> u16 (expand)
-  [u16 ...X -> +++X] SWS_OP_SWIZZLE      : 1203
   [u16 ...X -> ++++] SWS_OP_CLEAR        : {_ _ _ 65535}
-  [u16 .... -> ++++] SWS_OP_WRITE        : 4 elem(s) planar >> 0
+  [u16 .... -> ++++] SWS_OP_WRITE        : 4 elem(s) planar >> 0, via {2, 0, 1, 3}

 yuv444p -> vuya:
-  [ u8 XXXX -> +++X] SWS_OP_READ         : 3 elem(s) planar >> 0
-  [ u8 ...X -> +++X] SWS_OP_SWIZZLE      : 2103
+  [ u8 XXXX -> +++X] SWS_OP_READ         : 3 elem(s) planar >> 0, via {2, 1, 0}
   [ u8 ...X -> ++++] SWS_OP_CLEAR        : {_ _ _ 255}
   [ u8 .... -> ++++] SWS_OP_WRITE        : 4 elem(s) packed >> 0

Sponsored-by: Sovereign Tech Fund
Signed-off-by: Niklas Haas <git@haasn.dev>

libswscale/ops_optimizer.c

@@ -405,6 +405,34 @@ retry:
                 ff_sws_op_list_remove_at(ops, n + 1, 1);
                 goto retry;
             }
+
+            /* Swizzle planes instead of components, if possible */
+            if (prev->op == SWS_OP_READ && !prev->rw.packed) {
+                for (int dst = 0; dst < prev->rw.elems; dst++) {
+                    const int src = op->swizzle.in[dst];
+                    if (src > dst && src < prev->rw.elems) {
+                        FFSWAP(int, ops->order_src.in[dst], ops->order_src.in[src]);
+                        for (int i = dst; i < 4; i++) {
+                            if (op->swizzle.in[i] == dst)
+                                op->swizzle.in[i] = src;
+                            else if (op->swizzle.in[i] == src)
+                                op->swizzle.in[i] = dst;
+                        }
+                        goto retry;
+                    }
+                }
+            }
+
+            if (next->op == SWS_OP_WRITE && !next->rw.packed) {
+                for (int dst = 0; dst < next->rw.elems; dst++) {
+                    const int src = op->swizzle.in[dst];
+                    if (src > dst && src < next->rw.elems) {
+                        FFSWAP(int, ops->order_dst.in[dst], ops->order_dst.in[src]);
+                        FFSWAP(int, op->swizzle.in[dst], op->swizzle.in[src]);
+                        goto retry;
+                    }
+                }
+            }
             break;
 
         case SWS_OP_CONVERT:

tests/ref/fate/sws-ops-list

@@ -1 +1 @@
-8312bc72ff9e05a8a6ab8d1c394783d6
+30ceeaa73f093642f28c1f17b3ee4e3e