swscale/ops: add explicit row offset to SwsDitherOp

To improve decorrelation between components, we offset the dither matrix
slightly for each component. This is currently done by adding a hard-coded
offset of {0, 3, 2, 5} to each of the four components, respectively.

However, this represents a serious challenge when re-ordering SwsDitherOp
past a swizzle, or when splitting an SwsOpList into multiple sub-operations
(e.g. for decoupling luma from subsampled chroma when they are independent).

To fix this on a fundamental level, we have to keep track of the offset per
channel as part of the SwsDitherOp metadata, and respect those values at
runtime.

This commit merely adds the metadata; the update to the underlying backends
will come in a follow-up commit. The FATE change is merely due to the
added offsets in the op list print-out.

libswscale/format.c

@@ -1195,6 +1195,13 @@ static int fmt_dither(SwsContext *ctx, SwsOpList *ops,
         dither.matrix = generate_bayer_matrix(dither.size_log2);
         if (!dither.matrix)
             return AVERROR(ENOMEM);
+
+        /* Brute-forced offsets; minimizes quantization error across a 16x16
+         * bayer dither pattern for standard RGBA and YUVA pixel formats */
+        const int offsets_16x16[4] = {0, 3, 2, 5};
+        for (int i = 0; i < 4; i++)
+            dither.y_offset[i] = offsets_16x16[i];
+
         return ff_sws_op_list_append(ops, &(SwsOp) {
             .op     = SWS_OP_DITHER,
             .type   = type,

libswscale/ops.c

@@ -460,8 +460,10 @@ void ff_sws_op_list_print(void *log, int lev, const SwsOpList *ops)
                    op->convert.expand ? " (expand)" : "");
             break;
         case SWS_OP_DITHER:
-            av_log(log, lev, "%-20s: %dx%d matrix\n", "SWS_OP_DITHER",
-                    1 << op->dither.size_log2, 1 << op->dither.size_log2);
+            av_log(log, lev, "%-20s: %dx%d matrix + {%d %d %d %d}\n", "SWS_OP_DITHER",
+                    1 << op->dither.size_log2, 1 << op->dither.size_log2,
+                    op->dither.y_offset[0], op->dither.y_offset[1],
+                    op->dither.y_offset[2], op->dither.y_offset[3]);
             break;
         case SWS_OP_MIN:
             av_log(log, lev, "%-20s: x <= {%s %s %s %s}\n", "SWS_OP_MIN",

libswscale/ops.h

@@ -132,6 +132,7 @@ typedef struct SwsConvertOp {
 typedef struct SwsDitherOp {
     AVRational *matrix; /* tightly packed dither matrix (refstruct) */
     int size_log2; /* size (in bits) of the dither matrix */
+    uint8_t y_offset[4]; /* row offset for each component */
 } SwsDitherOp;
 
 typedef struct SwsLinearOp {

tests/checkasm/sw_ops.c

@@ -624,6 +624,7 @@ static void check_dither(void)
         /* Test all sizes up to 256x256 */
         for (int size_log2 = 0; size_log2 <= 8; size_log2++) {
             const int size = 1 << size_log2;
+            const int mask = size - 1;
             AVRational *matrix = av_refstruct_allocz(size * size * sizeof(*matrix));
             if (!matrix) {
                 fail();
@@ -641,7 +642,8 @@ static void check_dither(void)
                 .op = SWS_OP_DITHER,
                 .type = t,
                 .dither.size_log2 = size_log2,
-                .dither.matrix = matrix,
+                .dither.matrix    = matrix,
+                .dither.y_offset  = {0, 3 & mask, 2 & mask, 5 & mask},
             });
 
             av_refstruct_unref(&matrix);

tests/ref/fate/sws-ops-list

@@ -1 +1 @@
-eae3a49ac3af42c13ad274883611ac21
+0f38d0a1cb1f9367352c92b23bcb954e