swscale/format: don't add chroma noise when dithering grayscale content

On the surface, this trades a tiny bit of PSNR for not introducing chroma
noise into grayscale images. However, the main reason for this change is
actually motivated by a desire to avoid regressing the status quo of
duplicating swizzles being able to be commuted past dither ops.

libswscale/format.c

@@ -1162,11 +1162,11 @@ static int fmt_dither(SwsContext *ctx, SwsOpList *ops,
 {
     SwsDither mode = ctx->dither;
     SwsDitherOp dither;
+    const int bpc = dst.desc->comp[0].depth;
 
     if (mode == SWS_DITHER_AUTO) {
         /* Visual threshold of perception: 12 bits for SDR, 14 bits for HDR */
         const int jnd_bits = trc_is_hdr(dst.color.trc) ? 14 : 12;
-        const int bpc = dst.desc->comp[0].depth;
         mode = bpc >= jnd_bits ? SWS_DITHER_NONE : SWS_DITHER_BAYER;
     }
 
@@ -1203,6 +1203,21 @@ static int fmt_dither(SwsContext *ctx, SwsOpList *ops,
         for (int i = 0; i < 4; i++)
             dither.y_offset[i] = offsets_16x16[i];
 
+        if (src.desc->nb_components < 3 && bpc >= 8) {
+            /**
+             * For high-bit-depth sources without chroma, use same matrix
+             * offset for all color channels. This prevents introducing color
+             * noise in grayscale images; and also allows optimizing the dither
+             * operation. Skipped for low bit depth (<8 bpc) as the loss in
+             * PSNR, from the inability to diffuse error among all three
+             * channels, can be substantial.
+             *
+             * This shifts: { X, Y, Z, W } -> { X, X, X, Y }
+             */
+            dither.y_offset[3] = dither.y_offset[1];
+            dither.y_offset[1] = dither.y_offset[2] = dither.y_offset[0];
+        }
+
         return ff_sws_op_list_append(ops, &(SwsOp) {
             .op     = SWS_OP_DITHER,
             .type   = type,

tests/ref/fate/sws-ops-list

@@ -1 +1 @@
-0f38d0a1cb1f9367352c92b23bcb954e
+0c7f5082617b0b4b83111b67bec74f2d