diff --git a/libswscale/format.c b/libswscale/format.c
index eb56994beb..017df9f69a 100644
--- a/libswscale/format.c
+++ b/libswscale/format.c
@@ -936,7 +936,8 @@ int ff_sws_decode_pixfmt(SwsOpList *ops, enum AVPixelFormat fmt)
 
     /* Set baseline pixel content flags */
     const int integer = ff_sws_pixel_type_is_int(raw_type);
-    const int swapped = (desc->flags & AV_PIX_FMT_FLAG_BE) != NATIVE_ENDIAN_FLAG;
+    const int swapped = ff_sws_pixel_type_size(raw_type) > 1 &&
+                        (desc->flags & AV_PIX_FMT_FLAG_BE) != NATIVE_ENDIAN_FLAG;
     for (int i = 0; i < rw_op.elems; i++) {
         comps->flags[i] = (integer ? SWS_COMP_EXACT   : 0) |
                           (swapped ? SWS_COMP_SWAPPED : 0);
@@ -1057,7 +1058,8 @@ int ff_sws_encode_pixfmt(SwsOpList *ops, enum AVPixelFormat fmt)
         }));
     }
 
-    if ((desc->flags & AV_PIX_FMT_FLAG_BE) != NATIVE_ENDIAN_FLAG) {
+    if (ff_sws_pixel_type_size(raw_type) > 1 &&
+        (desc->flags & AV_PIX_FMT_FLAG_BE) != NATIVE_ENDIAN_FLAG) {
         RET(ff_sws_op_list_append(ops, &(SwsOp) {
             .op   = SWS_OP_SWAP_BYTES,
             .type = raw_type,