/** * Copyright (C) 2025 Niklas Haas * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #ifndef SWSCALE_OPS_H #define SWSCALE_OPS_H #include #include #include #include "libavutil/bprint.h" #include "graph.h" #include "filters.h" typedef enum SwsPixelType { SWS_PIXEL_NONE = 0, SWS_PIXEL_U8, SWS_PIXEL_U16, SWS_PIXEL_U32, SWS_PIXEL_F32, SWS_PIXEL_TYPE_NB } SwsPixelType; const char *ff_sws_pixel_type_name(SwsPixelType type); int ff_sws_pixel_type_size(SwsPixelType type) av_const; bool ff_sws_pixel_type_is_int(SwsPixelType type) av_const; typedef enum SwsOpType { SWS_OP_INVALID = 0, /* Defined for all types; but implemented for integers only */ SWS_OP_READ, /* gather raw pixels from planes */ SWS_OP_WRITE, /* write raw pixels to planes */ SWS_OP_SWAP_BYTES, /* swap byte order (for differing endianness) */ SWS_OP_SWIZZLE, /* rearrange channel order, or duplicate channels */ /* Bit manipulation operations. Defined for integers only. */ SWS_OP_UNPACK, /* split tightly packed data into components */ SWS_OP_PACK, /* compress components into tightly packed data */ SWS_OP_LSHIFT, /* logical left shift of raw pixel values */ SWS_OP_RSHIFT, /* right shift of raw pixel values */ /* Generic arithmetic. Defined and implemented for all types */ SWS_OP_CLEAR, /* clear pixel values */ SWS_OP_CONVERT, /* convert (cast) between formats */ SWS_OP_MIN, /* numeric minimum */ SWS_OP_MAX, /* numeric maximum */ SWS_OP_SCALE, /* multiplication by scalar */ /* Floating-point only arithmetic operations. */ SWS_OP_LINEAR, /* generalized linear affine transform */ SWS_OP_DITHER, /* add dithering noise */ /* Filtering operations. Always output floating point. */ SWS_OP_FILTER_H, /* horizontal filtering */ SWS_OP_FILTER_V, /* vertical filtering */ SWS_OP_TYPE_NB, } SwsOpType; const char *ff_sws_op_type_name(SwsOpType op); /** * Bit-mask of components. Exact meaning depends on the usage context. */ typedef uint8_t SwsCompMask; enum { SWS_COMP_NONE = 0, SWS_COMP_ALL = 0xF, #define SWS_COMP(X) (1 << (X)) #define SWS_COMP_TEST(mask, X) (!!((mask) & SWS_COMP(X))) #define SWS_COMP_INV(mask) ((mask) ^ SWS_COMP_ALL) #define SWS_COMP_ELEMS(N) ((1 << (N)) - 1) #define SWS_COMP_MASK(X, Y, Z, W) \ (((X) ? SWS_COMP(0) : 0) | \ ((Y) ? SWS_COMP(1) : 0) | \ ((Z) ? SWS_COMP(2) : 0) | \ ((W) ? SWS_COMP(3) : 0)) }; /* Compute SwsCompMask from values with denominator != 0 */ SwsCompMask ff_sws_comp_mask_q4(const AVRational q[4]); typedef enum SwsCompFlags { SWS_COMP_GARBAGE = 1 << 0, /* contents are undefined / garbage data */ SWS_COMP_EXACT = 1 << 1, /* value is an exact integer */ SWS_COMP_ZERO = 1 << 2, /* known to be a constant zero */ SWS_COMP_SWAPPED = 1 << 3, /* byte order is swapped */ } SwsCompFlags; typedef struct SwsComps { SwsCompFlags flags[4]; /* knowledge about (output) component contents */ /* Keeps track of the known possible value range, or {0, 0} for undefined * or (unknown range) floating point inputs */ AVRational min[4], max[4]; } SwsComps; typedef struct SwsReadWriteOp { /** * Examples: * rgba = 4x u8 packed * yuv444p = 3x u8 * rgb565 = 1x u16 <- use SWS_OP_UNPACK to unpack * monow = 1x u8 (frac 3) * rgb4 = 1x u8 (frac 1) */ uint8_t elems; /* number of elements (of type `op.type`) to read/write */ uint8_t frac; /* fractional pixel step factor (log2) */ bool packed; /* read multiple elements from a single plane */ /** * Filter kernel to apply to each plane while sampling. Currently, only * one shared filter kernel is supported for all planes. (Optional) * * Note: As with SWS_OP_FILTER_*, if a filter kernel is in use, the read * operation will always output floating point values. */ SwsOpType filter; /* some value of SWS_OP_FILTER_* */ SwsFilterWeights *kernel; /* (refstruct) */ } SwsReadWriteOp; typedef struct SwsPackOp { /** * Packed bits are assumed to be LSB-aligned within the underlying * integer type; i.e. (msb) 0 ... X Y Z W (lsb). */ uint8_t pattern[4]; /* bit depth pattern, from MSB to LSB */ } SwsPackOp; typedef struct SwsSwizzleOp { /** * Input component for each output component: * Out[x] := In[swizzle.in[x]] */ union { uint32_t mask; uint8_t in[4]; struct { uint8_t x, y, z, w; }; }; } SwsSwizzleOp; #define SWS_SWIZZLE(X,Y,Z,W) ((SwsSwizzleOp) { .in = {X, Y, Z, W} }) SwsCompMask ff_sws_comp_mask_swizzle(SwsCompMask mask, SwsSwizzleOp swiz); typedef struct SwsShiftOp { uint8_t amount; /* number of bits to shift */ } SwsShiftOp; typedef struct SwsClearOp { SwsCompMask mask; /* mask of components to clear */ AVRational value[4]; /* value to set */ } SwsClearOp; typedef struct SwsConvertOp { SwsPixelType to; /* type of pixel to convert to */ bool expand; /* if true, integers are expanded to the full range */ } SwsConvertOp; typedef struct SwsClampOp { AVRational limit[4]; /* per-component min/max value */ } SwsClampOp; typedef struct SwsScaleOp { AVRational factor; /* scalar multiplication factor */ } SwsScaleOp; typedef struct SwsDitherOp { AVRational *matrix; /* tightly packed dither matrix (refstruct) */ AVRational min, max; /* minimum/maximum value in `matrix` */ int size_log2; /* size (in bits) of the dither matrix */ int8_t y_offset[4]; /* row offset for each component, or -1 for ignored */ } SwsDitherOp; typedef struct SwsLinearOp { /** * Generalized 5x5 affine transformation: * [ Out.x ] = [ A B C D E ] * [ Out.y ] = [ F G H I J ] * [ x y z w 1 ] * [ Out.z ] = [ K L M N O ] * [ Out.w ] = [ P Q R S T ] * * The mask keeps track of which components differ from an identity matrix. * There may be more efficient implementations of particular subsets, for * example the common subset of {A, E, G, J, M, O} can be implemented with * just three fused multiply-add operations. */ AVRational m[4][5]; uint32_t mask; /* m[i][j] <-> 1 << (5 * i + j) */ } SwsLinearOp; #define SWS_MASK(I, J) (1 << (5 * (I) + (J))) #define SWS_MASK_OFF(I) SWS_MASK(I, 4) #define SWS_MASK_ROW(I) (0x1F << (5 * (I))) #define SWS_MASK_COL(J) (0x8421 << J) enum { SWS_MASK_ALL = (1 << 20) - 1, SWS_MASK_LUMA = SWS_MASK(0, 0) | SWS_MASK_OFF(0), SWS_MASK_ALPHA = SWS_MASK(3, 3), SWS_MASK_DIAG3 = SWS_MASK(0, 0) | SWS_MASK(1, 1) | SWS_MASK(2, 2), SWS_MASK_OFF3 = SWS_MASK_OFF(0) | SWS_MASK_OFF(1) | SWS_MASK_OFF(2), SWS_MASK_MAT3 = SWS_MASK(0, 0) | SWS_MASK(0, 1) | SWS_MASK(0, 2) | SWS_MASK(1, 0) | SWS_MASK(1, 1) | SWS_MASK(1, 2) | SWS_MASK(2, 0) | SWS_MASK(2, 1) | SWS_MASK(2, 2), SWS_MASK_DIAG4 = SWS_MASK_DIAG3 | SWS_MASK(3, 3), SWS_MASK_OFF4 = SWS_MASK_OFF3 | SWS_MASK_OFF(3), SWS_MASK_MAT4 = SWS_MASK_ALL & ~SWS_MASK_OFF4, }; /* Helper function to compute the correct mask */ uint32_t ff_sws_linear_mask(SwsLinearOp); typedef struct SwsFilterOp { SwsFilterWeights *kernel; /* filter kernel (refstruct) */ } SwsFilterOp; typedef struct SwsOp { SwsOpType op; /* operation to perform */ SwsPixelType type; /* pixel type to operate on */ union { SwsLinearOp lin; SwsReadWriteOp rw; SwsPackOp pack; SwsSwizzleOp swizzle; SwsShiftOp shift; SwsClearOp clear; SwsConvertOp convert; SwsClampOp clamp; SwsScaleOp scale; SwsDitherOp dither; SwsFilterOp filter; }; /** * Metadata about the operation's input/output components. Discarded * and regenerated automatically by `ff_sws_op_list_update_comps()`. * * Note that backends may rely on the presence and accuracy of this * metadata for all operations, during ff_sws_ops_compile(). */ SwsComps comps; } SwsOp; #define SWS_OP_NEEDED(op, idx) (!((op)->comps.flags[idx] & SWS_COMP_GARBAGE)) /* Compute SwsCompMask from a mask of needed components */ SwsCompMask ff_sws_comp_mask_needed(const SwsOp *op); /** * Describe an operation in human-readable form. */ void ff_sws_op_desc(AVBPrint *bp, const SwsOp *op); /** * Frees any allocations associated with an SwsOp and sets it to {0}. */ void ff_sws_op_uninit(SwsOp *op); /** * Apply an operation to an AVRational. No-op for read/write operations. */ void ff_sws_apply_op_q(const SwsOp *op, AVRational x[4]); /** * Helper struct for representing a list of operations. */ typedef struct SwsOpList { SwsOp *ops; int num_ops; /* Metadata associated with this operation list */ SwsFormat src, dst; /* Input/output plane indices */ uint8_t plane_src[4], plane_dst[4]; /** * Source component metadata associated with pixel values from each * corresponding component (in plane/memory order, i.e. not affected by * `plane_src`). Lets the optimizer know additional information about * the value range and/or pixel data to expect. * * The default value of {0} is safe to pass in the case that no additional * information is known. */ SwsComps comps_src; } SwsOpList; SwsOpList *ff_sws_op_list_alloc(void); void ff_sws_op_list_free(SwsOpList **ops); /** * Returns a duplicate of `ops`, or NULL on OOM. */ SwsOpList *ff_sws_op_list_duplicate(const SwsOpList *ops); /** * Returns the input operation for a given op list, or NULL if there is none * (e.g. for a pure CLEAR-only operation list). * * This will always be an op of type SWS_OP_READ. */ const SwsOp *ff_sws_op_list_input(const SwsOpList *ops); /** * Returns the output operation for a given op list, or NULL if there is none. * * This will always be an op of type SWS_OP_WRITE. */ const SwsOp *ff_sws_op_list_output(const SwsOpList *ops); /** * Returns whether an op list represents a true no-op operation, i.e. may be * eliminated entirely from an execution graph. */ bool ff_sws_op_list_is_noop(const SwsOpList *ops); /** * Returns the size of the largest pixel type used in `ops`. */ int ff_sws_op_list_max_size(const SwsOpList *ops); /** * These will take over ownership of `op` and set it to {0}, even on failure. */ int ff_sws_op_list_append(SwsOpList *ops, SwsOp *op); int ff_sws_op_list_insert_at(SwsOpList *ops, int index, SwsOp *op); void ff_sws_op_list_remove_at(SwsOpList *ops, int index, int count); /** * Print out the contents of an operation list. */ void ff_sws_op_list_print(void *log_ctx, int log_level, int log_level_extra, const SwsOpList *ops); /** * Infer + propagate known information about components. Called automatically * when needed by the optimizer and compiler. */ void ff_sws_op_list_update_comps(SwsOpList *ops); /** * Fuse compatible and eliminate redundant operations, as well as replacing * some operations with more efficient alternatives. */ int ff_sws_op_list_optimize(SwsOpList *ops); enum SwsOpCompileFlags { /* Automatically optimize the operations when compiling */ SWS_OP_FLAG_OPTIMIZE = 1 << 0, }; /** * Resolves an operation list to a graph pass. The first and last operations * must be a read/write respectively. `flags` is a list of SwsOpCompileFlags. * * Takes over ownership of `ops` and sets it to NULL, even on failure. * * Note: `ops` may be modified by this function. */ int ff_sws_compile_pass(SwsGraph *graph, SwsOpList **ops, int flags, SwsPass *input, SwsPass **output); /** * Helper function to enumerate over all possible (optimized) operation lists, * under the current set of options in `ctx`, and run the given callback on * each list. * * @param src_fmt If set (not AV_PIX_FMT_NONE), constrain the source format * @param dst_fmt If set (not AV_PIX_FMT_NONE), constrain the destination format * @return 0 on success, the return value if cb() < 0, or a negative error code * * @note `ops` belongs to ff_sws_enum_op_lists(), but may be mutated by `cb`. * @see ff_sws_enum_ops() */ int ff_sws_enum_op_lists(SwsContext *ctx, void *opaque, enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt, int (*cb)(SwsContext *ctx, void *opaque, SwsOpList *ops)); /** * Helper function to enumerate over all possible operations, under the current * set of options in `ctx`, and run the given callback on each operation. * * @param src_fmt If set (not AV_PIX_FMT_NONE), constrain the source format * @param dst_fmt If set (not AV_PIX_FMT_NONE), constrain the destination format * @return 0 on success, the return value if cb() < 0, or a negative error code * * @note May contain duplicates. `op` belongs to ff_sws_enum_ops(), but may be * mutated by `cb`. * @see ff_sws_num_op_lists() */ int ff_sws_enum_ops(SwsContext *ctx, void *opaque, enum AVPixelFormat src_fmt, enum AVPixelFormat dst_fmt, int (*cb)(SwsContext *ctx, void *opaque, SwsOp *op)); #endif