swscale/graph: allow setup() to return an error code

Useful for a handful of reasons, including Vulkan (which depends on external
device resources), but also a change I want to make to the tail handling.

Sponsored-by: Sovereign Tech Fund
Signed-off-by: Niklas Haas <git@haasn.dev>

libswscale/graph.c

@@ -257,8 +257,8 @@ static void free_legacy_swscale(void *priv)
     sws_free_context(&sws);
 }
 
-static void setup_legacy_swscale(const SwsFrame *out, const SwsFrame *in,
-                                 const SwsPass *pass)
+static int setup_legacy_swscale(const SwsFrame *out, const SwsFrame *in,
+                                const SwsPass *pass)
 {
     SwsContext *sws = pass->priv;
     SwsInternal *c = sws_internal(sws);
@@ -269,6 +269,8 @@ static void setup_legacy_swscale(const SwsFrame *out, const SwsFrame *in,
 
     if (usePal(sws->src_format))
         ff_update_palette(c, (const uint32_t *) in->data[1]);
+
+    return 0;
 }
 
 static inline SwsContext *slice_ctx(const SwsPass *pass, int y)
@@ -634,12 +636,13 @@ static void free_lut3d(void *priv)
     ff_sws_lut3d_free(&lut);
 }
 
-static void setup_lut3d(const SwsFrame *out, const SwsFrame *in, const SwsPass *pass)
+static int setup_lut3d(const SwsFrame *out, const SwsFrame *in, const SwsPass *pass)
 {
     SwsLut3D *lut = pass->priv;
 
     /* Update dynamic frame metadata from the original source frame */
     ff_sws_lut3d_update(lut, &pass->graph->src.color);
+    return 0;
 }
 
 static void run_lut3d(const SwsFrame *out, const SwsFrame *in, int y, int h,
@@ -882,7 +885,7 @@ static void get_field(SwsGraph *graph, const AVFrame *avframe, SwsFrame *frame)
     frame->height = (frame->height + (graph->field == FIELD_TOP)) >> 1;
 }
 
-void ff_sws_graph_run(SwsGraph *graph, const AVFrame *dst, const AVFrame *src)
+int ff_sws_graph_run(SwsGraph *graph, const AVFrame *dst, const AVFrame *src)
 {
     av_assert0(dst->format == graph->dst.hw_format || dst->format == graph->dst.format);
     av_assert0(src->format == graph->src.hw_format || src->format == graph->src.format);
@@ -896,8 +899,11 @@ void ff_sws_graph_run(SwsGraph *graph, const AVFrame *dst, const AVFrame *src)
         graph->exec.pass   = pass;
         graph->exec.input  = pass->input ? &pass->input->output->frame : &src_field;
         graph->exec.output = pass->output->avframe ? &pass->output->frame : &dst_field;
-        if (pass->setup)
-            pass->setup(graph->exec.output, graph->exec.input, pass);
+        if (pass->setup) {
+            int ret = pass->setup(graph->exec.output, graph->exec.input, pass);
+            if (ret < 0)
+                return ret;
+        }
 
         if (pass->num_slices == 1) {
             pass->run(graph->exec.output, graph->exec.input, 0, pass->height, pass);
@@ -905,4 +911,6 @@ void ff_sws_graph_run(SwsGraph *graph, const AVFrame *dst, const AVFrame *src)
             avpriv_slicethread_execute(graph->slicethread, pass->num_slices, 0);
         }
     }
+
+    return 0;
 }

libswscale/graph.h

@@ -48,8 +48,8 @@ typedef void (*SwsPassFunc)(const SwsFrame *out, const SwsFrame *in,
 /**
  * Function to run from the main thread before processing any lines.
  */
-typedef void (*SwsPassSetup)(const SwsFrame *out, const SwsFrame *in,
-                             const SwsPass *pass);
+typedef int (*SwsPassSetup)(const SwsFrame *out, const SwsFrame *in,
+                            const SwsPass *pass);
 
 /**
  * Represents an allocated output buffer for a filter pass.
@@ -93,6 +93,7 @@ struct SwsPass {
 
     /**
      * Called once from the main thread before running the filter. Optional.
+     * Returns 0 or a negative error code.
      */
     SwsPassSetup setup;
 
@@ -195,6 +196,6 @@ int ff_sws_graph_reinit(SwsContext *ctx, const SwsFormat *dst, const SwsFormat *
  * Dispatch the filter graph on a single field of the given frames. Internally
  * threaded.
  */
-void ff_sws_graph_run(SwsGraph *graph, const AVFrame *dst, const AVFrame *src);
+int ff_sws_graph_run(SwsGraph *graph, const AVFrame *dst, const AVFrame *src);
 
 #endif /* SWSCALE_GRAPH_H */

libswscale/ops_dispatch.c

@@ -117,8 +117,8 @@ static inline void get_row_data(const SwsOpPass *p, const int y,
         out[i] = base->out[i] + (y >> base->out_sub_y[i]) * base->out_stride[i];
 }
 
-static void op_pass_setup(const SwsFrame *out, const SwsFrame *in,
-                          const SwsPass *pass)
+static int op_pass_setup(const SwsFrame *out, const SwsFrame *in,
+                         const SwsPass *pass)
 {
     const AVPixFmtDescriptor *indesc  = av_pix_fmt_desc_get(in->format);
     const AVPixFmtDescriptor *outdesc = av_pix_fmt_desc_get(out->format);
@@ -180,6 +180,8 @@ static void op_pass_setup(const SwsFrame *out, const SwsFrame *in,
         exec->in_bump[i]  = exec->in_stride[i]  - blocks_main * exec->block_size_in;
         exec->out_bump[i] = exec->out_stride[i] - blocks_main * exec->block_size_out;
     }
+
+    return 0;
 }
 
 /* Dispatch kernel over the last column of the image using memcpy */

libswscale/swscale.c

@@ -1395,8 +1395,11 @@ int sws_scale_frame(SwsContext *sws, AVFrame *dst, const AVFrame *src)
         return ret;
 
 process_frame:
-    for (int field = 0; field < (bot ? 2 : 1); field++)
-        ff_sws_graph_run(c->graph[field], dst, src);
+    for (int field = 0; field < (bot ? 2 : 1); field++) {
+        ret = ff_sws_graph_run(c->graph[field], dst, src);
+        if (ret < 0)
+            return ret;
+    }
 
     return 0;
 }