diff --git a/libswresample/rematrix.c b/libswresample/rematrix.c
index c88b22d8fb..906f6e0f34 100644
--- a/libswresample/rematrix.c
+++ b/libswresample/rematrix.c
@@ -466,8 +466,7 @@ av_cold int swri_rematrix_init(SwrContext *s){
     if (s->midbuf.fmt == AV_SAMPLE_FMT_S16P){
         int maxsum = 0;
         s->native_matrix = av_calloc(nb_in * nb_out, sizeof(int));
-        s->native_one    = av_mallocz(sizeof(int));
-        if (!s->native_matrix || !s->native_one)
+        if (!s->native_matrix)
             return AVERROR(ENOMEM);
         for (i = 0; i < nb_out; i++) {
             double rem = 0;
@@ -481,7 +480,7 @@ av_cold int swri_rematrix_init(SwrContext *s){
             }
             maxsum = FFMAX(maxsum, sum);
         }
-        *((int*)s->native_one) = 32768;
+        s->native_one.i = 32768;
         if (maxsum <= 32768) {
             s->mix_1_1_f = copy_s16;
             s->mix_2_1_f = sum2_s16;
@@ -493,37 +492,30 @@ av_cold int swri_rematrix_init(SwrContext *s){
         }
     }else if(s->midbuf.fmt == AV_SAMPLE_FMT_FLTP){
         s->native_matrix = av_calloc(nb_in * nb_out, sizeof(float));
-        s->native_one    = av_mallocz(sizeof(float));
-        if (!s->native_matrix || !s->native_one)
+        if (!s->native_matrix)
             return AVERROR(ENOMEM);
         for (i = 0; i < nb_out; i++)
             for (j = 0; j < nb_in; j++)
                 ((float*)s->native_matrix)[i * nb_in + j] = s->matrix[i][j];
-        *((float*)s->native_one) = 1.0;
+        s->native_one.f = 1.0;
         s->mix_1_1_f = copy_float;
         s->mix_2_1_f = sum2_float;
         s->mix_any_f = get_mix_any_func_float(s);
     }else if(s->midbuf.fmt == AV_SAMPLE_FMT_DBLP){
         s->native_matrix = av_calloc(nb_in * nb_out, sizeof(double));
-        s->native_one    = av_mallocz(sizeof(double));
-        if (!s->native_matrix || !s->native_one)
+        if (!s->native_matrix)
             return AVERROR(ENOMEM);
         for (i = 0; i < nb_out; i++)
             for (j = 0; j < nb_in; j++)
                 ((double*)s->native_matrix)[i * nb_in + j] = s->matrix[i][j];
-        *((double*)s->native_one) = 1.0;
+        s->native_one.d = 1.0;
         s->mix_1_1_f = copy_double;
         s->mix_2_1_f = sum2_double;
         s->mix_any_f = get_mix_any_func_double(s);
     }else if(s->midbuf.fmt == AV_SAMPLE_FMT_S32P){
-        s->native_one    = av_mallocz(sizeof(int));
-        if (!s->native_one)
-            return AVERROR(ENOMEM);
         s->native_matrix = av_calloc(nb_in * nb_out, sizeof(int));
-        if (!s->native_matrix) {
-            av_freep(&s->native_one);
+        if (!s->native_matrix)
             return AVERROR(ENOMEM);
-        }
         for (i = 0; i < nb_out; i++) {
             double rem = 0;
 
@@ -533,7 +525,7 @@ av_cold int swri_rematrix_init(SwrContext *s){
                 rem += target - ((int*)s->native_matrix)[i * nb_in + j];
             }
         }
-        *((int*)s->native_one) = 32768;
+        s->native_one.i = 32768;
         s->mix_1_1_f = copy_s32;
         s->mix_2_1_f = sum2_s32;
         s->mix_any_f = get_mix_any_func_s32(s);
@@ -569,7 +561,6 @@ av_cold int swri_rematrix_init(SwrContext *s){
 
 av_cold void swri_rematrix_free(SwrContext *s){
     av_freep(&s->native_matrix);
-    av_freep(&s->native_one);
     av_freep(&s->native_simd_matrix);
     av_freep(&s->native_simd_one);
 }
diff --git a/libswresample/swresample.c b/libswresample/swresample.c
index 0a4e216f9b..67fb7964b2 100644
--- a/libswresample/swresample.c
+++ b/libswresample/swresample.c
@@ -691,10 +691,10 @@ static int swr_convert_internal(struct SwrContext *s, AudioData *out, int out_co
                             s->mix_2_1_simd(conv_src->ch[ch], preout->ch[ch], s->dither.noise.ch[ch] + s->dither.noise.bps * s->dither.noise_pos, s->native_simd_one, 0, 0, len1);
                     if(out_count != len1)
                         for(ch=0; ch<preout->ch_count; ch++)
-                            s->mix_2_1_f(conv_src->ch[ch] + off, preout->ch[ch] + off, s->dither.noise.ch[ch] + s->dither.noise.bps * s->dither.noise_pos + off, s->native_one, 0, 0, out_count - len1);
+                            s->mix_2_1_f(conv_src->ch[ch] + off, preout->ch[ch] + off, s->dither.noise.ch[ch] + s->dither.noise.bps * s->dither.noise_pos + off, &s->native_one, 0, 0, out_count - len1);
                 } else {
                     for(ch=0; ch<preout->ch_count; ch++)
-                        s->mix_2_1_f(conv_src->ch[ch], preout->ch[ch], s->dither.noise.ch[ch] + s->dither.noise.bps * s->dither.noise_pos, s->native_one, 0, 0, out_count);
+                        s->mix_2_1_f(conv_src->ch[ch], preout->ch[ch], s->dither.noise.ch[ch] + s->dither.noise.bps * s->dither.noise_pos, &s->native_one, 0, 0, out_count);
                 }
             } else {
                 switch(s->int_sample_fmt) {
diff --git a/libswresample/swresample_internal.h b/libswresample/swresample_internal.h
index c2b5e18a2c..1c889272c8 100644
--- a/libswresample/swresample_internal.h
+++ b/libswresample/swresample_internal.h
@@ -173,8 +173,12 @@ struct SwrContext {
         int32_t matrix32[SWR_CH_MAX][SWR_CH_MAX];   ///< 17.15 fixed point rematrixing coefficients
                                                     ///< valid iff int_sample_fmt is != AV_SAMPLE_FMT_FLTP, AV_SAMPLE_FMT_DBLP
     };
+    union {
+        int    i;
+        float  f;
+        double d;
+    } native_one;
     uint8_t *native_matrix;
-    uint8_t *native_one;
     uint8_t *native_simd_one;
     uint8_t *native_simd_matrix;
     uint8_t matrix_ch[SWR_CH_MAX][SWR_CH_MAX+1];    ///< Lists of input channels per output channel that have non zero rematrixing coefficients
diff --git a/libswresample/x86/rematrix_init.c b/libswresample/x86/rematrix_init.c
index 623e154f5d..5d2c7d9a37 100644
--- a/libswresample/x86/rematrix_init.c
+++ b/libswresample/x86/rematrix_init.c
@@ -79,7 +79,7 @@ av_cold int swri_rematrix_init_x86(struct SwrContext *s){
         if (!s->native_simd_matrix || !s->native_simd_one)
             return AVERROR(ENOMEM);
         memcpy(s->native_simd_matrix, s->native_matrix, num * sizeof(float));
-        memcpy(s->native_simd_one, s->native_one, sizeof(float));
+        memcpy(s->native_simd_one, &s->native_one.f, sizeof(float));
     }
 #endif