diff --git a/libavcodec/opus/enc.c b/libavcodec/opus/enc.c
index 65e6a09575..587e4ed69d 100644
--- a/libavcodec/opus/enc.c
+++ b/libavcodec/opus/enc.c
@@ -548,7 +548,7 @@ static int opus_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
                              const AVFrame *frame, int *got_packet_ptr)
 {
     OpusEncContext *s = avctx->priv_data;
-    int ret, frame_size, alloc_size = 0;
+    int ret, frame_size, discard_padding, alloc_size = 0;
 
     if (frame) { /* Add new frame to queue */
         if ((ret = ff_af_queue_add(&s->afq, frame)) < 0)
@@ -600,11 +600,13 @@ static int opus_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
 
     /* Remove samples from queue and skip if needed */
     ff_af_queue_remove(&s->afq, s->packet.frames*frame_size, &avpkt->pts, &avpkt->duration);
-    if (s->packet.frames*frame_size > avpkt->duration) {
+
+    discard_padding = s->packet.frames*frame_size - ff_samples_from_time_base(avctx, avpkt->duration);
+    if (discard_padding > 0) {
         uint8_t *side = av_packet_new_side_data(avpkt, AV_PKT_DATA_SKIP_SAMPLES, 10);
         if (!side)
             return AVERROR(ENOMEM);
-        AV_WL32(&side[4], s->packet.frames*frame_size - avpkt->duration);
+        AV_WL32(&side[4], discard_padding);
     }
 
     *got_packet_ptr = 1;