diff --git a/libavcodec/libvorbisenc.c b/libavcodec/libvorbisenc.c
index a3f02f3cb9..c54ebebae8 100644
--- a/libavcodec/libvorbisenc.c
+++ b/libavcodec/libvorbisenc.c
@@ -421,7 +421,7 @@ static int libvorbis_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
 
         ff_af_queue_remove(&s->afq, duration, &avpkt->pts, &avpkt->duration);
 
-        discard_padding = ff_samples_to_time_base(avctx, duration) - avpkt->duration;
+        discard_padding = duration - ff_samples_from_time_base(avctx, avpkt->duration);
         if (discard_padding > 0) {
             uint8_t *side_data = av_packet_new_side_data(avpkt,
                                                          AV_PKT_DATA_SKIP_SAMPLES,