Fix some media attachments being converted with too high framerates (m…

…astodon#17619) Video files with variable framerates are converted to constant framerate videos and the output framerate picked by ffmpeg is based on the original file's container framerate (which can be different from the average framerate). This means that an input video with variable framerate with about 30 frames per second on average, but a maximum of 120 fps will be converted to a constant 120 fps file, which won't be processed by other Mastodon servers. This commit changes it so that input files with VFR and a maximum framerate above the framerate threshold are converted to VFR files with the maximum frame rate enforced.
hachyderm · Feb 22, 2022 · 166f6e4 · 166f6e4
1 parent 51e67f3
commit 166f6e4
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 8 deletions.
diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb
@@ -2,7 +2,7 @@
 
 class VideoMetadataExtractor
   attr_reader :duration, :bitrate, :video_codec, :audio_codec,
-              :colorspace, :width, :height, :frame_rate
+              :colorspace, :width, :height, :frame_rate, :r_frame_rate
 
   def initialize(path)
     @path     = path
@@ -42,6 +42,7 @@ def parse_metadata
         @width       = video_stream[:width]
         @height      = video_stream[:height]
         @frame_rate  = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate])
+        @r_frame_rate = video_stream[:r_frame_rate] == '0/0' ? nil : Rational(video_stream[:r_frame_rate])
       end
 
       if (audio_stream = audio_streams.first)

diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
@@ -38,6 +38,12 @@ class MediaAttachment < ApplicationRecord
 
   MAX_DESCRIPTION_LENGTH = 1_500
 
+  IMAGE_LIMIT = 10.megabytes
+  VIDEO_LIMIT = 40.megabytes
+
+  MAX_VIDEO_MATRIX_LIMIT = 2_304_000 # 1920x1200px
+  MAX_VIDEO_FRAME_RATE   = 60
+
   IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif).freeze
   VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze
   AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp .wma).freeze
@@ -75,6 +81,7 @@ class MediaAttachment < ApplicationRecord
   VIDEO_FORMAT = {
     format: 'mp4',
     content_type: 'video/mp4',
+    vfr_frame_rate_threshold: MAX_VIDEO_FRAME_RATE,
     convert_options: {
       output: {
         'loglevel' => 'fatal',
@@ -152,12 +159,6 @@ class MediaAttachment < ApplicationRecord
     all: '-quality 90 -strip +set modify-date +set create-date',
   }.freeze
 
-  IMAGE_LIMIT = 10.megabytes
-  VIDEO_LIMIT = 40.megabytes
-
-  MAX_VIDEO_MATRIX_LIMIT = 2_304_000 # 1920x1200px
-  MAX_VIDEO_FRAME_RATE   = 60
-
   belongs_to :account,          inverse_of: :media_attachments, optional: true
   belongs_to :status,           inverse_of: :media_attachments, optional: true
   belongs_to :scheduled_status, inverse_of: :media_attachments, optional: true

diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb
@@ -13,6 +13,7 @@ def initialize(file, options = {}, attachment = nil)
       @time                = options[:time] || 3
       @passthrough_options = options[:passthrough_options]
       @convert_options     = options[:convert_options].dup
+      @vfr_threshold       = options[:vfr_frame_rate_threshold]
     end
 
     def make
@@ -41,6 +42,11 @@ def make
       when 'mp4'
         @output_options['acodec'] = 'aac'
         @output_options['strict'] = 'experimental'
+
+        if high_vfr?(metadata) && !eligible_to_passthrough?(metadata)
+          @output_options['vsync'] = 'vfr'
+          @output_options['r'] = @vfr_threshold
+        end
       end
 
       command_arguments, interpolations = prepare_command(destination)
@@ -88,13 +94,21 @@ def prepare_command(destination)
     end
 
     def update_options_from_metadata(metadata)
-      return unless @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace)
+      return unless eligible_to_passthrough?(metadata)
 
       @format          = @passthrough_options[:options][:format] || @format
       @time            = @passthrough_options[:options][:time]   || @time
       @convert_options = @passthrough_options[:options][:convert_options].dup
     end
 
+    def high_vfr?(metadata)
+      @vfr_threshold && metadata.r_frame_rate && metadata.r_frame_rate > @vfr_threshold
+    end
+
+    def eligible_to_passthrough?(metadata)
+      @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace)
+    end
+
     def update_attachment_type(metadata)
       @attachment.instance.type = MediaAttachment.types[:gifv] unless metadata.audio_codec
     end