Merge tag 'v3.5.0rc1'

lib/mastodon/emoji_cli.rb

@@ -41,7 +41,7 @@ module Mastodon
 
       Gem::Package::TarReader.new(Zlib::GzipReader.open(path)) do |tar|
         tar.each do |entry|
-          next unless entry.file? && entry.full_name.end_with?('.png')
+          next unless entry.file? && entry.full_name.end_with?('.png', '.gif')
 
           filename = File.basename(entry.full_name, '.*')
 

lib/mastodon/maintenance_cli.rb

@@ -13,8 +13,8 @@ module Mastodon
       true
     end
 
-    MIN_SUPPORTED_VERSION = 2019_10_01_213028
-    MAX_SUPPORTED_VERSION = 2022_01_18_183123
+    MIN_SUPPORTED_VERSION = 2019_10_01_213028 # rubocop:disable Style/NumericLiterals
+    MAX_SUPPORTED_VERSION = 2022_03_10_060959 # rubocop:disable Style/NumericLiterals
 
     # Stubs to enjoy ActiveRecord queries while not depending on a particular
     # version of the code/database
@@ -44,6 +44,7 @@ module Mastodon
     class WebauthnCredential < ApplicationRecord; end
     class FollowRecommendationSuppression < ApplicationRecord; end
     class CanonicalEmailBlock < ApplicationRecord; end
+    class Appeal < ApplicationRecord; end
 
     class PreviewCard < ApplicationRecord
       self.inheritance_column = false
@@ -92,6 +93,7 @@ module Mastodon
         owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes)
         owned_classes << FollowRecommendationSuppression if ActiveRecord::Base.connection.table_exists?(:follow_recommendation_suppressions)
         owned_classes << AccountIdentityProof if ActiveRecord::Base.connection.table_exists?(:account_identity_proofs)
+        owned_classes << Appeal if ActiveRecord::Base.connection.table_exists?(:appeals)
 
         owned_classes.each do |klass|
           klass.where(account_id: other_account.id).find_each do |record|
@@ -121,6 +123,12 @@ module Mastodon
             record.update_attribute(:reference_account_id, id)
           end
         end
+
+        if ActiveRecord::Base.connection.table_exists?(:appeals)
+          Appeal.where(account_warning_id: other_account.id).find_each do |record|
+            record.update_attribute(:account_warning_id, id)
+          end
+        end
       end
     end
 
@@ -199,7 +207,7 @@ module Mastodon
       end
 
       @prompt.say 'Restoring index_accounts_on_username_and_domain_lower…'
-      if ActiveRecord::Migrator.current_version < 20200620164023
+      if ActiveRecord::Migrator.current_version < 20200620164023 # rubocop:disable Style/NumericLiterals
         ActiveRecord::Base.connection.add_index :accounts, 'lower (username), lower(domain)', name: 'index_accounts_on_username_and_domain_lower', unique: true
       else
         ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
@@ -242,7 +250,7 @@ module Mastodon
         end
       end
 
-      if ActiveRecord::Migrator.current_version < 20220118183010
+      if ActiveRecord::Migrator.current_version < 20220118183010 # rubocop:disable Style/NumericLiterals
         ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row|
           users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1)
           @prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}"
@@ -266,7 +274,12 @@ module Mastodon
       ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true
       ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true
       ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 20220118183010
-      ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
+
+      if ActiveRecord::Migrator.current_version < 20220310060641 # rubocop:disable Style/NumericLiterals
+        ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true
+      else
+        ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true, where: 'reset_password_token IS NOT NULL', opclass: :text_pattern_ops
+      end
     end
 
     def deduplicate_account_domain_blocks!
@@ -325,7 +338,11 @@ module Mastodon
       end
 
       @prompt.say 'Restoring conversations indexes…'
-      ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
+      if ActiveRecord::Migrator.current_version < 20220307083603 # rubocop:disable Style/NumericLiterals
+        ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true
+      else
+        ActiveRecord::Base.connection.add_index :conversations, ['uri'], name: 'index_conversations_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
+      end
     end
 
     def deduplicate_custom_emojis!
@@ -438,7 +455,11 @@ module Mastodon
       end
 
       @prompt.say 'Restoring media_attachments indexes…'
-      ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
+      if ActiveRecord::Migrator.current_version < 20220310060626 # rubocop:disable Style/NumericLiterals
+        ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true
+      else
+        ActiveRecord::Base.connection.add_index :media_attachments, ['shortcode'], name: 'index_media_attachments_on_shortcode', unique: true, where: 'shortcode IS NOT NULL', opclass: :text_pattern_ops
+      end
     end
 
     def deduplicate_preview_cards!
@@ -467,7 +488,11 @@ module Mastodon
       end
 
       @prompt.say 'Restoring statuses indexes…'
-      ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
+      if ActiveRecord::Migrator.current_version < 20220310060706 # rubocop:disable Style/NumericLiterals
+        ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true
+      else
+        ActiveRecord::Base.connection.add_index :statuses, ['uri'], name: 'index_statuses_on_uri', unique: true, where: 'uri IS NOT NULL', opclass: :text_pattern_ops
+      end
     end
 
     def deduplicate_tags!
@@ -510,7 +535,7 @@ module Mastodon
       accounts = accounts.sort_by(&:id).reverse
 
       @prompt.warn "Multiple local accounts were found for username '#{accounts.first.username}'."
-      @prompt.warn 'All those accounts are distinct accounts but only the most recently-created one is fully-functionnal.'
+      @prompt.warn 'All those accounts are distinct accounts but only the most recently-created one is fully-functional.'
 
       accounts.each_with_index do |account, idx|
         @prompt.say '%2d. %s: created at: %s; updated at: %s; last logged in at: %s; statuses: %5d; last status at: %s' % [idx, account.username, account.created_at, account.updated_at, account.user&.last_sign_in_at&.to_s || 'N/A', account.account_stat&.statuses_count || 0, account.account_stat&.last_status_at || 'N/A']

lib/mastodon/migration_helpers.rb

@@ -42,8 +42,14 @@
 module Mastodon
   module MigrationHelpers
     class CorruptionError < StandardError
-      def initialize(message = nil)
-        super(message.presence || 'Migration failed because of index corruption, see https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/#fixing')
+      attr_reader :index_name
+
+      def initialize(index_name)
+        @index_name = index_name
+
+        super "The index `#{index_name}` seems to be corrupted, it contains duplicate rows. " \
+          'For information on how to fix this, see our documentation: ' \
+          'https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/'
       end
 
       def cause
@@ -802,6 +808,24 @@ module Mastodon
       columns(table).find { |column| column.name == name }
     end
 
+    # Update the configuration of an index by creating a new one and then
+    # removing the old one
+    def update_index(table_name, index_name, columns, **index_options)
+      if index_name_exists?(table_name, "#{index_name}_new") && index_name_exists?(table_name, index_name)
+        remove_index table_name, "#{index_name}_new"
+      end
+
+      begin
+        add_index table_name, columns, **index_options.merge(name: "#{index_name}_new", algorithm: :concurrently)
+      rescue ActiveRecord::RecordNotUnique
+        remove_index table_name, name: "#{index_name}_new"
+        raise CorruptionError.new(index_name)
+      end
+
+      remove_index table_name, name: index_name if index_name_exists?(table_name, index_name)
+      rename_index table_name, "#{index_name}_new", index_name
+    end
+
     # This will replace the first occurrence of a string in a column with
     # the replacement
     # On postgresql we can use `regexp_replace` for that.

lib/mastodon/statuses_cli.rb

@@ -54,7 +54,7 @@ module Mastodon
 
       ActiveRecord::Base.connection.add_index(:media_attachments, :remote_url, name: :index_media_attachments_remote_url, where: 'remote_url is not null', algorithm: :concurrently, if_not_exists: true)
 
-      max_id   = Mastodon::Snowflake.id_at(options[:days].days.ago)
+      max_id   = Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false)
       start_at = Time.now.to_f
 
       unless options[:continue] && ActiveRecord::Base.connection.table_exists?('statuses_to_be_deleted')
@@ -156,7 +156,7 @@ module Mastodon
 
         ActiveRecord::Base.connection.add_index(:statuses, :conversation_id, name: :index_statuses_conversation_id, algorithm: :concurrently, if_not_exists: true)
 
-        say('Extract the deletion target from coversations... This might take a while...')
+        say('Extract the deletion target from conversations... This might take a while...')
 
         ActiveRecord::Base.connection.create_table('conversations_to_be_deleted', force: true)
 

lib/mastodon/version.rb

@@ -9,15 +9,15 @@ module Mastodon
     end
 
     def minor
-      4
+      5
     end
 
     def patch
-      6
+      0
     end
 
     def flags
-      ''
+      'rc1'
     end
 
     def suffix

lib/paperclip/transcoder.rb

@@ -13,6 +13,7 @@ module Paperclip
       @time                = options[:time] || 3
       @passthrough_options = options[:passthrough_options]
       @convert_options     = options[:convert_options].dup
+      @vfr_threshold       = options[:vfr_frame_rate_threshold]
     end
 
     def make
@@ -41,6 +42,11 @@ module Paperclip
       when 'mp4'
         @output_options['acodec'] = 'aac'
         @output_options['strict'] = 'experimental'
+
+        if high_vfr?(metadata) && !eligible_to_passthrough?(metadata)
+          @output_options['vsync'] = 'vfr'
+          @output_options['r'] = @vfr_threshold
+        end
       end
 
       command_arguments, interpolations = prepare_command(destination)
@@ -88,13 +94,21 @@ module Paperclip
     end
 
     def update_options_from_metadata(metadata)
-      return unless @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace)
+      return unless eligible_to_passthrough?(metadata)
 
       @format          = @passthrough_options[:options][:format] || @format
       @time            = @passthrough_options[:options][:time]   || @time
       @convert_options = @passthrough_options[:options][:convert_options].dup
     end
 
+    def high_vfr?(metadata)
+      @vfr_threshold && metadata.r_frame_rate && metadata.r_frame_rate > @vfr_threshold
+    end
+
+    def eligible_to_passthrough?(metadata)
+      @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace)
+    end
+
     def update_attachment_type(metadata)
       @attachment.instance.type = MediaAttachment.types[:gifv] unless metadata.audio_codec
     end

lib/tasks/db.rake

@@ -17,23 +17,10 @@ namespace :db do
     end
   end
 
-  task :post_migration_hook do
-    at_exit do
-      unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate'])
-        warn <<~WARNING
-          Your database collation may be susceptible to index corruption.
-            (This warning does not indicate that index corruption has occurred, and it can be ignored if you've previously checked for index corruption)
-            (To learn more, visit: https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/)
-        WARNING
-      end
-    end
-  end
-
   task :pre_migration_check do
     version = ActiveRecord::Base.connection.select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
-    abort 'ERROR: This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon.' if version < 90_500
+    abort 'This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon' if version < 90_500
   end
 
   Rake::Task['db:migrate'].enhance(['db:pre_migration_check'])
-  Rake::Task['db:migrate'].enhance(['db:post_migration_hook'])
 end

lib/tasks/repo.rake

@@ -96,7 +96,8 @@ namespace :repo do
     end.uniq.compact
 
     missing_available_locales = locales_in_files - I18n.available_locales
-    missing_locale_names = I18n.available_locales.reject { |locale| LanguagesHelper::HUMAN_LOCALES.key?(locale) }
+    supported_locale_codes    = Set.new(LanguagesHelper::SUPPORTED_LOCALES.keys + LanguagesHelper::REGIONAL_LOCALE_NAMES.keys)
+    missing_locale_names      = I18n.available_locales.reject { |locale| supported_locale_codes.include?(locale) }
 
     critical = false
 
@@ -123,7 +124,7 @@ namespace :repo do
 
     unless missing_locale_names.empty?
       puts pastel.yellow("You are missing human-readable names for these locales: #{pastel.bold(missing_locale_names.join(', '))}")
-      puts pastel.yellow("Add them to #{pastel.bold('HUMAN_LOCALES')} in app/helpers/settings_helper.rb or remove the locales from #{pastel.bold('I18n.available_locales')} in config/application.rb")
+      puts pastel.yellow("Add them to app/helpers/languages_helper.rb or remove the locales from #{pastel.bold('I18n.available_locales')} in config/application.rb")
     end
 
     if critical

lib/tasks/tests.rake

@@ -2,6 +2,50 @@
 
 namespace :tests do
   namespace :migrations do
+    desc 'Check that database state is consistent with a successful migration from populated data'
+    task check_database: :environment do
+      unless Account.find_by(username: 'admin', domain: nil)&.hide_collections? == false
+        puts 'Unexpected value for Account#hide_collections? for user @admin'
+        exit(1)
+      end
+
+      unless Account.find_by(username: 'user', domain: nil)&.hide_collections? == true
+        puts 'Unexpected value for Account#hide_collections? for user @user'
+        exit(1)
+      end
+
+      unless Account.find_by(username: 'evil', domain: 'activitypub.com')&.suspended?
+        puts 'Unexpected value for Account#suspended? for user @evil@activitypub.com'
+        exit(1)
+      end
+
+      unless Status.find(6).account_id == Status.find(7).account_id
+        puts 'Users @remote@remote.com and @Remote@remote.com not properly merged'
+        exit(1)
+      end
+
+      if Account.where(domain: Rails.configuration.x.local_domain).exists?
+        puts 'Faux remote accounts not properly claned up'
+        exit(1)
+      end
+
+      unless AccountConversation.first&.last_status_id == 11
+        puts 'AccountConversation records not created as expected'
+        exit(1)
+      end
+    end
+
+    desc 'Populate the database with test data for 2.4.0'
+    task populate_v2_4: :environment do
+      ActiveRecord::Base.connection.execute(<<~SQL)
+        INSERT INTO "settings"
+          (id, thing_type, thing_id, var, value, created_at, updated_at)
+        VALUES
+          (1, 'User', 1, 'hide_network', E'--- false\n', now(), now()),
+          (2, 'User', 2, 'hide_network', E'--- true\n', now(), now());
+      SQL
+    end
+
     desc 'Populate the database with test data for 2.0.0'
     task populate_v2: :environment do
       admin_key   = OpenSSL::PKey::RSA.new(2048)
@@ -34,7 +78,7 @@ namespace :tests do
            'https://remote.com/@remote', 'https://remote.com/salmon/1'),
           (4, 'Remote', 'remote.com', NULL, #{remote_public_key}, now(), now(),
            'https://remote.com/@Remote', 'https://remote.com/salmon/1'),
-          (5, 'REMOTE', 'Remote.com', NULL, #{remote_public_key2}, now(), now(),
+          (5, 'REMOTE', 'Remote.com', NULL, #{remote_public_key2}, now() - interval '1 year', now() - interval '1 year',
            'https://remote.com/stale/@REMOTE', 'https://remote.com/stale/salmon/1');
 
         INSERT INTO "accounts"
@@ -49,6 +93,13 @@ namespace :tests do
           (7, 'user', #{local_domain}, #{user_private_key}, #{user_public_key}, now(), now()),
           (8, 'pt_user', NULL, #{user_private_key}, #{user_public_key}, now(), now());
 
+        INSERT INTO "accounts"
+          (id, username, domain, private_key, public_key, created_at, updated_at, protocol, inbox_url, outbox_url, followers_url, suspended)
+        VALUES
+          (9, 'evil', 'activitypub.com', NULL, #{remote_public_key_ap}, now(), now(),
+           1, 'https://activitypub.com/users/evil/inbox', 'https://activitypub.com/users/evil/outbox',
+           'https://activitypub.com/users/evil/followers', true);
+
         -- users
 
         INSERT INTO "users"
@@ -62,6 +113,9 @@ namespace :tests do
         VALUES
           (3, 7, 'ptuser@localhost', now(), now(), false, 'pt');
 
+        -- conversations
+        INSERT INTO "conversations" (id, created_at, updated_at) VALUES (1, now(), now());
+
         -- statuses
 
         INSERT INTO "statuses"
@@ -97,14 +151,22 @@ namespace :tests do
         VALUES
           (9, 1, 2, now(), now());
 
+        INSERT INTO "statuses"
+          (id, account_id, text, in_reply_to_id, conversation_id, visibility, created_at, updated_at)
+        VALUES
+          (10, 2, '@admin hey!', NULL, 1, 3, now(), now()),
+          (11, 1, '@user hey!', 10, 1, 3, now(), now());
+
         -- mentions (from previous statuses)
 
         INSERT INTO "mentions"
-          (status_id, account_id, created_at, updated_at)
+          (id, status_id, account_id, created_at, updated_at)
         VALUES
-          (2, 3, now(), now()),
-          (3, 4, now(), now()),
-          (4, 5, now(), now());
+          (1, 2, 3, now(), now()),
+          (2, 3, 4, now(), now()),
+          (3, 4, 5, now(), now()),
+          (4, 10, 1, now(), now()),
+          (5, 11, 2, now(), now());
 
         -- stream entries
 
@@ -121,7 +183,6 @@ namespace :tests do
           (8, 5, 'status', now(), now()),
           (9, 1, 'status', now(), now());
 
-
         -- custom emoji
 
         INSERT INTO "custom_emojis"
@@ -161,12 +222,12 @@ namespace :tests do
         -- follows
 
         INSERT INTO "follows"
-          (account_id, target_account_id, created_at, updated_at)
+          (id, account_id, target_account_id, created_at, updated_at)
         VALUES
-          (1, 5, now(), now()),
-          (6, 2, now(), now()),
-          (5, 2, now(), now()),
-          (6, 1, now(), now());
+          (1, 1, 5, now(), now()),
+          (2, 6, 2, now(), now()),
+          (3, 5, 2, now(), now()),
+          (4, 6, 1, now(), now());
 
         -- follow requests
 
@@ -175,6 +236,15 @@ namespace :tests do
         VALUES
           (2, 5, now(), now()),
           (5, 1, now(), now());
+
+        -- notifications
+
+        INSERT INTO "notifications"
+          (id, from_account_id, account_id, activity_type, activity_id, created_at, updated_at)
+        VALUES
+          (1, 6, 2, 'Follow', 2, now(), now()),
+          (2, 2, 1, 'Mention', 4, now(), now()),
+          (3, 1, 2, 'Mention', 5, now(), now());
       SQL
     end
   end