Merge tag 'v4.0.0rc3'

app/models/account.rb

@@ -64,6 +64,7 @@ class Account < ApplicationRecord
   USERNAME_RE   = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
   MENTION_RE    = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i
   URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/
+  USERNAME_ONLY_RE = /\A#{USERNAME_RE}\z/i
 
   include Attachmentable
   include AccountAssociations
@@ -84,7 +85,7 @@ class Account < ApplicationRecord
   validates_with UniqueUsernameValidator, if: -> { will_save_change_to_username? }
 
   # Remote user validations
-  validates :username, format: { with: /\A#{USERNAME_RE}\z/i }, if: -> { !local? && will_save_change_to_username? }
+  validates :username, format: { with: USERNAME_ONLY_RE }, if: -> { !local? && will_save_change_to_username? }
 
   # Local user validations
   validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: 30 }, if: -> { local? && will_save_change_to_username? && actor_type != 'Application' }
@@ -295,7 +296,7 @@ class Account < ApplicationRecord
 
   def fields
     (self[:fields] || []).map do |f|
-      Field.new(self, f)
+      Account::Field.new(self, f)
     rescue
       nil
     end.compact
@@ -401,48 +402,6 @@ class Account < ApplicationRecord
     requires_review? && !requested_review?
   end
 
-  class Field < ActiveModelSerializers::Model
-    attributes :name, :value, :verified_at, :account
-
-    def initialize(account, attributes)
-      @original_field = attributes
-      string_limit = account.local? ? 255 : 2047
-      super(
-        account:     account,
-        name:        attributes['name'].strip[0, string_limit],
-        value:       attributes['value'].strip[0, string_limit],
-        verified_at: attributes['verified_at']&.to_datetime,
-      )
-    end
-
-    def verified?
-      verified_at.present?
-    end
-
-    def value_for_verification
-      @value_for_verification ||= begin
-        if account.local?
-          value
-        else
-          ActionController::Base.helpers.strip_tags(value)
-        end
-      end
-    end
-
-    def verifiable?
-      value_for_verification.present? && value_for_verification.start_with?('http://', 'https://')
-    end
-
-    def mark_verified!
-      self.verified_at = Time.now.utc
-      @original_field['verified_at'] = verified_at
-    end
-
-    def to_h
-      { name: name, value: value, verified_at: verified_at }
-    end
-  end
-
   class << self
     DISALLOWED_TSQUERY_CHARACTERS = /['?\\:‘’]/.freeze
     TEXTSEARCH = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))"

app/models/account/field.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+class Account::Field < ActiveModelSerializers::Model
+  MAX_CHARACTERS_LOCAL  = 255
+  MAX_CHARACTERS_COMPAT = 2_047
+  ACCEPTED_SCHEMES      = %w(https).freeze
+
+  attributes :name, :value, :verified_at, :account
+
+  def initialize(account, attributes)
+    # Keeping this as reference allows us to update the field on the account
+    # from methods in this class, so that changes can be saved.
+    @original_field = attributes
+    @account        = account
+
+    super(
+      name:        sanitize(attributes['name']),
+      value:       sanitize(attributes['value']),
+      verified_at: attributes['verified_at']&.to_datetime,
+    )
+  end
+
+  def verified?
+    verified_at.present?
+  end
+
+  def value_for_verification
+    @value_for_verification ||= begin
+      if account.local?
+        value
+      else
+        extract_url_from_html
+      end
+    end
+  end
+
+  def verifiable?
+    return false if value_for_verification.blank?
+
+    # This is slower than checking through a regular expression, but we
+    # need to confirm that it's not an IDN domain.
+
+    parsed_url = Addressable::URI.parse(value_for_verification)
+
+    ACCEPTED_SCHEMES.include?(parsed_url.scheme) &&
+      parsed_url.user.nil? &&
+      parsed_url.password.nil? &&
+      parsed_url.host.present? &&
+      parsed_url.normalized_host == parsed_url.host
+  rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
+    false
+  end
+
+  def requires_verification?
+    !verified? && verifiable?
+  end
+
+  def mark_verified!
+    @original_field['verified_at'] = self.verified_at = Time.now.utc
+  end
+
+  def to_h
+    { name: name, value: value, verified_at: verified_at }
+  end
+
+  private
+
+  def sanitize(str)
+    str.strip[0, character_limit]
+  end
+
+  def character_limit
+    account.local? ? MAX_CHARACTERS_LOCAL : MAX_CHARACTERS_COMPAT
+  end
+
+  def extract_url_from_html
+    doc = Nokogiri::HTML(value).at_xpath('//body')
+
+    return if doc.children.size > 1
+
+    element = doc.children.first
+
+    return if element.name != 'a' || element['href'] != element.text
+
+    element['href']
+  end
+end

app/models/account_alias.rb

@@ -29,7 +29,7 @@ class AccountAlias < ApplicationRecord
   end
 
   def pretty_acct
-    username, domain = acct.split('@')
+    username, domain = acct.split('@', 2)
     domain.nil? ? username : "#{username}@#{Addressable::IDNA.to_unicode(domain)}"
   end
 

app/models/account_migration.rb

@@ -58,7 +58,7 @@ class AccountMigration < ApplicationRecord
   private
 
   def set_target_account
-    self.target_account = ResolveAccountService.new.call(acct)
+    self.target_account = ResolveAccountService.new.call(acct, skip_cache: true)
   rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error
     # Validation will take care of it
   end

app/models/account_statuses_cleanup_policy.rb

@@ -139,7 +139,12 @@ class AccountStatusesCleanupPolicy < ApplicationRecord
     # Filtering on `id` rather than `min_status_age` ago will treat
     # non-snowflake statuses as older than they really are, but Mastodon
     # has switched to snowflake IDs significantly over 2 years ago anyway.
-    max_id = [max_id, Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false)].compact.min
+    snowflake_id = Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false)
+
+    if max_id.nil? || snowflake_id < max_id
+      max_id = snowflake_id
+    end
+
     Status.where(Status.arel_table[:id].lteq(max_id))
   end
 

app/models/concerns/account_avatar.rb

@@ -18,7 +18,7 @@ module AccountAvatar
 
   included do
     # Avatar upload
-    has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
+    has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '+profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail]
     validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES
     validates_attachment_size :avatar, less_than: LIMIT
     remotable_attachment :avatar, LIMIT, suppress_errors: false

app/models/concerns/account_header.rb

@@ -19,7 +19,7 @@ module AccountHeader
 
   included do
     # Header upload
-    has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
+    has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '+profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail]
     validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES
     validates_attachment_size :header, less_than: LIMIT
     remotable_attachment :header, LIMIT, suppress_errors: false

app/models/concerns/status_threading_concern.rb

@@ -7,8 +7,8 @@ module StatusThreadingConcern
     find_statuses_from_tree_path(ancestor_ids(limit), account)
   end
 
-  def descendants(limit, account = nil, max_child_id = nil, since_child_id = nil, depth = nil)
-    find_statuses_from_tree_path(descendant_ids(limit, max_child_id, since_child_id, depth), account, promote: true)
+  def descendants(limit, account = nil, depth = nil)
+    find_statuses_from_tree_path(descendant_ids(limit, depth), account, promote: true)
   end
 
   def self_replies(limit)
@@ -50,22 +50,17 @@ module StatusThreadingConcern
     SQL
   end
 
-  def descendant_ids(limit, max_child_id, since_child_id, depth)
-    descendant_statuses(limit, max_child_id, since_child_id, depth).pluck(:id)
-  end
-
-  def descendant_statuses(limit, max_child_id, since_child_id, depth)
+  def descendant_ids(limit, depth)
     # use limit + 1 and depth + 1 because 'self' is included
     depth += 1 if depth.present?
     limit += 1 if limit.present?
 
-    descendants_with_self = Status.find_by_sql([<<-SQL.squish, id: id, limit: limit, max_child_id: max_child_id, since_child_id: since_child_id, depth: depth])
-      WITH RECURSIVE search_tree(id, path)
-      AS (
+    descendants_with_self = Status.find_by_sql([<<-SQL.squish, id: id, limit: limit, depth: depth])
+      WITH RECURSIVE search_tree(id, path) AS (
         SELECT id, ARRAY[id]
         FROM statuses
-        WHERE id = :id AND COALESCE(id < :max_child_id, TRUE) AND COALESCE(id > :since_child_id, TRUE)
-        UNION ALL
+        WHERE id = :id
+      UNION ALL
         SELECT statuses.id, path || statuses.id
         FROM search_tree
         JOIN statuses ON statuses.in_reply_to_id = search_tree.id
@@ -77,7 +72,7 @@ module StatusThreadingConcern
       LIMIT :limit
     SQL
 
-    descendants_with_self - [self]
+    descendants_with_self.pluck(:id) - [id]
   end
 
   def find_statuses_from_tree_path(ids, account, promote: false)

app/models/custom_emoji.rb

@@ -30,18 +30,19 @@ class CustomEmoji < ApplicationRecord
   SCAN_RE = /(?<=[^[:alnum:]:]|\n|^)
     :(#{SHORTCODE_RE_FRAGMENT}):
     (?=[^[:alnum:]:]|$)/x
+  SHORTCODE_ONLY_RE = /\A#{SHORTCODE_RE_FRAGMENT}\z/
 
   IMAGE_MIME_TYPES = %w(image/png image/gif image/webp).freeze
 
   belongs_to :category, class_name: 'CustomEmojiCategory', optional: true
   has_one :local_counterpart, -> { where(domain: nil) }, class_name: 'CustomEmoji', primary_key: :shortcode, foreign_key: :shortcode
 
-  has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false
+  has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set modify-date +set create-date' } }, validate_media_type: false
 
   before_validation :downcase_domain
 
   validates_attachment :image, content_type: { content_type: IMAGE_MIME_TYPES }, presence: true, size: { less_than: LIMIT }
-  validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 }
+  validates :shortcode, uniqueness: { scope: :domain }, format: { with: SHORTCODE_ONLY_RE }, length: { minimum: 2 }
 
   scope :local, -> { where(domain: nil) }
   scope :remote, -> { where.not(domain: nil) }

app/models/featured_tag.rb

@@ -17,7 +17,7 @@ class FeaturedTag < ApplicationRecord
   belongs_to :account, inverse_of: :featured_tags
   belongs_to :tag, inverse_of: :featured_tags, optional: true # Set after validation
 
-  validates :name, presence: true, format: { with: /\A(#{Tag::HASHTAG_NAME_RE})\z/i }, on: :create
+  validates :name, presence: true, format: { with: Tag::HASHTAG_NAME_RE }, on: :create
 
   validate :validate_tag_uniqueness, on: :create
   validate :validate_featured_tags_limit, on: :create
@@ -63,6 +63,8 @@ class FeaturedTag < ApplicationRecord
   end
 
   def validate_featured_tags_limit
+    return unless account.local?
+
     errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= LIMIT
   end
 

app/models/media_attachment.rb

@@ -167,7 +167,7 @@ class MediaAttachment < ApplicationRecord
   }.freeze
 
   GLOBAL_CONVERT_OPTIONS = {
-    all: '-quality 90 -strip +set modify-date +set create-date',
+    all: '-quality 90 +profile "!icc,*" +set modify-date +set create-date',
   }.freeze
 
   belongs_to :account,          inverse_of: :media_attachments, optional: true

app/models/preview_card.rb

@@ -50,7 +50,7 @@ class PreviewCard < ApplicationRecord
   has_and_belongs_to_many :statuses
   has_one :trend, class_name: 'PreviewCardTrend', inverse_of: :preview_card, dependent: :destroy
 
-  has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 80 -strip' }, validate_media_type: false
+  has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 90 +profile "!icc,*" +set modify-date +set create-date' }, validate_media_type: false
 
   validates :url, presence: true, uniqueness: true
   validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES
@@ -122,7 +122,7 @@ class PreviewCard < ApplicationRecord
         original: {
           geometry: '400x400>',
           file_geometry_parser: FastGeometryParser,
-          convert_options: '-coalesce -strip',
+          convert_options: '-coalesce',
           blurhash: BLURHASH_OPTIONS,
         },
       }

app/models/preview_card_provider.rb

@@ -25,7 +25,7 @@ class PreviewCardProvider < ApplicationRecord
 
   validates :domain, presence: true, uniqueness: true, domain: true
 
-  has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false
+  has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce +profile "!icc,*" +set modify-date +set create-date' } }, validate_media_type: false
   validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT }
   remotable_attachment :icon, LIMIT
 

app/models/public_feed.rb

@@ -8,7 +8,6 @@ class PublicFeed
   # @option [Boolean] :local
   # @option [Boolean] :remote
   # @option [Boolean] :only_media
-  # @option [String]  :locale
   def initialize(account, options = {})
     @account = account
     @options = options
@@ -28,7 +27,7 @@ class PublicFeed
     scope.merge!(remote_only_scope) if remote_only?
     scope.merge!(account_filters_scope) if account?
     scope.merge!(media_only_scope) if media_only?
-    scope.merge!(language_scope)
+    scope.merge!(language_scope) if account&.chosen_languages.present?
 
     scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id)
   end
@@ -86,13 +85,7 @@ class PublicFeed
   end
 
   def language_scope
-    if account&.chosen_languages.present?
-      Status.where(language: account.chosen_languages)
-    elsif @options[:locale].present?
-      Status.where(language: @options[:locale])
-    else
-      Status.all
-    end
+    Status.where(language: account.chosen_languages)
   end
 
   def account_filters_scope

app/models/site_upload.rb

@@ -40,7 +40,7 @@ class SiteUpload < ApplicationRecord
     mascot: {}.freeze,
   }.freeze
 
-  has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce -strip' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector]
+  has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce +profile "!icc,*" +set modify-date +set create-date' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector]
 
   validates_attachment_content_type :file, content_type: /\Aimage\/.*\z/
   validates :file, presence: true

app/models/tag.rb

@@ -27,11 +27,14 @@ class Tag < ApplicationRecord
   has_many :followers, through: :passive_relationships, source: :account
 
   HASHTAG_SEPARATORS = "_\u00B7\u200c"
-  HASHTAG_NAME_RE    = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)"
-  HASHTAG_RE         = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
+  HASHTAG_NAME_PAT = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)"
 
-  validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
-  validates :display_name, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_PAT})/i
+  HASHTAG_NAME_RE = /\A(#{HASHTAG_NAME_PAT})\z/i
+  HASHTAG_INVALID_CHARS_RE = /[^[:alnum:]#{HASHTAG_SEPARATORS}]/
+
+  validates :name, presence: true, format: { with: HASHTAG_NAME_RE }
+  validates :display_name, format: { with: HASHTAG_NAME_RE }
   validate :validate_name_change, if: -> { !new_record? && name_changed? }
   validate :validate_display_name_change, if: -> { !new_record? && display_name_changed? }
 
@@ -102,7 +105,7 @@ class Tag < ApplicationRecord
       names = Array(name_or_names).map { |str| [normalize(str), str] }.uniq(&:first)
 
       names.map do |(normalized_name, display_name)|
-        tag = matching_name(normalized_name).first || create(name: normalized_name, display_name: display_name.gsub(/[^[:alnum:]#{HASHTAG_SEPARATORS}]/, ''))
+        tag = matching_name(normalized_name).first || create(name: normalized_name, display_name: display_name.gsub(HASHTAG_INVALID_CHARS_RE, ''))
 
         yield tag if block_given?
 

app/models/tag_feed.rb

@@ -12,7 +12,6 @@ class TagFeed < PublicFeed
   # @option [Boolean] :local
   # @option [Boolean] :remote
   # @option [Boolean] :only_media
-  # @option [String]  :locale
   def initialize(tag, account, options = {})
     @tag = tag
     super(account, options)