Merge tag 'v4.0.0rc1'

lib/assets/wordmark.dark.css

@@ -0,0 +1 @@
+// Not needed

lib/assets/wordmark.light.css

@@ -0,0 +1 @@
+use { color: #000 !important; }

lib/exceptions.rb

@@ -25,4 +25,13 @@ module Mastodon
       end
     end
   end
+
+  class PrivateNetworkAddressError < HostValidationError
+    attr_reader :host
+
+    def initialize(host)
+      @host = host
+      super()
+    end
+  end
 end

lib/mastodon/accounts_cli.rb

@@ -54,10 +54,10 @@ module Mastodon
 
     option :email, required: true
     option :confirmed, type: :boolean
-    option :role, default: 'user', enum: %w(user moderator admin)
+    option :role
     option :reattach, type: :boolean
     option :force, type: :boolean
-    desc 'create USERNAME', 'Create a new user'
+    desc 'create USERNAME', 'Create a new user account'
     long_desc <<-LONG_DESC
       Create a new user account with a given USERNAME and an
       e-mail address provided with --email.
@@ -65,8 +65,7 @@ module Mastodon
       With the --confirmed option, the confirmation e-mail will
       be skipped and the account will be active straight away.
 
-      With the --role option one of  "user", "admin" or "moderator"
-      can be supplied. Defaults to "user"
+      With the --role option, the role can be supplied.
 
       With the --reattach option, the new user will be reattached
       to a given existing username of an old account. If the old
@@ -75,9 +74,22 @@ module Mastodon
       username to the new account anyway.
     LONG_DESC
     def create(username)
+      role_id  = nil
+
+      if options[:role]
+        role = UserRole.find_by(name: options[:role])
+
+        if role.nil?
+          say('Cannot find user role with that name', :red)
+          exit(1)
+        end
+
+        role_id = role.id
+      end
+
       account  = Account.new(username: username)
       password = SecureRandom.hex
-      user     = User.new(email: options[:email], password: password, agreement: true, approved: true, admin: options[:role] == 'admin', moderator: options[:role] == 'moderator', confirmed_at: options[:confirmed] ? Time.now.utc : nil, bypass_invite_request_check: true)
+      user     = User.new(email: options[:email], password: password, agreement: true, approved: true, role_id: role_id, confirmed_at: options[:confirmed] ? Time.now.utc : nil, bypass_invite_request_check: true)
 
       if options[:reattach]
         account = Account.find_local(username) || Account.new(username: username)
@@ -106,14 +118,15 @@ module Mastodon
         user.errors.to_h.each do |key, error|
           say('Failure/Error: ', :red)
           say(key)
-          say('    ' + error, :red)
+          say("    #{error}", :red)
         end
 
         exit(1)
       end
     end
 
-    option :role, enum: %w(user moderator admin)
+    option :role
+    option :remove_role, type: :boolean
     option :email
     option :confirm, type: :boolean
     option :enable, type: :boolean
@@ -121,12 +134,12 @@ module Mastodon
     option :disable_2fa, type: :boolean
     option :approve, type: :boolean
     option :reset_password, type: :boolean
-    desc 'modify USERNAME', 'Modify a user'
+    desc 'modify USERNAME', 'Modify a user account'
     long_desc <<-LONG_DESC
       Modify a user account.
 
-      With the --role option, update the user's role to one of "user",
-      "moderator" or "admin".
+      With the --role option, update the user's role. To remove the user's
+      role, i.e. demote to normal user, use --remove-role.
 
       With the --email option, update the user's e-mail address. With
       the --confirm option, mark the user's e-mail as confirmed.
@@ -152,8 +165,16 @@ module Mastodon
       end
 
       if options[:role]
-        user.admin = options[:role] == 'admin'
-        user.moderator = options[:role] == 'moderator'
+        role = UserRole.find_by(name: options[:role])
+
+        if role.nil?
+          say('Cannot find user role with that name', :red)
+          exit(1)
+        end
+
+        user.role_id = role.id
+      elsif options[:remove_role]
+        user.role_id = nil
       end
 
       password = SecureRandom.hex if options[:reset_password]
@@ -172,7 +193,7 @@ module Mastodon
         user.errors.to_h.each do |key, error|
           say('Failure/Error: ', :red)
           say(key)
-          say('    ' + error, :red)
+          say("    #{error}", :red)
         end
 
         exit(1)
@@ -319,7 +340,7 @@ module Mastodon
 
       unless skip_domains.empty?
         say('The following domains were not available during the check:', :yellow)
-        skip_domains.each { |domain| say('    ' + domain) }
+        skip_domains.each { |domain| say("    #{domain}") }
       end
     end
 

lib/mastodon/canonical_email_blocks_cli.rb

@@ -18,17 +18,15 @@ module Mastodon
       When suspending a local user, a hash of a "canonical" version of their e-mail
       address is stored to prevent them from signing up again.
 
-      This command can be used to find whether a known email address is blocked,
-      and if so, which account it was attached to.
+      This command can be used to find whether a known email address is blocked.
     LONG_DESC
     def find(email)
-      accts = CanonicalEmailBlock.find_blocks(email).map(&:reference_account).map(&:acct).to_a
+      accts = CanonicalEmailBlock.matching_email(email)
+
       if accts.empty?
-        say("#{email} is not blocked", :yellow)
+        say("#{email} is not blocked", :green)
       else
-        accts.each do |acct|
-          say(acct, :white)
-        end
+        say("#{email} is blocked", :red)
       end
     end
 
@@ -40,24 +38,13 @@ module Mastodon
       This command allows removing a canonical email block.
     LONG_DESC
     def remove(email)
-      blocks = CanonicalEmailBlock.find_blocks(email)
+      blocks = CanonicalEmailBlock.matching_email(email)
+
       if blocks.empty?
-        say("#{email} is not blocked", :yellow)
+        say("#{email} is not blocked", :green)
       else
         blocks.destroy_all
-        say("Removed canonical email block for #{email}", :green)
-      end
-    end
-
-    private
-
-    def color(processed, failed)
-      if !processed.zero? && failed.zero?
-        :green
-      elsif failed.zero?
-        :yellow
-      else
-        :red
+        say("Unblocked #{email}", :green)
       end
     end
   end

lib/mastodon/media_cli.rb

@@ -187,6 +187,7 @@ module Mastodon
     option :account, type: :string
     option :domain, type: :string
     option :status, type: :numeric
+    option :days, type: :numeric
     option :concurrency, type: :numeric, default: 5, aliases: [:c]
     option :verbose, type: :boolean, default: false, aliases: [:v]
     option :dry_run, type: :boolean, default: false
@@ -204,6 +205,8 @@ module Mastodon
 
       Use the --domain option to download attachments from a specific domain.
 
+      Use the --days option to limit attachments created within days.
+
       By default, attachments that are believed to be already downloaded will
       not be re-downloaded. To force re-download of every URL, use --force.
     DESC
@@ -224,10 +227,16 @@ module Mastodon
         scope = MediaAttachment.where(account_id: account.id)
       elsif options[:domain]
         scope = MediaAttachment.joins(:account).merge(Account.by_domain_and_subdomains(options[:domain]))
+      elsif options[:days].present?
+        scope = MediaAttachment.remote
       else
         exit(1)
       end
 
+      if options[:days].present?
+        scope = scope.where('media_attachments.id > ?', Mastodon::Snowflake.id_at(options[:days].days.ago, with_random: false))
+      end
+
       processed, aggregate = parallelize_with_progress(scope) do |media_attachment|
         next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?)
         next if DomainBlock.reject_media?(media_attachment.account.domain)

lib/mastodon/search_cli.rb

@@ -30,7 +30,7 @@ module Mastodon
       changed since the last run. Index upgrades erase index data.
 
       Even if creating or upgrading indices is not necessary, data from the
-      database will be imported into the indices, unless overriden with --no-import.
+      database will be imported into the indices, unless overridden with --no-import.
     LONG_DESC
     def deploy
       if options[:concurrency] < 1

lib/mastodon/version.rb

@@ -5,19 +5,19 @@ module Mastodon
     module_function
 
     def major
-      3
+      4
     end
 
     def minor
-      5
+      0
     end
 
     def patch
-      3
+      0
     end
 
     def flags
-      ''
+      'rc1'
     end
 
     def suffix

lib/paperclip/attachment_extensions.rb

@@ -81,6 +81,9 @@ module Paperclip
     # to respond or don't respond at all and as such minimize the
     # impact of object storage outages on application throughput
     def save
+      # Don't go through Stoplight if we don't have anything object-storage-oriented to do
+      return super if @queued_for_delete.empty? && @queued_for_write.empty? && !dirty?
+
       Stoplight('object-storage') { super }.with_threshold(STOPLIGHT_THRESHOLD).with_cool_off_time(STOPLIGHT_COOLDOWN).with_error_handler do |error, handle|
         if error.is_a?(Seahorse::Client::NetworkingError)
           handle.call(error)

lib/simple_navigation/item_extensions.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module SimpleNavigation
+  module ItemExtensions
+    def url
+      if @url.nil? && @sub_navigation
+        @sub_navigation.items.first.url
+      else
+        @url
+      end
+    end
+  end
+end
+
+SimpleNavigation::Item.prepend(SimpleNavigation::ItemExtensions)

lib/tasks/branding.rake

@@ -0,0 +1,77 @@
+namespace :branding do
+  desc 'Generate necessary graphic assets for branding from source SVG files'
+  task generate: :environment do
+    Rake::Task['branding:generate_app_icons'].invoke
+    Rake::Task['branding:generate_app_badge'].invoke
+    Rake::Task['branding:generate_github_assets'].invoke
+    Rake::Task['branding:generate_mailer_assets'].invoke
+  end
+
+  desc 'Generate PNG icons and logos for e-mail templates'
+  task generate_mailer_assets: :environment do
+    rsvg_convert = Terrapin::CommandLine.new('rsvg-convert', '-h :size --keep-aspect-ratio :input -o :output')
+    output_dest  = Rails.root.join('app', 'javascript', 'images', 'mailer')
+
+    # Displayed size is 64px, at 3x it's 192px
+    Dir[Rails.root.join('app', 'javascript', 'images', 'icons', '*.svg')].each do |path|
+      rsvg_convert.run(input: path, size: 192, output: output_dest.join("#{File.basename(path, '.svg')}.png"))
+    end
+
+    # Displayed size is 34px, at 3x it's 102px
+    rsvg_convert.run(input: Rails.root.join('app', 'javascript', 'images', 'logo-symbol-wordmark.svg'), size: 102, output: output_dest.join('wordmark.png'))
+
+    # Displayed size is 24px, at 3x it's 72px
+    rsvg_convert.run(input: Rails.root.join('app', 'javascript', 'images', 'logo-symbol-icon.svg'), size: 72, output: output_dest.join('logo.png'))
+  end
+
+  desc 'Generate light/dark logotypes for GitHub'
+  task generate_github_assets: :environment do
+    rsvg_convert = Terrapin::CommandLine.new('rsvg-convert', '--stylesheet :stylesheet -h :size --keep-aspect-ratio :input -o :output')
+    output_dest  = Rails.root.join('lib', 'assets')
+
+    rsvg_convert.run(stylesheet: Rails.root.join('lib', 'assets', 'wordmark.dark.css'), input: Rails.root.join('app', 'javascript', 'images', 'logo-symbol-wordmark.svg'), size: 102, output: output_dest.join('wordmark.dark.png'))
+    rsvg_convert.run(stylesheet: Rails.root.join('lib', 'assets', 'wordmark.light.css'), input: Rails.root.join('app', 'javascript', 'images', 'logo-symbol-wordmark.svg'), size: 102, output: output_dest.join('wordmark.light.png'))
+  end
+
+  desc 'Generate favicons and app icons from SVG source files'
+  task generate_app_icons: :environment do
+    favicon_source  = Rails.root.join('app', 'javascript', 'images', 'logo.svg')
+    app_icon_source = Rails.root.join('app', 'javascript', 'images', 'app-icon.svg')
+    output_dest     = Rails.root.join('app', 'javascript', 'icons')
+
+    rsvg_convert = Terrapin::CommandLine.new('rsvg-convert', '-w :size -h :size --keep-aspect-ratio :input -o :output')
+    convert = Terrapin::CommandLine.new('convert', ':input :output')
+
+    favicon_sizes      = [16, 32, 48]
+    apple_icon_sizes   = [57, 60, 72, 76, 114, 120, 144, 152, 167, 180, 1024]
+    android_icon_sizes = [36, 48, 72, 96, 144, 192, 256, 384, 512]
+
+    favicons = []
+
+    favicon_sizes.each do |size|
+      output_path = output_dest.join("favicon-#{size}x#{size}.png")
+      favicons << output_path
+      rsvg_convert.run(size: size, input: favicon_source, output: output_path)
+    end
+
+    convert.run(input: favicons, output: Rails.root.join('public', 'favicon.ico'))
+
+    apple_icon_sizes.each do |size|
+      rsvg_convert.run(size: size, input: app_icon_source, output: output_dest.join("apple-touch-icon-#{size}x#{size}.png"))
+    end
+
+    android_icon_sizes.each do |size|
+      rsvg_convert.run(size: size, input: app_icon_source, output: output_dest.join("android-chrome-#{size}x#{size}.png"))
+    end
+  end
+
+  desc 'Generate badge icon from SVG source files'
+  task generate_app_badge: :environment do
+    rsvg_convert = Terrapin::CommandLine.new('rsvg-convert', '--stylesheet :stylesheet -w :size -h :size --keep-aspect-ratio :input -o :output')
+    badge_source = Rails.root.join('app', 'javascript', 'images', 'logo-symbol-icon.svg')
+    output_dest  = Rails.root.join('public')
+    stylesheet   = Rails.root.join('lib', 'assets', 'wordmark.light.css')
+
+    rsvg_convert.run(stylesheet: stylesheet, input: badge_source, size: 192, output: output_dest.join('badge.png'))
+  end
+end

lib/tasks/emojis.rake

@@ -45,7 +45,7 @@ end
 namespace :emojis do
   desc 'Generate a unicode to filename mapping'
   task :generate do
-    source = 'http://www.unicode.org/Public/emoji/13.1/emoji-test.txt'
+    source = 'http://www.unicode.org/Public/emoji/14.0/emoji-test.txt'
     codes  = []
     dest   = Rails.root.join('app', 'javascript', 'mastodon', 'features', 'emoji', 'emoji_map.json')
 
@@ -69,7 +69,7 @@ namespace :emojis do
       end
     end
 
-    existence_maps = grouped_codes.map { |c| c.index_with { |cc| File.exist?(Rails.root.join('public', 'emoji', codepoints_to_filename(cc) + '.svg')) } }
+    existence_maps = grouped_codes.map { |c| c.index_with { |cc| File.exist?(Rails.root.join('public', 'emoji', "#{codepoints_to_filename(cc)}.svg")) } }
     map = {}
 
     existence_maps.each do |group|

lib/tasks/mastodon.rake

@@ -11,7 +11,7 @@ namespace :mastodon do
     # When the application code gets loaded, it runs `lib/mastodon/redis_configuration.rb`.
     # This happens before application environment configuration and sets REDIS_URL etc.
     # These variables are then used even when REDIS_HOST etc. are changed, so clear them
-    # out so they don't interfer with our new configuration.
+    # out so they don't interfere with our new configuration.
     ENV.delete('REDIS_URL')
     ENV.delete('CACHE_REDIS_URL')
     ENV.delete('SIDEKIQ_REDIS_URL')
@@ -433,9 +433,12 @@ namespace :mastodon do
 
           password = SecureRandom.hex(16)
 
-          user = User.new(admin: true, email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_invite_request_check: true)
+          owner_role = UserRole.find_by(name: 'Owner')
+          user = User.new(email: email, password: password, confirmed_at: Time.now.utc, account_attributes: { username: username }, bypass_invite_request_check: true, role: owner_role)
           user.save(validate: false)
 
+          Setting.site_contact_username = username
+
           prompt.ok "You can login with the password: #{password}"
           prompt.warn 'You can change your password once you login.'
         end

lib/tasks/tests.rake

@@ -38,10 +38,61 @@ namespace :tests do
         puts 'Instance actor does not have a private key'
         exit(1)
       end
+
+      unless Account.find_by(username: 'user', domain: nil).custom_filters.map { |filter| filter.keywords.pluck(:keyword) } == [['test'], ['take']]
+        puts 'CustomFilterKeyword records not created as expected'
+        exit(1)
+      end
+    end
+
+    desc 'Populate the database with test data for 2.4.3'
+    task populate_v2_4_3: :environment do # rubocop:disable Naming/VariableNumber
+      ActiveRecord::Base.connection.execute(<<~SQL)
+        INSERT INTO "custom_filters"
+          (id, account_id, phrase, context, whole_word, irreversible, created_at, updated_at)
+        VALUES
+          (1, 2, 'test', '{ "home", "public" }', true, true, now(), now()),
+          (2, 2, 'take', '{ "home" }', false, false, now(), now());
+
+        -- Orphaned admin action logs
+
+        INSERT INTO "admin_action_logs"
+          (account_id, action, target_type, target_id, created_at, updated_at)
+        VALUES
+          (1, 'destroy', 'Account', 1312, now(), now()),
+          (1, 'destroy', 'User', 1312, now(), now()),
+          (1, 'destroy', 'Report', 1312, now(), now()),
+          (1, 'destroy', 'DomainBlock', 1312, now(), now()),
+          (1, 'destroy', 'EmailDomainBlock', 1312, now(), now()),
+          (1, 'destroy', 'Status', 1312, now(), now()),
+          (1, 'destroy', 'CustomEmoji', 1312, now(), now());
+
+        -- Admin action logs with linked objects
+
+        INSERT INTO "domain_blocks"
+          (id, domain, created_at, updated_at)
+        VALUES
+          (1, 'example.org', now(), now());
+
+        INSERT INTO "email_domain_blocks"
+          (id, domain, created_at, updated_at)
+        VALUES
+          (1, 'example.org', now(), now());
+
+        INSERT INTO "admin_action_logs"
+          (account_id, action, target_type, target_id, created_at, updated_at)
+        VALUES
+          (1, 'destroy', 'Account', 1, now(), now()),
+          (1, 'destroy', 'User', 1, now(), now()),
+          (1, 'destroy', 'DomainBlock', 1312, now(), now()),
+          (1, 'destroy', 'EmailDomainBlock', 1312, now(), now()),
+          (1, 'destroy', 'Status', 1, now(), now()),
+          (1, 'destroy', 'CustomEmoji', 3, now(), now());
+      SQL
     end
 
     desc 'Populate the database with test data for 2.4.0'
-    task populate_v2_4: :environment do
+    task populate_v2_4: :environment do # rubocop:disable Naming/VariableNumber
       ActiveRecord::Base.connection.execute(<<~SQL)
         INSERT INTO "settings"
           (id, thing_type, thing_id, var, value, created_at, updated_at)
@@ -191,18 +242,18 @@ namespace :tests do
         -- custom emoji
 
         INSERT INTO "custom_emojis"
-          (shortcode, created_at, updated_at)
+          (id, shortcode, created_at, updated_at)
         VALUES
-          ('test', now(), now()),
-          ('Test', now(), now()),
-          ('blobcat', now(), now());
+          (1, 'test', now(), now()),
+          (2, 'Test', now(), now()),
+          (3, 'blobcat', now(), now());
 
         INSERT INTO "custom_emojis"
-          (shortcode, domain, uri, created_at, updated_at)
+          (id, shortcode, domain, uri, created_at, updated_at)
         VALUES
-          ('blobcat', 'remote.org', 'https://remote.org/emoji/blobcat', now(), now()),
-          ('blobcat', 'Remote.org', 'https://remote.org/emoji/blobcat', now(), now()),
-          ('Blobcat', 'remote.org', 'https://remote.org/emoji/Blobcat', now(), now());
+          (4, 'blobcat', 'remote.org', 'https://remote.org/emoji/blobcat', now(), now()),
+          (5, 'blobcat', 'Remote.org', 'https://remote.org/emoji/blobcat', now(), now()),
+          (6, 'Blobcat', 'remote.org', 'https://remote.org/emoji/Blobcat', now(), now());
 
         -- favourites