bgme/mastodon
1
0
Fork 0
mirror of https://github.com/yingziwu/mastodon.git synced 2026-02-19 00:33:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge remote-tracking branch 'upstream/master'

Browse source
This commit is contained in:
wuyingren 2020-05-14 16:32:30 +08:00
commit 71cec2d153
922 changed files with 9431 additions and 4194 deletions
Download patch file Download diff file Expand all files Collapse all files

4
lib/cli.rb
View file

@ -11,6 +11,7 @@ require_relative 'mastodon/statuses_cli'
require_relative 'mastodon/domains_cli'
require_relative 'mastodon/preview_cards_cli'
require_relative 'mastodon/cache_cli'
require_relative 'mastodon/upgrade_cli'
require_relative 'mastodon/version'
module Mastodon
@ -49,6 +50,9 @@ module Mastodon
desc 'cache SUBCOMMAND ...ARGS', 'Manage cache'
subcommand 'cache', Mastodon::CacheCLI
desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities'
subcommand 'upgrade', Mastodon::UpgradeCLI
option :dry_run, type: :boolean
desc 'self-destruct', 'Erase the server from the federation'
long_desc <<~LONG_DESC

4
lib/mastodon/cli_helper.rb
View file

@ -10,6 +10,10 @@ Paperclip.options[:log] = false
module Mastodon
module CLIHelper
def dry_run?
options[:dry_run]
end
def create_progress_bar(total = nil)
ProgressBar.create(total: total, format: '%c/%u |%b%i| %e')
end

44
lib/mastodon/emoji_cli.rb
View file

@ -23,7 +23,7 @@ module Mastodon
Existing emoji will be skipped unless the --overwrite option
is provided, in which case they will be overwritten.
You can specifiy a --category under which the emojis will be
You can specify a --category under which the emojis will be
grouped together.
With the --prefix option, a prefix can be added to all
@ -72,6 +72,48 @@ module Mastodon
say("Imported #{imported}, skipped #{skipped}, failed to import #{failed}", color(imported, skipped, failed))
end
option :category
option :overwrite, type: :boolean
desc 'export PATH', 'Export emoji to a TAR GZIP archive at PATH'
long_desc <<-LONG_DESC
Exports custom emoji to 'export.tar.gz' at PATH.
The --category option dumps only the specified category.
If this option is not specified, all emoji will be exported.
The --overwrite option will overwrite an existing archive.
LONG_DESC
def export(path)
exported = 0
category = CustomEmojiCategory.find_by(name: options[:category])
export_file_name = File.join(path, 'export.tar.gz')
if File.file?(export_file_name) && !options[:overwrite]
say("Archive already exists! Use '--overwrite' to overwrite it!")
exit 1
end
if category.nil? && options[:category]
say("Unable to find category '#{options[:category]}'!")
exit 1
end
File.open(export_file_name, 'wb') do |file|
Zlib::GzipWriter.wrap(file) do |gzip|
Gem::Package::TarWriter.new(gzip) do |tar|
scope = !options[:category] || category.nil? ? CustomEmoji.local : category.emojis
scope.find_each do |emoji|
say("Adding '#{emoji.shortcode}'...")
tar.add_file_simple(emoji.shortcode + File.extname(emoji.image_file_name), 0o644, emoji.image_file_size) do |io|
io.write Paperclip.io_adapters.for(emoji.image).read
exported += 1
end
end
end
end
end
say("Exported #{exported}")
end
option :remote_only, type: :boolean
desc 'purge', 'Remove all custom emoji'
long_desc <<-LONG_DESC

37
lib/mastodon/media_cli.rb
View file

@ -85,7 +85,9 @@ module Mastodon
record_map = preload_records_from_mixed_objects(objects)
objects.each do |object|
path_segments = object.key.split('/')
path_segments = object.key.split('/')
path_segments.delete('cache')
model_name = path_segments.first.classify
attachment_name = path_segments[1].singularize
record_id = path_segments[2..-2].join.to_i
@ -120,8 +122,11 @@ module Mastodon
Find.find(File.join(*[root_path, prefix].compact)) do |path|
next if File.directory?(path)
key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
path_segments = key.split(File::SEPARATOR)
key = path.gsub("#{root_path}#{File::SEPARATOR}", '')
path_segments = key.split(File::SEPARATOR)
path_segments.delete('cache')
model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i
attachment_name = path_segments[1].singularize
@ -139,7 +144,14 @@ module Mastodon
begin
size = File.size(path)
File.delete(path) unless options[:dry_run]
unless options[:dry_run]
File.delete(path)
begin
FileUtils.rmdir(File.dirname(path), parents: true)
rescue Errno::ENOTEMPTY
# OK
end
end
reclaimed_bytes += size
removed += 1
@ -186,7 +198,7 @@ module Mastodon
if options[:status]
scope = MediaAttachment.where(status_id: options[:status])
elsif options[:account]
username, domain = username.split('@')
username, domain = options[:account].split('@')
account = Account.find_remote(username, domain)
if account.nil?
@ -205,7 +217,7 @@ module Mastodon
next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?)
unless options[:dry_run]
media_attachment.reset_file!
media_attachment.file_remote_url = media_attachment.remote_url
media_attachment.save
end
@ -229,10 +241,13 @@ module Mastodon
desc 'lookup URL', 'Lookup where media is displayed by passing a media URL'
def lookup(url)
path = Addressable::URI.parse(url).path
path = Addressable::URI.parse(url).path
path_segments = path.split('/')[2..-1]
model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i
path_segments.delete('cache')
model_name = path_segments.first.classify
record_id = path_segments[2..-2].join.to_i
unless PRELOAD_MODEL_WHITELIST.include?(model_name)
say("Cannot find corresponding model: #{model_name}", :red)
@ -276,7 +291,9 @@ module Mastodon
preload_map = Hash.new { |hash, key| hash[key] = [] }
objects.map do |object|
segments = object.key.split('/').first
segments = object.key.split('/')
segments.delete('cache')
model_name = segments.first.classify
record_id = segments[2..-2].join.to_i

148
lib/mastodon/upgrade_cli.rb Normal file
View file

@ -0,0 +1,148 @@
# frozen_string_literal: true
require_relative '../../config/boot'
require_relative '../../config/environment'
require_relative 'cli_helper'
module Mastodon
class UpgradeCLI < Thor
include CLIHelper
def self.exit_on_failure?
true
end
CURRENT_STORAGE_SCHEMA_VERSION = 1
option :dry_run, type: :boolean, default: false
option :verbose, type: :boolean, default: false, aliases: [:v]
desc 'storage-schema', 'Upgrade storage schema of various file attachments to the latest version'
long_desc <<~LONG_DESC
Iterates over every file attachment of every record and, if its storage schema is outdated, performs the
necessary upgrade to the latest one. In practice this means e.g. moving files to different directories.
Will most likely take a long time.
LONG_DESC
def storage_schema
progress = create_progress_bar(nil)
dry_run = dry_run? ? ' (DRY RUN)' : ''
records = 0
klasses = [
Account,
CustomEmoji,
MediaAttachment,
PreviewCard,
]
klasses.each do |klass|
attachment_names = klass.attachment_definitions.keys
klass.find_each do |record|
attachment_names.each do |attachment_name|
attachment = record.public_send(attachment_name)
next if attachment.blank? || attachment.storage_schema_version >= CURRENT_STORAGE_SCHEMA_VERSION
attachment.styles.each_key do |style|
case Paperclip::Attachment.default_options[:storage]
when :s3
upgrade_storage_s3(progress, attachment, style)
when :fog
upgrade_storage_fog(progress, attachment, style)
when :filesystem
upgrade_storage_filesystem(progress, attachment, style)
end
progress.increment
end
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
end
if record.changed?
record.save unless dry_run?
records += 1
end
end
end
progress.total = progress.progress
progress.finish
say("Upgraded storage schema of #{records} records#{dry_run}", :green, true)
end
private
def upgrade_storage_s3(progress, attachment, style)
previous_storage_schema_version = attachment.storage_schema_version
object = attachment.s3_object(style)
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
upgraded_path = attachment.path(style)
if upgraded_path != object.key && object.exists?
progress.log("Moving #{object.key} to #{upgraded_path}") if options[:verbose]
begin
object.move_to(upgraded_path) unless dry_run?
rescue => e
progress.log(pastel.red("Error processing #{object.key}: #{e}"))
end
end
# Because we move files style-by-style, it's important to restore
# previous version at the end. The upgrade will be recorded after
# all styles are updated
attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
end
def upgrade_storage_fog(_progress, _attachment, _style)
say('The fog storage driver is not supported for this operation at this time', :red)
exit(1)
end
def upgrade_storage_filesystem(progress, attachment, style)
previous_storage_schema_version = attachment.storage_schema_version
previous_path = attachment.path(style)
attachment.instance_write(:storage_schema_version, CURRENT_STORAGE_SCHEMA_VERSION)
upgraded_path = attachment.path(style)
if upgraded_path != previous_path && File.exist?(previous_path)
progress.log("Moving #{previous_path} to #{upgraded_path}") if options[:verbose]
begin
unless dry_run?
FileUtils.mkdir_p(File.dirname(upgraded_path))
FileUtils.mv(previous_path, upgraded_path)
begin
FileUtils.rmdir(File.dirname(previous_path), parents: true)
rescue Errno::ENOTEMPTY
# OK
end
end
rescue => e
progress.log(pastel.red("Error processing #{previous_path}: #{e}"))
unless dry_run?
begin
FileUtils.rmdir(File.dirname(upgraded_path), parents: true)
rescue Errno::ENOTEMPTY
# OK
end
end
end
end
# Because we move files style-by-style, it's important to restore
# previous version at the end. The upgrade will be recorded after
# all styles are updated
attachment.instance_write(:storage_schema_version, previous_storage_schema_version)
end
end
end

2
lib/mastodon/version.rb
View file

@ -13,7 +13,7 @@ module Mastodon
end
def patch
3
4
end
def flags

14
lib/paperclip/attachment_extensions.rb
View file

@ -14,15 +14,13 @@ module Paperclip
end
end
def reprocess_original!
old_original_path = path(:original)
reprocess!(:original)
new_original_path = path(:original)
def storage_schema_version
instance_read(:storage_schema_version) || 0
end
if new_original_path != old_original_path
@queued_for_delete << old_original_path
flush_deletes
end
def assign_attributes
super
instance_write(:storage_schema_version, 1)
end
def variant?(other_filename)

2
lib/tasks/emojis.rake
View file

@ -15,7 +15,7 @@ end
namespace :emojis do
desc 'Generate a unicode to filename mapping'
task :generate do
source = 'http://www.unicode.org/Public/emoji/11.0/emoji-test.txt'
source = 'http://www.unicode.org/Public/emoji/12.0/emoji-test.txt'
codes = []
dest = Rails.root.join('app', 'javascript', 'mastodon', 'features', 'emoji', 'emoji_map.json')