autonomic-cooperative/hometown
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge tag 'v3.4.0' into hometown-dev

Browse Source
This commit is contained in:
Darius Kazemi
2021-05-17 13:48:27 -07:00
parent 0e3f729c33 4c7efdba40
commit f57cda3855
897 changed files with 26918 additions and 14941 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
lib/action_dispatch/cookie_jar_extensions.rb Normal file
View File

@ -0,0 +1,25 @@
# frozen_string_literal: true
module ActionDispatch
module CookieJarExtensions
private
# Monkey-patch ActionDispatch to serve secure cookies to Tor Hidden Service
# users. Otherwise, ActionDispatch would drop the cookie over HTTP.
def write_cookie?(*)
request.host.end_with?('.onion') || super
end
end
end
ActionDispatch::Cookies::CookieJar.prepend(ActionDispatch::CookieJarExtensions)
module Rack
module SessionPersistedExtensions
def security_matches?(request, options)
request.host.end_with?('.onion') || super
end
end
end
Rack::Session::Abstract::Persisted.prepend(Rack::SessionPersistedExtensions)

44
lib/active_record/batches.rb Normal file
View File

@ -0,0 +1,44 @@
# frozen_string_literal: true
module ActiveRecord
module Batches
def pluck_each(*column_names)
relation = self
options = column_names.extract_options!
flatten = column_names.size == 1
batch_limit = options[:batch_limit] || 1_000
order = options[:order] || :asc
column_names.unshift(primary_key)
relation = relation.reorder(batch_order(order)).limit(batch_limit)
relation.skip_query_cache!
batch_relation = relation
loop do
batch = batch_relation.pluck(*column_names)
break if batch.empty?
primary_key_offset = batch.last[0]
batch.each do |record|
if flatten
yield record[1]
else
yield record[1..-1]
end
end
break if batch.size < batch_limit
batch_relation = relation.where(
predicate_builder[primary_key, primary_key_offset, order == :desc ? :lt : :gt]
)
end
end
end
end

20
lib/active_record/database_tasks_extensions.rb Normal file
View File

@ -0,0 +1,20 @@
# frozen_string_literal: true
require_relative '../mastodon/snowflake'
module ActiveRecord
module Tasks
module DatabaseTasks
original_load_schema = instance_method(:load_schema)
define_method(:load_schema) do |db_config, *args|
ActiveRecord::Base.establish_connection(db_config)
Mastodon::Snowflake.define_timestamp_id
original_load_schema.bind(self).call(db_config, *args)
Mastodon::Snowflake.ensure_id_sequences_exist
end
end
end
end

26
lib/enumerable.rb Normal file
View File

@ -0,0 +1,26 @@
# frozen_string_literal: true
module Enumerable
# TODO: Remove this once stop to support Ruby 2.6
if RUBY_VERSION < '2.7.0'
def filter_map
if block_given?
result = []
each do |element|
res = yield element
result << res if res
end
result
else
Enumerator.new do |yielder|
result = []
each do |element|
res = yielder.yield element
result << res if res
end
result
end
end
end
end
end

27
lib/exceptions.rb Normal file
View File

@ -0,0 +1,27 @@
# frozen_string_literal: true
module Mastodon
class Error < StandardError; end
class NotPermittedError < Error; end
class ValidationError < Error; end
class HostValidationError < ValidationError; end
class LengthValidationError < ValidationError; end
class DimensionsValidationError < ValidationError; end
class StreamValidationError < ValidationError; end
class RaceConditionError < Error; end
class RateLimitExceededError < Error; end
class UnexpectedResponseError < Error
attr_reader :response
def initialize(response = nil)
@response = response
if response.respond_to? :uri
super("#{response.uri} returned code #{response.code}")
else
super
end
end
end
end

1
lib/json_ld/identity.rb
View File

@ -1,4 +1,3 @@
# -*- encoding: utf-8 -*-
# frozen_string_literal: true
# This file generated automatically from http://w3id.org/identity/v1
require 'json/ld'

1
lib/json_ld/security.rb
View File

@ -1,4 +1,3 @@
# -*- encoding: utf-8 -*-
# frozen_string_literal: true
# This file generated automatically from http://w3id.org/security/v1
require 'json/ld'

2
lib/mastodon/accounts_cli.rb
View File

@ -402,7 +402,7 @@ module Mastodon
exit(1)
end
parallelize_with_progress(target_account.followers.local) do |account|
processed, = parallelize_with_progress(target_account.followers.local) do |account|
UnfollowService.new.call(account, target_account)
end

4
lib/mastodon/cli_helper.rb
View File

@ -25,7 +25,9 @@ module Mastodon
exit(1)
end
ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] + 1
db_config = ActiveRecord::Base.configurations[Rails.env].dup
db_config['pool'] = options[:concurrency] + 1
ActiveRecord::Base.establish_connection(db_config)
progress = create_progress_bar(scope.count)
pool = Concurrent::FixedThreadPool.new(options[:concurrency])

2
lib/mastodon/domains_cli.rb
View File

@ -93,7 +93,7 @@ module Mastodon
work_unit = ->(domain) do
next if stats.key?(domain)
next if options[:exclude_suspended] && domain.match(blocked_domains)
next if options[:exclude_suspended] && domain.match?(blocked_domains)
stats[domain] = nil

2
lib/mastodon/email_domain_blocks_cli.rb
View File

@ -113,7 +113,7 @@ module Mastodon
result = entry.destroy
if result
processed += 1 + children_count
processed += children_count + 1
else
say("#{domain} could not be unblocked.", :red)
failed += 1

9
lib/mastodon/emoji_cli.rb
View File

@ -43,8 +43,13 @@ module Mastodon
tar.each do |entry|
next unless entry.file? && entry.full_name.end_with?('.png')
shortcode = [options[:prefix], File.basename(entry.full_name, '.*'), options[:suffix]].compact.join
custom_emoji = CustomEmoji.local.find_by(shortcode: shortcode)
filename = File.basename(entry.full_name, '.*')
# Skip macOS shadow files
next if filename.start_with?('._')
shortcode = [options[:prefix], filename, options[:suffix]].compact.join
custom_emoji = CustomEmoji.local.find_by("LOWER(shortcode) = ?", shortcode.downcase)
if custom_emoji && !options[:overwrite]
skipped += 1

24
lib/mastodon/maintenance_cli.rb
View File

@ -14,7 +14,7 @@ module Mastodon
end
MIN_SUPPORTED_VERSION = 2019_10_01_213028
MAX_SUPPORTED_VERSION = 2020_12_18_054746
MAX_SUPPORTED_VERSION = 2021_05_07_001928
# Stubs to enjoy ActiveRecord queries while not depending on a particular
# version of the code/database
@ -42,6 +42,8 @@ module Mastodon
class CustomEmojiCategory < ApplicationRecord; end
class Bookmark < ApplicationRecord; end
class WebauthnCredential < ApplicationRecord; end
class FollowRecommendationSuppression < ApplicationRecord; end
class CanonicalEmailBlock < ApplicationRecord; end
class PreviewCard < ApplicationRecord
self.inheritance_column = false
@ -88,6 +90,7 @@ module Mastodon
]
owned_classes << AccountDeletionRequest if ActiveRecord::Base.connection.table_exists?(:account_deletion_requests)
owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes)
owned_classes << FollowRecommendationSuppression if ActiveRecord::Base.connection.table_exists?(:follow_recommendation_suppressions)
owned_classes.each do |klass|
klass.where(account_id: other_account.id).find_each do |record|
@ -111,6 +114,12 @@ module Mastodon
end
end
end
if ActiveRecord::Base.connection.table_exists?(:canonical_email_blocks)
CanonicalEmailBlock.where(reference_account_id: other_account.id).find_each do |record|
record.update_attribute(:reference_account_id, id)
end
end
end
end
@ -142,7 +151,6 @@ module Mastodon
@prompt.warn 'Please make sure to stop Mastodon and have a backup.'
exit(1) unless @prompt.yes?('Continue?')
deduplicate_accounts!
deduplicate_users!
deduplicate_account_domain_blocks!
deduplicate_account_identity_proofs!
@ -157,9 +165,11 @@ module Mastodon
deduplicate_media_attachments!
deduplicate_preview_cards!
deduplicate_statuses!
deduplicate_accounts!
deduplicate_tags!
deduplicate_webauthn_credentials!
Scenic.database.refresh_materialized_view('instances', concurrently: true, cascade: false) if ActiveRecord::Migrator.current_version >= 2020_12_06_004238
Rails.cache.clear
@prompt.say 'Finished!'
@ -188,6 +198,11 @@ module Mastodon
else
ActiveRecord::Base.connection.add_index :accounts, "lower (username), COALESCE(lower(domain), '')", name: 'index_accounts_on_username_and_domain_lower', unique: true
end
@prompt.say 'Reindexing textual indexes on accounts…'
ActiveRecord::Base.connection.execute('REINDEX INDEX search_index;')
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_uri;')
ActiveRecord::Base.connection.execute('REINDEX INDEX index_accounts_on_url;')
end
def deduplicate_users!
@ -460,6 +475,11 @@ module Mastodon
@prompt.say 'Restoring tags indexes…'
ActiveRecord::Base.connection.add_index :tags, 'lower((name)::text)', name: 'index_tags_on_name_lower', unique: true
if ActiveRecord::Base.connection.indexes(:tags).any? { |i| i.name == 'index_tags_on_name_lower_btree' }
@prompt.say 'Reindexing textual indexes on tags…'
ActiveRecord::Base.connection.execute('REINDEX INDEX index_tags_on_name_lower_btree;')
end
end
def deduplicate_webauthn_credentials!

2
lib/mastodon/media_cli.rb
View File

@ -326,7 +326,7 @@ module Mastodon
end
preload_map.each_with_object({}) do |(model_name, record_ids), model_map|
model_map[model_name] = model_name.constantize.where(id: record_ids).each_with_object({}) { |record, record_map| record_map[record.id] = record }
model_map[model_name] = model_name.constantize.where(id: record_ids).index_by(&:id)
end
end
end

106
lib/mastodon/migration_helpers.rb
View File

@ -41,42 +41,18 @@
module Mastodon
module MigrationHelpers
# Stub for Database.postgresql? from GitLab
def self.postgresql?
ActiveRecord::Base.configurations[Rails.env]['adapter'].casecmp('postgresql').zero?
end
# Stub for Database.mysql? from GitLab
def self.mysql?
ActiveRecord::Base.configurations[Rails.env]['adapter'].casecmp('mysql2').zero?
end
# Model that can be used for querying permissions of a SQL user.
class Grant < ActiveRecord::Base
self.table_name =
if Mastodon::MigrationHelpers.postgresql?
'information_schema.role_table_grants'
else
'mysql.user'
end
self.table_name = 'information_schema.role_table_grants'
def self.scope_to_current_user
if Mastodon::MigrationHelpers.postgresql?
where('grantee = user')
else
where("CONCAT(User, '@', Host) = current_user()")
end
where('grantee = user')
end
# Returns true if the current user can create and execute triggers on the
# given table.
def self.create_and_execute_trigger?(table)
priv =
if Mastodon::MigrationHelpers.postgresql?
where(privilege_type: 'TRIGGER', table_name: table)
else
where(Trigger_priv: 'Y')
end
priv = where(privilege_type: 'TRIGGER', table_name: table)
priv.scope_to_current_user.any?
end
@ -119,7 +95,7 @@ module Mastodon
allow_null: options[:null]
)
else
add_column(table_name, column_name, :datetime_with_timezone, options)
add_column(table_name, column_name, :datetime_with_timezone, **options)
end
end
end
@ -141,12 +117,10 @@ module Mastodon
'in the body of your migration class'
end
if MigrationHelpers.postgresql?
options = options.merge({ algorithm: :concurrently })
disable_statement_timeout
end
options = options.merge({ algorithm: :concurrently })
disable_statement_timeout
add_index(table_name, column_name, options)
add_index(table_name, column_name, **options)
end
# Removes an existed index, concurrently when supported
@ -170,7 +144,7 @@ module Mastodon
disable_statement_timeout
end
remove_index(table_name, options.merge({ column: column_name }))
remove_index(table_name, **options.merge({ column: column_name }))
end
# Removes an existing index, concurrently when supported
@ -194,13 +168,11 @@ module Mastodon
disable_statement_timeout
end
remove_index(table_name, options.merge({ name: index_name }))
remove_index(table_name, **options.merge({ name: index_name }))
end
# Only available on Postgresql >= 9.2
def supports_drop_index_concurrently?
return false unless MigrationHelpers.postgresql?
version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
version >= 90200
@ -226,13 +198,7 @@ module Mastodon
# While MySQL does allow disabling of foreign keys it has no equivalent
# of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall
# back to the normal foreign key procedure.
if MigrationHelpers.mysql?
return add_foreign_key(source, target,
column: column,
on_delete: on_delete)
else
on_delete = 'SET NULL' if on_delete == :nullify
end
on_delete = 'SET NULL' if on_delete == :nullify
disable_statement_timeout
@ -270,7 +236,7 @@ module Mastodon
# the database. Disable the session's statement timeout to ensure
# migrations don't get killed prematurely. (PostgreSQL only)
def disable_statement_timeout
execute('SET statement_timeout TO 0') if MigrationHelpers.postgresql?
execute('SET statement_timeout TO 0')
end
# Updates the value of a column in batches.
@ -319,7 +285,7 @@ module Mastodon
count_arel = table.project(Arel.star.count.as('count'))
count_arel = yield table, count_arel if block_given?
total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i
total = exec_query(count_arel.to_sql).to_ary.first['count'].to_i
return if total == 0
end
@ -335,7 +301,7 @@ module Mastodon
start_arel = table.project(table[:id]).order(table[:id].asc).take(1)
start_arel = yield table, start_arel if block_given?
first_row = exec_query(start_arel.to_sql).to_hash.first
first_row = exec_query(start_arel.to_sql).to_ary.first
# In case there are no rows but we didn't catch it in the estimated size:
return unless first_row
start_id = first_row['id'].to_i
@ -356,7 +322,7 @@ module Mastodon
.skip(batch_size)
stop_arel = yield table, stop_arel if block_given?
stop_row = exec_query(stop_arel.to_sql).to_hash.first
stop_row = exec_query(stop_arel.to_sql).to_ary.first
update_arel = Arel::UpdateManager.new
.table(table)
@ -487,11 +453,7 @@ module Mastodon
# If we were in the middle of update_column_in_batches, we should remove
# the old column and start over, as we have no idea where we were.
if column_for(table, new)
if MigrationHelpers.postgresql?
remove_rename_triggers_for_postgresql(table, trigger_name)
else
remove_rename_triggers_for_mysql(trigger_name)
end
remove_rename_triggers_for_postgresql(table, trigger_name)
remove_column(table, new)
end
@ -510,7 +472,7 @@ module Mastodon
col_opts[:limit] = old_col.limit
end
add_column(table, new, new_type, col_opts)
add_column(table, new, new_type, **col_opts)
# We set the default value _after_ adding the column so we don't end up
# updating any existing data with the default value. This isn't
@ -521,13 +483,8 @@ module Mastodon
quoted_old = quote_column_name(old)
quoted_new = quote_column_name(new)
if MigrationHelpers.postgresql?
install_rename_triggers_for_postgresql(trigger_name, quoted_table,
quoted_old, quoted_new)
else
install_rename_triggers_for_mysql(trigger_name, quoted_table,
quoted_old, quoted_new)
end
install_rename_triggers_for_postgresql(trigger_name, quoted_table,
quoted_old, quoted_new)
update_column_in_batches(table, new, Arel::Table.new(table)[old])
@ -553,10 +510,10 @@ module Mastodon
new_pk_index_name = "index_#{table}_on_#{column}_cm"
unless indexes_for(table, column).find{|i| i.name == old_pk_index_name}
add_concurrent_index(table, [temp_column], {
add_concurrent_index(table, [temp_column],
unique: true,
name: new_pk_index_name
})
)
end
end
end
@ -685,11 +642,7 @@ module Mastodon
check_trigger_permissions!(table)
if MigrationHelpers.postgresql?
remove_rename_triggers_for_postgresql(table, trigger_name)
else
remove_rename_triggers_for_mysql(trigger_name)
end
remove_rename_triggers_for_postgresql(table, trigger_name)
remove_column(table, old)
end
@ -810,7 +763,7 @@ module Mastodon
options[:using] = index.using if index.using
options[:where] = index.where if index.where
add_concurrent_index(table, new_columns, options)
add_concurrent_index(table, new_columns, **options)
end
end
@ -844,18 +797,9 @@ module Mastodon
quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s)
quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s)
if MigrationHelpers.mysql?
locate = Arel::Nodes::NamedFunction
.new('locate', [quoted_pattern, column])
insert_in_place = Arel::Nodes::NamedFunction
.new('insert', [column, locate, pattern.size, quoted_replacement])
Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql)
else
replace = Arel::Nodes::NamedFunction
.new("regexp_replace", [column, quoted_pattern, quoted_replacement])
Arel::Nodes::SqlLiteral.new(replace.to_sql)
end
replace = Arel::Nodes::NamedFunction
.new("regexp_replace", [column, quoted_pattern, quoted_replacement])
Arel::Nodes::SqlLiteral.new(replace.to_sql)
end
def remove_foreign_key_without_error(*args)

14
lib/mastodon/redis_config.rb
View File

@ -22,11 +22,21 @@ end
setup_redis_env_url
setup_redis_env_url(:cache, false)
setup_redis_env_url(:sidekiq, false)
namespace = ENV.fetch('REDIS_NAMESPACE', nil)
cache_namespace = namespace ? namespace + '_cache' : 'cache'
namespace = ENV.fetch('REDIS_NAMESPACE', nil)
cache_namespace = namespace ? namespace + '_cache' : 'cache'
sidekiq_namespace = namespace
REDIS_CACHE_PARAMS = {
driver: :hiredis,
url: ENV['CACHE_REDIS_URL'],
expires_in: 10.minutes,
namespace: cache_namespace,
}.freeze
REDIS_SIDEKIQ_PARAMS = {
driver: :hiredis,
url: ENV['SIDEKIQ_REDIS_URL'],
namespace: sidekiq_namespace,
}.freeze

4
lib/mastodon/search_cli.rb
View File

@ -53,7 +53,9 @@ module Mastodon
index.specification.lock!
end
ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] + 1
db_config = ActiveRecord::Base.configurations[Rails.env].dup
db_config['pool'] = options[:concurrency] + 1
ActiveRecord::Base.establish_connection(db_config)
pool = Concurrent::FixedThreadPool.new(options[:concurrency])
added = Concurrent::AtomicFixnum.new(0)

2
lib/mastodon/version.rb
View File

@ -9,7 +9,7 @@ module Mastodon
end
def minor
3
4
end
def patch

6
lib/paperclip/attachment_extensions.rb
View File

@ -2,6 +2,10 @@
module Paperclip
module AttachmentExtensions
def meta
instance_read(:meta)
end
# We overwrite this method to support delayed processing in
# Sidekiq. Since we process the original file to reduce disk
# usage, and we still want to generate thumbnails straight
@ -27,7 +31,7 @@ module Paperclip
return true if original_filename == other_filename
return false if original_filename.nil?
formats = styles.values.map(&:format).compact
formats = styles.values.filter_map(&:format)
return false if formats.empty?

6
lib/paperclip/color_extractor.rb
View File

@ -55,7 +55,7 @@ module Paperclip
# If we don't have enough colors for accent and foreground, generate
# new ones by manipulating the background color
(2 - foreground_colors.size).times do |i|
foreground_colors << lighten_or_darken(background_color, 35 + (15 * i))
foreground_colors << lighten_or_darken(background_color, 35 + (i * 15))
end
# We want the color with the highest contrast to background to be the foreground one,
@ -142,12 +142,12 @@ module Paperclip
g = 0.0
b = 0.0
if s == 0.0
if s.zero?
r = l.to_f
g = l.to_f
b = l.to_f # achromatic
else
q = l < 0.5 ? l * (1 + s) : l + s - l * s
q = l < 0.5 ? l * (s + 1) : l + s - l * s
p = 2 * l - q
r = hue_to_rgb(p, q, h + 1 / 3.0)
g = hue_to_rgb(p, q, h)

13
lib/paperclip/gif_transcoder.rb
View File

@ -100,16 +100,19 @@ end
module Paperclip
# This transcoder is only to be used for the MediaAttachment model
# to convert animated gifs to webm
# to convert animated GIFs to videos
class GifTranscoder < Paperclip::Processor
def make
return File.open(@file.path) unless needs_convert?
final_file = Paperclip::Transcoder.make(file, options, attachment)
attachment.instance.file_file_name = File.basename(attachment.instance.file_file_name, '.*') + '.mp4'
attachment.instance.file_content_type = 'video/mp4'
attachment.instance.type = MediaAttachment.types[:gifv]
if options[:style] == :original
attachment.instance.file_file_name = File.basename(attachment.instance.file_file_name, '.*') + '.mp4'
attachment.instance.file_content_type = 'video/mp4'
attachment.instance.type = MediaAttachment.types[:gifv]
end
final_file
end
@ -117,7 +120,7 @@ module Paperclip
private
def needs_convert?
options[:style] == :original && GifReader.animated?(file.path)
GifReader.animated?(file.path)
end
end
end

14
lib/paperclip/image_extractor.rb
View File

@ -31,21 +31,17 @@ module Paperclip
private
def extract_image_from_file!
::Av.logger = Paperclip.logger
cli = ::Av.cli
dst = Tempfile.new([File.basename(@file.path, '.*'), '.png'])
dst.binmode
cli.add_source(@file.path)
cli.add_destination(dst.path)
cli.add_output_param loglevel: 'fatal'
begin
cli.run
rescue Cocaine::ExitStatusError, ::Av::CommandError
command = Terrapin::CommandLine.new('ffmpeg', '-i :source -loglevel :loglevel -y :destination', logger: Paperclip.logger)
command.run(source: @file.path, destination: dst.path, loglevel: 'fatal')
rescue Terrapin::ExitStatusError
dst.close(true)
return nil
rescue Terrapin::CommandNotFoundError
raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.'
end
dst

37
lib/paperclip/schema_extensions.rb Normal file
View File

@ -0,0 +1,37 @@
# frozen_string_literal: true
# Monkey-patch various Paperclip methods for Ruby 3.0 compatibility
module Paperclip
module Schema
module StatementsExtensions
def add_attachment(table_name, *attachment_names)
raise ArgumentError, 'Please specify attachment name in your add_attachment call in your migration.' if attachment_names.empty?
options = attachment_names.extract_options!
attachment_names.each do |attachment_name|
COLUMNS.each_pair do |column_name, column_type|
column_options = options.merge(options[column_name.to_sym] || {})
add_column(table_name, "#{attachment_name}_#{column_name}", column_type, **column_options)
end
end
end
end
module TableDefinitionExtensions
def attachment(*attachment_names)
options = attachment_names.extract_options!
attachment_names.each do |attachment_name|
COLUMNS.each_pair do |column_name, column_type|
column_options = options.merge(options[column_name.to_sym] || {})
column("#{attachment_name}_#{column_name}", column_type, **column_options)
end
end
end
end
end
end
Paperclip::Schema::Statements.prepend(Paperclip::Schema::StatementsExtensions)
Paperclip::Schema::TableDefinition.prepend(Paperclip::Schema::TableDefinitionExtensions)

102
lib/paperclip/transcoder.rb Normal file
View File

@ -0,0 +1,102 @@
# frozen_string_literal: true
module Paperclip
# This transcoder is only to be used for the MediaAttachment model
# to check when uploaded videos are actually gifv's
class Transcoder < Paperclip::Processor
def initialize(file, options = {}, attachment = nil)
super
@current_format = File.extname(@file.path)
@basename = File.basename(@file.path, @current_format)
@format = options[:format]
@time = options[:time] || 3
@passthrough_options = options[:passthrough_options]
@convert_options = options[:convert_options].dup
end
def make
metadata = VideoMetadataExtractor.new(@file.path)
unless metadata.valid?
log("Unsupported file #{@file.path}")
return File.open(@file.path)
end
update_attachment_type(metadata)
update_options_from_metadata(metadata)
destination = Tempfile.new([@basename, @format ? ".#{@format}" : ''])
destination.binmode
@output_options = @convert_options[:output]&.dup || {}
@input_options = @convert_options[:input]&.dup || {}
case @format.to_s
when /jpg$/, /jpeg$/, /png$/, /gif$/
@input_options['ss'] = @time
@output_options['f'] = 'image2'
@output_options['vframes'] = 1
when 'mp4'
@output_options['acodec'] = 'aac'
@output_options['strict'] = 'experimental'
end
command_arguments, interpolations = prepare_command(destination)
begin
command = Terrapin::CommandLine.new('ffmpeg', command_arguments.join(' '), logger: Paperclip.logger)
command.run(interpolations)
rescue Terrapin::ExitStatusError => e
raise Paperclip::Error, "Error while transcoding #{@basename}: #{e}"
rescue Terrapin::CommandNotFoundError
raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffmpeg` command. Please install ffmpeg.'
end
destination
end
private
def prepare_command(destination)
command_arguments = ['-nostdin']
interpolations = {}
interpolation_keys = 0
@input_options.each_pair do |key, value|
interpolation_key = interpolation_keys
command_arguments << "-#{key} :#{interpolation_key}"
interpolations[interpolation_key] = value
interpolation_keys += 1
end
command_arguments << '-i :source'
interpolations[:source] = @file.path
@output_options.each_pair do |key, value|
interpolation_key = interpolation_keys
command_arguments << "-#{key} :#{interpolation_key}"
interpolations[interpolation_key] = value
interpolation_keys += 1
end
command_arguments << '-y :destination'
interpolations[:destination] = destination.path
[command_arguments, interpolations]
end
def update_options_from_metadata(metadata)
return unless @passthrough_options && @passthrough_options[:video_codecs].include?(metadata.video_codec) && @passthrough_options[:audio_codecs].include?(metadata.audio_codec) && @passthrough_options[:colorspaces].include?(metadata.colorspace)
@format = @passthrough_options[:options][:format] || @format
@time = @passthrough_options[:options][:time] || @time
@convert_options = @passthrough_options[:options][:convert_options].dup
end
def update_attachment_type(metadata)
@attachment.instance.type = MediaAttachment.types[:gifv] unless metadata.audio_codec
end
end
end

14
lib/paperclip/transcoder_extensions.rb
View File

@ -1,14 +0,0 @@
# frozen_string_literal: true
module Paperclip
module TranscoderExtensions
# Prevent the transcoder from modifying our meta hash
def initialize(file, options = {}, attachment = nil)
meta_value = attachment&.instance_read(:meta)
super
attachment&.instance_write(:meta, meta_value)
end
end
end
Paperclip::Transcoder.prepend(Paperclip::TranscoderExtensions)

58
lib/paperclip/validation_extensions.rb Normal file
View File

@ -0,0 +1,58 @@
# frozen_string_literal: true
# Monkey-patch various Paperclip validators for Ruby 3.0 compatibility
module Paperclip
module Validators
module AttachmentSizeValidatorExtensions
def validate_each(record, attr_name, _value)
base_attr_name = attr_name
attr_name = "#{attr_name}_file_size".to_sym
value = record.send(:read_attribute_for_validation, attr_name)
if value.present?
options.slice(*Paperclip::Validators::AttachmentSizeValidator::AVAILABLE_CHECKS).each do |option, option_value|
option_value = option_value.call(record) if option_value.is_a?(Proc)
option_value = extract_option_value(option, option_value)
next if value.send(Paperclip::Validators::AttachmentSizeValidator::CHECKS[option], option_value)
error_message_key = options[:in] ? :in_between : option
[attr_name, base_attr_name].each do |error_attr_name|
record.errors.add(error_attr_name, error_message_key, **filtered_options(value).merge(
min: min_value_in_human_size(record),
max: max_value_in_human_size(record),
count: human_size(option_value)
))
end
end
end
end
end
module AttachmentContentTypeValidatorExtensions
def mark_invalid(record, attribute, types)
record.errors.add attribute, :invalid, **options.merge({ types: types.join(', ') })
end
end
module AttachmentPresenceValidatorExtensions
def validate_each(record, attribute, _value)
if record.send("#{attribute}_file_name").blank?
record.errors.add(attribute, :blank, **options)
end
end
end
module AttachmentFileNameValidatorExtensions
def mark_invalid(record, attribute, patterns)
record.errors.add attribute, :invalid, options.merge({ names: patterns.join(', ') })
end
end
end
end
Paperclip::Validators::AttachmentSizeValidator.prepend(Paperclip::Validators::AttachmentSizeValidatorExtensions)
Paperclip::Validators::AttachmentContentTypeValidator.prepend(Paperclip::Validators::AttachmentContentTypeValidatorExtensions)
Paperclip::Validators::AttachmentPresenceValidator.prepend(Paperclip::Validators::AttachmentPresenceValidatorExtensions)
Paperclip::Validators::AttachmentFileNameValidator.prepend(Paperclip::Validators::AttachmentFileNameValidatorExtensions)

26
lib/paperclip/video_transcoder.rb
View File

@ -1,26 +0,0 @@
# frozen_string_literal: true
module Paperclip
# This transcoder is only to be used for the MediaAttachment model
# to check when uploaded videos are actually gifv's
class VideoTranscoder < Paperclip::Processor
def make
movie = FFMPEG::Movie.new(@file.path)
attachment.instance.type = MediaAttachment.types[:gifv] unless movie.audio_codec
Paperclip::Transcoder.make(file, actual_options(movie), attachment)
end
private
def actual_options(movie)
opts = options[:passthrough_options]
if opts && opts[:video_codecs].include?(movie.video_codec) && opts[:audio_codecs].include?(movie.audio_codec) && opts[:colorspaces].include?(movie.colorspace)
opts[:options]
else
options
end
end
end
end

11
lib/rails/engine_extensions.rb Normal file
View File

@ -0,0 +1,11 @@
module Rails
module EngineExtensions
# Rewrite task loading code to filter digitalocean.rake task
def run_tasks_blocks(app)
Railtie.instance_method(:run_tasks_blocks).bind(self).call(app)
paths["lib/tasks"].existent.reject { |ext| ext.end_with?('digitalocean.rake') }.sort.each { |ext| load(ext) }
end
end
end
Rails::Engine.prepend(Rails::EngineExtensions)

128
lib/sanitize_ext/sanitize_config.rb Normal file
View File

@ -0,0 +1,128 @@
# frozen_string_literal: true
class Sanitize
module Config
HTTP_PROTOCOLS = %w(
http
https
).freeze
LINK_PROTOCOLS = %w(
http
https
dat
dweb
ipfs
ipns
ssb
gopher
xmpp
magnet
gemini
).freeze
CLASS_WHITELIST_TRANSFORMER = lambda do |env|
node = env[:node]
class_list = node['class']&.split(/[\t\n\f\r ]/)
return unless class_list
class_list.keep_if do |e|
next true if /^(h|p|u|dt|e)-/.match?(e) # microformats classes
next true if /^(mention|hashtag)$/.match?(e) # semantic classes
next true if /^(ellipsis|invisible)$/.match?(e) # link formatting classes
end
node['class'] = class_list.join(' ')
end
UNSUPPORTED_HREF_TRANSFORMER = lambda do |env|
return unless env[:node_name] == 'a'
current_node = env[:node]
scheme = begin
if current_node['href'] =~ Sanitize::REGEX_PROTOCOL
Regexp.last_match(1).downcase
else
:relative
end
end
current_node.replace(current_node.text) unless LINK_PROTOCOLS.include?(scheme)
end
UNSUPPORTED_ELEMENTS_TRANSFORMER = lambda do |env|
return unless %w(h6).include?(env[:node_name])
current_node = env[:node]
case env[:node_name]
when 'li'
current_node.traverse do |node|
next unless %w(p ul ol li).include?(node.name)
node.add_next_sibling('<br>') if node.next_sibling
node.replace(node.children) unless node.text?
end
else
current_node.name = 'p'
end
end
MASTODON_STRICT ||= freeze_config(
elements: %w(p br span a abbr del pre blockquote code b strong i em h1 h2 h3 h4 h5 ul ol li img),
attributes: {
'a' => %w(href rel class title),
'span' => %w(class),
'abbr' => %w(title),
'blockquote' => %w(cite),
'img' => %w(src alt),
},
add_attributes: {
'a' => {
'rel' => 'nofollow noopener noreferrer',
'target' => '_blank',
},
'span' => {
'class' => 'article-type',
},
},
protocols: {
'a' => { 'href' => HTTP_PROTOCOLS },
'blockquote' => { 'cite' => HTTP_PROTOCOLS },
},
transformers: [
CLASS_WHITELIST_TRANSFORMER,
UNSUPPORTED_ELEMENTS_TRANSFORMER,
UNSUPPORTED_HREF_TRANSFORMER,
]
)
MASTODON_OEMBED ||= freeze_config merge(
RELAXED,
elements: RELAXED[:elements] + %w(audio embed iframe source video),
attributes: merge(
RELAXED[:attributes],
'audio' => %w(controls),
'embed' => %w(height src type width),
'iframe' => %w(allowfullscreen frameborder height scrolling src width),
'source' => %w(src type),
'video' => %w(controls height loop width),
'div' => [:data]
),
protocols: merge(
RELAXED[:protocols],
'embed' => { 'src' => HTTP_PROTOCOLS },
'iframe' => { 'src' => HTTP_PROTOCOLS },
'source' => { 'src' => HTTP_PROTOCOLS }
)
)
end
end

66
lib/tasks/db.rake
View File

@ -1,36 +1,5 @@
# frozen_string_literal: true
require_relative '../mastodon/snowflake'
def each_schema_load_environment
# If we're in development, also run this for the test environment.
# This is a somewhat hacky way to do this, so here's why:
# 1. We have to define this before we load the schema, or we won't
# have a timestamp_id function when we get to it in the schema.
# 2. db:setup calls db:schema:load_if_ruby, which calls
# db:schema:load, which we define above as having a prerequisite
# of this task.
# 3. db:schema:load ends up running
# ActiveRecord::Tasks::DatabaseTasks.load_schema_current, which
# calls a private method `each_current_configuration`, which
# explicitly also does the loading for the `test` environment
# if the current environment is `development`, so we end up
# needing to do the same, and we can't even use the same method
# to do it.
if Rails.env.development?
test_conf = ActiveRecord::Base.configurations['test']
if test_conf['database']&.present?
ActiveRecord::Base.establish_connection(:test)
yield
ActiveRecord::Base.establish_connection(Rails.env.to_sym)
end
end
yield
end
namespace :db do
namespace :migrate do
desc 'Setup the db or migrate depending on state of db'
@ -50,40 +19,21 @@ namespace :db do
task :post_migration_hook do
at_exit do
unless %w(C POSIX).include?(ActiveRecord::Base.connection.execute('SELECT datcollate FROM pg_database WHERE datname = current_database();').first['datcollate'])
unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate'])
warn <<~WARNING
Your database collation is susceptible to index corruption.
(This warning does not indicate that index corruption has occured and can be ignored)
(This warning does not indicate that index corruption has occurred and can be ignored)
(To learn more, visit: https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/)
WARNING
end
end
end
task :pre_migration_check do
version = ActiveRecord::Base.connection.select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i
abort 'ERROR: This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon.' if version < 90_500
end
Rake::Task['db:migrate'].enhance(['db:pre_migration_check'])
Rake::Task['db:migrate'].enhance(['db:post_migration_hook'])
# Before we load the schema, define the timestamp_id function.
# Idiomatically, we might do this in a migration, but then it
# wouldn't end up in schema.rb, so we'd need to figure out a way to
# get it in before doing db:setup as well. This is simpler, and
# ensures it's always in place.
Rake::Task['db:schema:load'].enhance ['db:define_timestamp_id']
# After we load the schema, make sure we have sequences for each
# table using timestamp IDs.
Rake::Task['db:schema:load'].enhance do
Rake::Task['db:ensure_id_sequences_exist'].invoke
end
task :define_timestamp_id do
each_schema_load_environment do
Mastodon::Snowflake.define_timestamp_id
end
end
task :ensure_id_sequences_exist do
each_schema_load_environment do
Mastodon::Snowflake.ensure_id_sequences_exist
end
end
end

4
lib/tasks/emojis.rake
View File

@ -69,7 +69,7 @@ namespace :emojis do
end
end
existence_maps = grouped_codes.map { |c| c.map { |cc| [cc, File.exist?(Rails.root.join('public', 'emoji', codepoints_to_filename(cc) + '.svg'))] }.to_h }
existence_maps = grouped_codes.map { |c| c.index_with { |cc| File.exist?(Rails.root.join('public', 'emoji', codepoints_to_filename(cc) + '.svg')) } }
map = {}
existence_maps.each do |group|
@ -91,7 +91,7 @@ namespace :emojis do
desc 'Generate emoji variants with white borders'
task :generate_borders do
src = Rails.root.join('app', 'javascript', 'mastodon', 'features', 'emoji', 'emoji_map.json')
emojis = '🎱🐜⚫🖤⬛◼️◾◼️✒️▪️💣🎳📷📸♣️🕶️✴️🔌💂‍♀️📽️🍳🦍💂🔪🕳️🕹️🕋🖊️🖋️💂‍♂️🎤🎓🎥🎼♠️🎩🦃📼📹🎮🐃🏴🐞🕺👽⚾🐔☁️💨🕊️👀🍥👻🐐❕❔⛸️🌩️🔊🔇📃🌧️🐏🍚🍙🐓🐑💀☠️🌨️🔉🔈💬💭🏐🏳️⚪⬜◽◻️▫️'
emojis = '🎱🐜⚫🖤⬛◼️◾◼️✒️▪️💣🎳📷📸♣️🕶️✴️🔌💂‍♀️📽️🍳🦍💂🔪🕳️🕹️🕋🖊️🖋️💂‍♂️🎤🎓🎥🎼♠️🎩🦃📼📹🎮🐃🏴🐞🕺📱📲🚲👽⚾🐔☁️💨🕊️👀🍥👻🐐❕❔⛸️🌩️🔊🔇📃🌧️🐏🍚🍙🐓🐑💀☠️🌨️🔉🔈💬💭🏐🏳️⚪⬜◽◻️▫️'
map = Oj.load(File.read(src))

22
lib/tasks/mastodon.rake
View File

@ -371,18 +371,20 @@ namespace :mastodon do
end
end
prompt.say "\n"
prompt.say 'The final step is compiling CSS/JS assets.'
prompt.say 'This may take a while and consume a lot of RAM.'
unless using_docker
prompt.say "\n"
prompt.say 'The final step is compiling CSS/JS assets.'
prompt.say 'This may take a while and consume a lot of RAM.'
if prompt.yes?('Compile the assets now?')
prompt.say 'Running `RAILS_ENV=production rails assets:precompile` ...'
prompt.say "\n\n"
if prompt.yes?('Compile the assets now?')
prompt.say 'Running `RAILS_ENV=production rails assets:precompile` ...'
prompt.say "\n\n"
if !system(env.transform_values(&:to_s).merge({ 'RAILS_ENV' => 'production' }), 'rails assets:precompile')
prompt.error 'That failed! Maybe you need swap space?'
else
prompt.say 'Done!'
if !system(env.transform_values(&:to_s).merge({ 'RAILS_ENV' => 'production' }), 'rails assets:precompile')
prompt.error 'That failed! Maybe you need swap space?'
else
prompt.say 'Done!'
end
end
end

59
lib/tasks/repo.rake
View File

@ -1,27 +1,34 @@
# frozen_string_literal: true
REPOSITORY_NAME = 'tootsuite/mastodon'
namespace :repo do
desc 'Generate the AUTHORS.md file'
task :authors do
file = File.open(Rails.root.join('AUTHORS.md'), 'w')
file << <<~HEADER
Authors
=======
Mastodon is available on [GitHub](https://github.com/tootsuite/mastodon)
Mastodon is available on [GitHub](https://github.com/#{REPOSITORY_NAME})
and provided thanks to the work of the following contributors:
HEADER
url = 'https://api.github.com/repos/tootsuite/mastodon/contributors?anon=1'
url = "https://api.github.com/repos/#{REPOSITORY_NAME}/contributors?anon=1"
HttpLog.config.compact_log = true
while url.present?
response = HTTP.get(url)
response = HTTP.get(url)
contributors = Oj.load(response.body)
contributors.each do |c|
file << "* [#{c['login']}](#{c['html_url']})\n" if c['login']
file << "* [#{c['name']}](mailto:#{c['email']})\n" if c['name']
end
url = LinkHeader.parse(response.headers['Link']).find_link(%w(rel next))&.href
end
@ -47,7 +54,7 @@ namespace :repo do
response = nil
loop do
response = HTTP.headers('Authorization' => "token #{ENV['GITHUB_API_TOKEN']}").get("https://api.github.com/repos/tootsuite/mastodon/pulls/#{pull_request_number}")
response = HTTP.headers('Authorization' => "token #{ENV['GITHUB_API_TOKEN']}").get("https://api.github.com/repos/#{REPOSITORY_NAME}/pulls/#{pull_request_number}")
if response.code == 403
sleep_for = (response.headers['X-RateLimit-Reset'].to_i - Time.now.to_i).abs
@ -83,12 +90,46 @@ namespace :repo do
missing_yaml_files = I18n.available_locales.reject { |locale| File.exist?(Rails.root.join('config', 'locales', "#{locale}.yml")) }
missing_json_files = I18n.available_locales.reject { |locale| File.exist?(Rails.root.join('app', 'javascript', 'mastodon', 'locales', "#{locale}.json")) }
if missing_json_files.empty? && missing_yaml_files.empty?
puts pastel.green('OK')
else
puts pastel.red("Missing YAML files: #{pastel.bold(missing_yaml_files.join(', '))}") unless missing_yaml_files.empty?
puts pastel.red("Missing JSON files: #{pastel.bold(missing_json_files.join(', '))}") unless missing_json_files.empty?
locales_in_files = Dir[Rails.root.join('config', 'locales', '*.yml')].map do |path|
file_name = File.basename(path)
file_name.gsub(/\A(doorkeeper|devise|activerecord|simple_form)\./, '').gsub(/\.yml\z/, '').to_sym
end.uniq.compact
missing_available_locales = locales_in_files - I18n.available_locales
missing_locale_names = I18n.available_locales.reject { |locale| SettingsHelper::HUMAN_LOCALES.key?(locale) }
critical = false
unless missing_json_files.empty?
critical = true
puts pastel.red("You are missing JSON files for these locales: #{pastel.bold(missing_json_files.join(', '))}")
puts pastel.red('This will lead to runtime errors for users who have selected those locales')
puts pastel.red("Add the missing files or remove the locales from #{pastel.bold('I18n.available_locales')} in config/application.rb")
end
unless missing_yaml_files.empty?
critical = true
puts pastel.red("You are missing YAML files for these locales: #{pastel.bold(missing_yaml_files.join(', '))}")
puts pastel.red('This will lead to runtime errors for users who have selected those locales')
puts pastel.red("Add the missing files or remove the locales from #{pastel.bold('I18n.available_locales')} in config/application.rb")
end
unless missing_available_locales.empty?
puts pastel.yellow("You have locale files that are not enabled: #{pastel.bold(missing_available_locales.join(', '))}")
puts pastel.yellow("Add them to #{pastel.bold('I18n.available_locales')} in config/application.rb or remove them")
end
unless missing_locale_names.empty?
puts pastel.yellow("You are missing human-readable names for these locales: #{pastel.bold(missing_locale_names.join(', '))}")
puts pastel.yellow("Add them to #{pastel.bold('HUMAN_LOCALES')} in app/helpers/settings_helper.rb or remove the locales from #{pastel.bold('I18n.available_locales')} in config/application.rb")
end
if critical
exit(1)
else
puts pastel.green('OK')
end
end
end

66
lib/terrapin/multi_pipe_extensions.rb Normal file
View File

@ -0,0 +1,66 @@
# frozen_string_literal: false
require 'fcntl'
module Terrapin
module MultiPipeExtensions
def initialize
@stdout_in, @stdout_out = IO.pipe
@stderr_in, @stderr_out = IO.pipe
clear_nonblocking_flags!
end
def pipe_options
# Add some flags to explicitly close the other end of the pipes
{ out: @stdout_out, err: @stderr_out, @stdout_in => :close, @stderr_in => :close }
end
def read
# While we are patching Terrapin, fix child process potentially getting stuck on writing
# to stderr.
@stdout_output = +''
@stderr_output = +''
fds_to_read = [@stdout_in, @stderr_in]
until fds_to_read.empty?
rs, = IO.select(fds_to_read)
read_nonblocking!(@stdout_in, @stdout_output, fds_to_read) if rs.include?(@stdout_in)
read_nonblocking!(@stderr_in, @stderr_output, fds_to_read) if rs.include?(@stderr_in)
end
end
private
# @param [IO] io IO Stream to read until there is nothing to read
# @param [String] result Mutable string to which read values will be appended to
# @param [Array<IO>] fds_to_read Mutable array from which `io` should be removed on EOF
def read_nonblocking!(io, result, fds_to_read)
while (partial_result = io.read_nonblock(8192))
result << partial_result
end
rescue IO::WaitReadable
# Do nothing
rescue EOFError
fds_to_read.delete(io)
end
def clear_nonblocking_flags!
# Ruby 3.0 sets pipes to non-blocking mode, and resets the flags as
# needed when calling fork/exec-related syscalls, but posix-spawn does
# not currently do that, so we need to do it manually for the time being
# so that the child process do not error out when the buffers are full.
stdout_flags = @stdout_out.fcntl(Fcntl::F_GETFL)
@stdout_out.fcntl(Fcntl::F_SETFL, stdout_flags & ~Fcntl::O_NONBLOCK) if stdout_flags & Fcntl::O_NONBLOCK
stderr_flags = @stderr_out.fcntl(Fcntl::F_GETFL)
@stderr_out.fcntl(Fcntl::F_SETFL, stderr_flags & ~Fcntl::O_NONBLOCK) if stderr_flags & Fcntl::O_NONBLOCK
rescue NameError, NotImplementedError, Errno::EINVAL
# Probably on windows, where pipes are blocking by default
end
end
end
Terrapin::CommandLine::MultiPipe.prepend(Terrapin::MultiPipeExtensions)