autonomic-cooperative/hometown
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge tag 'v3.4.0' into hometown-dev

Browse Source
This commit is contained in:
Darius Kazemi
2021-05-17 13:48:27 -07:00
parent 0e3f729c33 4c7efdba40
commit f57cda3855
897 changed files with 26918 additions and 14941 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
app/lib/account_reach_finder.rb Normal file
View File

@ -0,0 +1,25 @@
# frozen_string_literal: true
class AccountReachFinder
def initialize(account)
@account = account
end
def inboxes
(followers_inboxes + reporters_inboxes + relay_inboxes).uniq
end
private
def followers_inboxes
@account.followers.inboxes
end
def reporters_inboxes
Account.where(id: @account.targeted_reports.select(:account_id)).inboxes
end
def relay_inboxes
Relay.enabled.pluck(:inbox_url)
end
end

14
app/lib/activitypub/activity.rb
View File

@ -144,7 +144,7 @@ class ActivityPub::Activity
end
def delete_later!(uri)
redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, uri)
redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, true)
end
def status_from_object
@ -210,12 +210,22 @@ class ActivityPub::Activity
end
end
def lock_or_return(key, expire_after = 7.days.seconds)
def lock_or_return(key, expire_after = 2.hours.seconds)
yield if redis.set(key, true, nx: true, ex: expire_after)
ensure
redis.del(key)
end
def lock_or_fail(key)
RedisLock.acquire({ redis: Redis.current, key: key }) do |lock|
if lock.acquired?
yield
else
raise Mastodon::RaceConditionError
end
end
end
def fetch?
!@options[:delivery]
end

42
app/lib/activitypub/activity/announce.rb
View File

@ -4,29 +4,29 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
def perform
return reject_payload! if delete_arrived_first?(@json['id']) || !related_to_local_activity?
RedisLock.acquire(lock_options) do |lock|
if lock.acquired?
original_status = status_from_object
lock_or_fail("announce:#{@object['id']}") do
original_status = status_from_object
return reject_payload! if original_status.nil? || !announceable?(original_status)
return reject_payload! if original_status.nil? || !announceable?(original_status)
@status = Status.find_by(account: @account, reblog: original_status)
@status = Status.find_by(account: @account, reblog: original_status)
return @status unless @status.nil?
return @status unless @status.nil?
@status = Status.create!(
account: @account,
reblog: original_status,
uri: @json['id'],
created_at: @json['published'],
override_timestamps: @options[:override_timestamps],
visibility: visibility_from_audience
)
@status = Status.create!(
account: @account,
reblog: original_status,
uri: @json['id'],
created_at: @json['published'],
override_timestamps: @options[:override_timestamps],
visibility: visibility_from_audience
)
distribute(@status)
else
raise Mastodon::RaceConditionError
original_status.tags.each do |tag|
tag.use!(@account)
end
distribute(@status)
end
@status
@ -43,9 +43,9 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
end
def visibility_from_audience
if audience_to.include?(ActivityPub::TagManager::COLLECTIONS[:public])
if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) }
:public
elsif audience_cc.include?(ActivityPub::TagManager::COLLECTIONS[:public])
elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) }
:unlisted
elsif audience_to.include?(@account.followers_url)
:private
@ -69,8 +69,4 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
def reblog_of_local_status?
status_from_uri(object_uri)&.account&.local?
end
def lock_options
{ redis: Redis.current, key: "announce:#{@object['id']}" }
end
end

7
app/lib/activitypub/activity/block.rb
View File

@ -11,8 +11,13 @@ class ActivityPub::Activity::Block < ActivityPub::Activity
return
end
UnfollowService.new.call(@account, target_account) if @account.following?(target_account)
UnfollowService.new.call(target_account, @account) if target_account.following?(@account)
RejectFollowService.new.call(target_account, @account) if target_account.requested?(@account)
@account.block!(target_account, uri: @json['id']) unless delete_arrived_first?(@json['id'])
unless delete_arrived_first?(@json['id'])
BlockWorker.perform_async(@account.id, target_account.id)
@account.block!(target_account, uri: @json['id'])
end
end
end

49
app/lib/activitypub/activity/create.rb
View File

@ -45,19 +45,15 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def create_status
return reject_payload! if unsupported_object_type? || invalid_origin?(object_uri) || tombstone_exists? || !related_to_local_activity?
RedisLock.acquire(lock_options) do |lock|
if lock.acquired?
return if delete_arrived_first?(object_uri) || poll_vote? # rubocop:disable Lint/NonLocalExitFromIterator
lock_or_fail("create:#{object_uri}") do
return if delete_arrived_first?(object_uri) || poll_vote?
@status = find_existing_status
@status = find_existing_status
if @status.nil?
process_status
elsif @options[:delivered_to_account_id].present?
postprocess_audience_and_deliver
end
else
raise Mastodon::RaceConditionError
if @status.nil?
process_status
elsif @options[:delivered_to_account_id].present?
postprocess_audience_and_deliver
end
end
@ -89,7 +85,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
resolve_thread(@status)
fetch_replies(@status)
check_for_spam
distribute(@status)
forward_for_reply
end
@ -175,7 +170,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def process_audience
(audience_to + audience_cc).uniq.each do |audience|
next if audience == ActivityPub::TagManager::COLLECTIONS[:public]
next if ActivityPub::TagManager.instance.public_collection?(audience)
# Unlike with tags, there is no point in resolving accounts we don't already
# know here, because silent mentions would only be used for local access
@ -221,7 +216,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def attach_tags(status)
@tags.each do |tag|
status.tags << tag
TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility?
tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility?
end
@mentions.each do |mention|
@ -366,13 +361,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
poll = replied_to_status.preloadable_poll
already_voted = true
RedisLock.acquire(poll_lock_options) do |lock|
if lock.acquired?
already_voted = poll.votes.where(account: @account).exists?
poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: object_uri)
else
raise Mastodon::RaceConditionError
end
lock_or_fail("vote:#{replied_to_status.poll_id}:#{@account.id}") do
already_voted = poll.votes.where(account: @account).exists?
poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: object_uri)
end
increment_voters_count! unless already_voted
@ -408,9 +399,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
end
def visibility_from_audience
if audience_to.include?(ActivityPub::TagManager::COLLECTIONS[:public])
if audience_to.any? { |to| ActivityPub::TagManager.instance.public_collection?(to) }
:public
elsif audience_cc.include?(ActivityPub::TagManager::COLLECTIONS[:public])
elsif audience_cc.any? { |cc| ActivityPub::TagManager.instance.public_collection?(cc) }
:unlisted
elsif audience_to.include?(@account.followers_url)
:private
@ -546,10 +537,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
Tombstone.exists?(uri: object_uri)
end
def check_for_spam
SpamCheck.perform(@status)
end
def forward_for_reply
return unless @status.distributable? && @json['signature'].present? && reply_to_local?
@ -567,12 +554,4 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
poll.reload
retry
end
def lock_options
{ redis: Redis.current, key: "create:#{object_uri}" }
end
def poll_lock_options
{ redis: Redis.current, key: "vote:#{replied_to_status.poll_id}:#{@account.id}" }
end
end

68
app/lib/activitypub/activity/delete.rb
View File

@ -20,33 +20,35 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
def delete_note
return if object_uri.nil?
unless invalid_origin?(object_uri)
RedisLock.acquire(lock_options) { |_lock| delete_later!(object_uri) }
Tombstone.find_or_create_by(uri: object_uri, account: @account)
lock_or_return("delete_status_in_progress:#{object_uri}", 5.minutes.seconds) do
unless invalid_origin?(object_uri)
# This lock ensures a concurrent `ActivityPub::Activity::Create` either
# does not create a status at all, or has finished saving it to the
# database before we try to load it.
# Without the lock, `delete_later!` could be called after `delete_arrived_first?`
# and `Status.find` before `Status.create!`
lock_or_fail("create:#{object_uri}") { delete_later!(object_uri) }
Tombstone.find_or_create_by(uri: object_uri, account: @account)
end
@status = Status.find_by(uri: object_uri, account: @account)
@status ||= Status.find_by(uri: @object['atomUri'], account: @account) if @object.is_a?(Hash) && @object['atomUri'].present?
return if @status.nil?
forward! if @json['signature'].present? && @status.distributable?
delete_now!
end
@status = Status.find_by(uri: object_uri, account: @account)
@status ||= Status.find_by(uri: @object['atomUri'], account: @account) if @object.is_a?(Hash) && @object['atomUri'].present?
return if @status.nil?
if @status.distributable?
forward_for_reply
forward_for_reblogs
end
delete_now!
end
def forward_for_reblogs
return if @json['signature'].blank?
def rebloggers_ids
return @rebloggers_ids if defined?(@rebloggers_ids)
@rebloggers_ids = @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id)
end
rebloggers_ids = @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id)
inboxes = Account.where(id: ::Follow.where(target_account_id: rebloggers_ids).select(:account_id)).inboxes - [@account.preferred_inbox_url]
ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url|
[payload, rebloggers_ids.first, inbox_url]
end
def inboxes_for_reblogs
Account.where(id: ::Follow.where(target_account_id: rebloggers_ids).select(:account_id)).inboxes
end
def replied_to_status
@ -58,13 +60,19 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
!replied_to_status.nil? && replied_to_status.account.local?
end
def forward_for_reply
return unless @json['signature'].present? && reply_to_local?
def inboxes_for_reply
replied_to_status.account.followers.inboxes
end
inboxes = replied_to_status.account.followers.inboxes - [@account.preferred_inbox_url]
def forward!
inboxes = inboxes_for_reblogs
inboxes += inboxes_for_reply if reply_to_local?
inboxes -= [@account.preferred_inbox_url]
ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url|
[payload, replied_to_status.account_id, inbox_url]
sender_id = reply_to_local? ? replied_to_status.account_id : rebloggers_ids.first
ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes.uniq) do |inbox_url|
[payload, sender_id, inbox_url]
end
end
@ -75,8 +83,4 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
def payload
@payload ||= Oj.dump(@json)
end
def lock_options
{ redis: Redis.current, key: "create:#{object_uri}" }
end
end

6
app/lib/activitypub/activity/flag.rb
View File

@ -4,12 +4,14 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
def perform
return if skip_reports?
target_accounts = object_uris.map { |uri| account_from_uri(uri) }.compact.select(&:local?)
target_statuses_by_account = object_uris.map { |uri| status_from_uri(uri) }.compact.select(&:local?).group_by(&:account_id)
target_accounts = object_uris.filter_map { |uri| account_from_uri(uri) }.select(&:local?)
target_statuses_by_account = object_uris.filter_map { |uri| status_from_uri(uri) }.select(&:local?).group_by(&:account_id)
target_accounts.each do |target_account|
target_statuses = target_statuses_by_account[target_account.id]
next if target_account.suspended?
ReportService.new.call(
@account,
target_account,

13
app/lib/activitypub/activity/follow.rb
View File

@ -6,7 +6,14 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
def perform
target_account = account_from_uri(object_uri)
return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) || @account.requested?(target_account)
return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id'])
# Update id of already-existing follow requests
existing_follow_request = ::FollowRequest.find_by(account: @account, target_account: target_account)
unless existing_follow_request.nil?
existing_follow_request.update!(uri: @json['id'])
return
end
if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? || target_account.instance_actor?
reject_follow_request!(target_account)
@ -14,7 +21,9 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
end
# Fast-forward repeat follow requests
if @account.following?(target_account)
existing_follow = ::Follow.find_by(account: @account, target_account: target_account)
unless existing_follow.nil?
existing_follow.update!(uri: @json['id'])
AuthorizeFollowService.new.call(@account, target_account, skip_follow_request: true, follow_request_uri: @json['id'])
return
end

11
app/lib/activitypub/activity/move.rb
View File

@ -4,9 +4,8 @@ class ActivityPub::Activity::Move < ActivityPub::Activity
PROCESSING_COOLDOWN = 7.days.seconds
def perform
return if origin_account.uri != object_uri || processed?
mark_as_processing!
return if origin_account.uri != object_uri
return unless mark_as_processing!
target_account = ActivityPub::FetchRemoteAccountService.new.call(target_uri)
@ -35,12 +34,8 @@ class ActivityPub::Activity::Move < ActivityPub::Activity
value_or_id(@json['target'])
end
def processed?
redis.exists?("move_in_progress:#{@account.id}")
end
def mark_as_processing!
redis.setex("move_in_progress:#{@account.id}", PROCESSING_COOLDOWN, true)
redis.set("move_in_progress:#{@account.id}", true, nx: true, ex: PROCESSING_COOLDOWN)
end
def unmark_as_processing!

4
app/lib/activitypub/tag_manager.rb
View File

@ -12,6 +12,10 @@ class ActivityPub::TagManager
public: 'https://www.w3.org/ns/activitystreams#Public',
}.freeze
def public_collection?(uri)
uri == COLLECTIONS[:public] || uri == 'as:Public' || uri == 'Public'
end
def url_for(target)
return target.url if target.respond_to?(:local?) && !target.local?

21
app/lib/admin/system_check.rb Normal file
View File

@ -0,0 +1,21 @@
# frozen_string_literal: true
class Admin::SystemCheck
ACTIVE_CHECKS = [
Admin::SystemCheck::DatabaseSchemaCheck,
Admin::SystemCheck::SidekiqProcessCheck,
Admin::SystemCheck::RulesCheck,
].freeze
def self.perform
ACTIVE_CHECKS.each_with_object([]) do |klass, arr|
check = klass.new
if check.pass?
arr
else
arr << check.message
end
end
end
end

11
app/lib/admin/system_check/base_check.rb Normal file
View File

@ -0,0 +1,11 @@
# frozen_string_literal: true
class Admin::SystemCheck::BaseCheck
def pass?
raise NotImplementedError
end
def message
raise NotImplementedError
end
end

11
app/lib/admin/system_check/database_schema_check.rb Normal file
View File

@ -0,0 +1,11 @@
# frozen_string_literal: true
class Admin::SystemCheck::DatabaseSchemaCheck < Admin::SystemCheck::BaseCheck
def pass?
!ActiveRecord::Base.connection.migration_context.needs_migration?
end
def message
Admin::SystemCheck::Message.new(:database_schema_check)
end
end

11
app/lib/admin/system_check/message.rb Normal file
View File

@ -0,0 +1,11 @@
# frozen_string_literal: true
class Admin::SystemCheck::Message
attr_reader :key, :value, :action
def initialize(key, value = nil, action = nil)
@key = key
@value = value
@action = action
end
end

13
app/lib/admin/system_check/rules_check.rb Normal file
View File

@ -0,0 +1,13 @@
# frozen_string_literal: true
class Admin::SystemCheck::RulesCheck < Admin::SystemCheck::BaseCheck
include RoutingHelper
def pass?
Rule.kept.exists?
end
def message
Admin::SystemCheck::Message.new(:rules_check, nil, admin_rules_path)
end
end

25
app/lib/admin/system_check/sidekiq_process_check.rb Normal file
View File

@ -0,0 +1,25 @@
# frozen_string_literal: true
class Admin::SystemCheck::SidekiqProcessCheck < Admin::SystemCheck::BaseCheck
SIDEKIQ_QUEUES = %w(
default
push
mailers
pull
scheduler
).freeze
def pass?
missing_queues.empty?
end
def message
Admin::SystemCheck::Message.new(:sidekiq_process_check, missing_queues.join(', '))
end
private
def missing_queues
@missing_queues ||= Sidekiq::ProcessSet.new.reduce(SIDEKIQ_QUEUES) { |queues, process| queues - process['queues'] }
end
end

4
app/lib/application_extension.rb
View File

@ -4,6 +4,8 @@ module ApplicationExtension
extend ActiveSupport::Concern
included do
validates :website, url: true, if: :website?
validates :name, length: { maximum: 60 }
validates :website, url: true, length: { maximum: 2_000 }, if: :website?
validates :redirect_uri, length: { maximum: 2_000 }
end
end

28
app/lib/delivery_failure_tracker.rb
View File

@ -17,6 +17,10 @@ class DeliveryFailureTracker
UnavailableDomain.find_by(domain: @host)&.destroy
end
def clear_failures!
Redis.current.del(exhausted_deliveries_key)
end
def days
Redis.current.scard(exhausted_deliveries_key) || 0
end
@ -25,11 +29,15 @@ class DeliveryFailureTracker
!UnavailableDomain.where(domain: @host).exists?
end
def exhausted_deliveries_days
Redis.current.smembers(exhausted_deliveries_key).sort.map { |date| Date.new(date.slice(0, 4).to_i, date.slice(4, 2).to_i, date.slice(6, 2).to_i) }
end
alias reset! track_success!
class << self
def without_unavailable(urls)
unavailable_domains_map = Rails.cache.fetch('unavailable_domains') { UnavailableDomain.pluck(:domain).each_with_object({}) { |domain, hash| hash[domain] = true } }
unavailable_domains_map = Rails.cache.fetch('unavailable_domains') { UnavailableDomain.pluck(:domain).index_with(true) }
urls.reject do |url|
host = Addressable::URI.parse(url).normalized_host
@ -44,6 +52,24 @@ class DeliveryFailureTracker
def reset!(url)
new(url).reset!
end
def warning_domains
domains = Redis.current.keys(exhausted_deliveries_key_by('*')).map do |key|
key.delete_prefix(exhausted_deliveries_key_by(''))
end
domains - UnavailableDomain.all.pluck(:domain)
end
def warning_domains_map
warning_domains.index_with { |domain| Redis.current.scard(exhausted_deliveries_key_by(domain)) }
end
private
def exhausted_deliveries_key_by(host)
"exhausted_deliveries:#{host}"
end
end
private

8
app/lib/entity_cache.rb
View File

@ -16,7 +16,9 @@ class EntityCache
end
def emoji(shortcodes, domain)
shortcodes = Array(shortcodes)
shortcodes = Array(shortcodes)
return [] if shortcodes.empty?
cached = Rails.cache.read_multi(*shortcodes.map { |shortcode| to_key(:emoji, shortcode, domain) })
uncached_ids = []
@ -25,11 +27,11 @@ class EntityCache
end
unless uncached_ids.empty?
uncached = CustomEmoji.where(shortcode: shortcodes, domain: domain, disabled: false).each_with_object({}) { |item, h| h[item.shortcode] = item }
uncached = CustomEmoji.where(shortcode: shortcodes, domain: domain, disabled: false).index_by(&:shortcode)
uncached.each_value { |item| Rails.cache.write(to_key(:emoji, item.shortcode, domain), item, expires_in: MAX_EXPIRATION) }
end
shortcodes.map { |shortcode| cached[to_key(:emoji, shortcode, domain)] || uncached[shortcode] }.compact
shortcodes.filter_map { |shortcode| cached[to_key(:emoji, shortcode, domain)] || uncached[shortcode] }
end
def to_key(type, *ids)

23
app/lib/exceptions.rb
View File

@ -1,23 +0,0 @@
# frozen_string_literal: true
module Mastodon
class Error < StandardError; end
class NotPermittedError < Error; end
class ValidationError < Error; end
class HostValidationError < ValidationError; end
class LengthValidationError < ValidationError; end
class DimensionsValidationError < ValidationError; end
class StreamValidationError < ValidationError; end
class RaceConditionError < Error; end
class RateLimitExceededError < Error; end
class UnexpectedResponseError < Error
def initialize(response = nil)
if response.respond_to? :uri
super("#{response.uri} returned code #{response.code}")
else
super
end
end
end
end

12
app/lib/extractor.rb
View File

@ -1,20 +1,20 @@
# frozen_string_literal: true
module Extractor
extend Twitter::Extractor
extend Twitter::TwitterText::Extractor
module_function
# :yields: username, list_slug, start, end
def extract_mentions_or_lists_with_indices(text)
return [] unless text =~ Twitter::Regex[:at_signs]
return [] unless Twitter::TwitterText::Regex[:at_signs].match?(text)
possible_entries = []
text.to_s.scan(Account::MENTION_RE) do |screen_name, _|
match_data = $LAST_MATCH_INFO
after = $'
unless after =~ Twitter::Regex[:end_mention_match]
unless Twitter::TwitterText::Regex[:end_mention_match].match?(after)
start_position = match_data.char_begin(1) - 1
end_position = match_data.char_end(1)
possible_entries << {
@ -33,7 +33,7 @@ module Extractor
end
def extract_hashtags_with_indices(text, **)
return [] unless text =~ /#/
return [] unless /#/.match?(text)
tags = []
text.scan(Tag::HASHTAG_RE) do |hash_text, _|
@ -41,10 +41,10 @@ module Extractor
start_position = match_data.char_begin(1) - 1
end_position = match_data.char_end(1)
after = $'
if after =~ %r{\A://}
if %r{\A://}.match?(after)
hash_text.match(/(.+)(https?\Z)/) do |matched|
hash_text = matched[1]
end_position -= matched[2].char_length
end_position -= matched[2].codepoint_length
end
end

48
app/lib/feed_manager.rb
View File

@ -194,6 +194,36 @@ class FeedManager
end
end
# Clear all statuses from or mentioning target_account from a list feed
# @param [List] list
# @param [Account] target_account
# @return [void]
def clear_from_list(list, target_account)
timeline_key = key(:list, list.id)
timeline_status_ids = redis.zrange(timeline_key, 0, -1)
statuses = Status.where(id: timeline_status_ids).select(:id, :reblog_of_id, :account_id).to_a
reblogged_ids = Status.where(id: statuses.map(&:reblog_of_id).compact, account: target_account).pluck(:id)
with_mentions_ids = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact, account: target_account).pluck(:status_id)
target_statuses = statuses.select do |status|
status.account_id == target_account.id || reblogged_ids.include?(status.reblog_of_id) || with_mentions_ids.include?(status.id) || with_mentions_ids.include?(status.reblog_of_id)
end
target_statuses.each do |status|
unpush_from_list(list, status)
end
end
# Clear all statuses from or mentioning target_account from an account's lists
# @param [Account] account
# @param [Account] target_account
# @return [void]
def clear_from_lists(account, target_account)
List.where(account: account).each do |list|
clear_from_list(list, target_account)
end
end
# Populate home feed of account from scratch
# @param [Account] account
# @return [void]
@ -403,8 +433,8 @@ class FeedManager
active_filters.map! do |filter|
if filter.whole_word
sb = filter.phrase =~ /\A[[:word:]]/ ? '\b' : ''
eb = filter.phrase =~ /[[:word:]]\z/ ? '\b' : ''
sb = /\A[[:word:]]/.match?(filter.phrase) ? '\b' : ''
eb = /[[:word:]]\z/.match?(filter.phrase) ? '\b' : ''
/(?mix:#{sb}#{Regexp.escape(filter.phrase)}#{eb})/
else
@ -424,7 +454,7 @@ class FeedManager
status.media_attachments.map(&:description).join("\n\n"),
].compact.join("\n\n")
!combined_regex.match(combined_text).nil?
combined_regex.match?(combined_text)
end
# Adds a status to an account's feed, returning true if a status was
@ -540,12 +570,12 @@ class FeedManager
arr
end
crutches[:following] = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:in_reply_to_account_id).compact).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
crutches[:hiding_reblogs] = Follow.where(account_id: receiver_id, target_account_id: statuses.map { |s| s.account_id if s.reblog? }.compact, show_reblogs: false).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
crutches[:blocking] = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
crutches[:muting] = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.map { |s| s.reblog&.account&.domain }.compact).pluck(:domain).each_with_object({}) { |domain, mapping| mapping[domain] = true }
crutches[:blocked_by] = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| s.reblog&.account_id }.compact).pluck(:account_id).each_with_object({}) { |id, mapping| mapping[id] = true }
crutches[:following] = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:in_reply_to_account_id).compact).pluck(:target_account_id).index_with(true)
crutches[:hiding_reblogs] = Follow.where(account_id: receiver_id, target_account_id: statuses.map { |s| s.account_id if s.reblog? }.compact, show_reblogs: false).pluck(:target_account_id).index_with(true)
crutches[:blocking] = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true)
crutches[:muting] = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true)
crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.map { |s| s.reblog&.account&.domain }.compact).pluck(:domain).index_with(true)
crutches[:blocked_by] = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| s.reblog&.account_id }.compact).pluck(:account_id).index_with(true)
crutches
end

44
app/lib/formatter.rb
View File

@ -1,7 +1,6 @@
# frozen_string_literal: true
require 'singleton'
require_relative './sanitize_config'
class Formatter
include Singleton
@ -86,7 +85,7 @@ class Formatter
end
def format_field(account, str, **options)
html = account.local? ? encode_and_link_urls(str, me: true) : reformat(str)
html = account.local? ? encode_and_link_urls(str, me: true, with_domain: true) : reformat(str)
html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify]
html.html_safe # rubocop:disable Rails/OutputSafety
end
@ -129,7 +128,7 @@ class Formatter
elsif entity[:hashtag]
link_to_hashtag(entity)
elsif entity[:screen_name]
link_to_mention(entity, accounts)
link_to_mention(entity, accounts, options)
end
end
end
@ -164,9 +163,9 @@ class Formatter
original_url, static_url = emoji
replacement = begin
if animate
"<img draggable=\"false\" class=\"emojione\" alt=\":#{encode(shortcode)}:\" title=\":#{encode(shortcode)}:\" src=\"#{encode(original_url)}\" />"
image_tag(original_url, draggable: false, class: 'emojione', alt: ":#{shortcode}:", title: ":#{shortcode}:")
else
"<img draggable=\"false\" class=\"emojione custom-emoji\" alt=\":#{encode(shortcode)}:\" title=\":#{encode(shortcode)}:\" src=\"#{encode(static_url)}\" data-original=\"#{original_url}\" data-static=\"#{static_url}\" />"
image_tag(original_url, draggable: false, class: 'emojione custom-emoji', alt: ":#{shortcode}:", title: ":#{shortcode}:", data: { original: original_url, static: static_url })
end
end
before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : ''
@ -228,7 +227,7 @@ class Formatter
escaped = text.chars.map do |c|
output = begin
if c.ord.to_s(16).length > 2 && UNICODE_ESCAPE_BLACKLIST_RE.match(c).nil?
if c.ord.to_s(16).length > 2 && !UNICODE_ESCAPE_BLACKLIST_RE.match?(c)
CGI.escape(c)
else
c
@ -266,27 +265,42 @@ class Formatter
html_attrs[:rel] = "me #{html_attrs[:rel]}" if options[:me]
Twitter::Autolink.send(:link_to_text, entity, link_html(entity[:url]), url, html_attrs)
Twitter::TwitterText::Autolink.send(:link_to_text, entity, link_html(entity[:url]), url, html_attrs)
rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError
encode(entity[:url])
end
def link_to_mention(entity, linkable_accounts)
def link_to_mention(entity, linkable_accounts, options = {})
acct = entity[:screen_name]
return link_to_account(acct) unless linkable_accounts
return link_to_account(acct, options) unless linkable_accounts
account = linkable_accounts.find { |item| TagManager.instance.same_acct?(item.acct, acct) }
account ? mention_html(account) : "@#{encode(acct)}"
same_username_hits = 0
account = nil
username, domain = acct.split('@')
domain = nil if TagManager.instance.local_domain?(domain)
linkable_accounts.each do |item|
same_username = item.username.casecmp(username).zero?
same_domain = item.domain.nil? ? domain.nil? : item.domain.casecmp(domain)&.zero?
if same_username && !same_domain
same_username_hits += 1
elsif same_username && same_domain
account = item
end
end
account ? mention_html(account, with_domain: same_username_hits.positive? || options[:with_domain]) : "@#{encode(acct)}"
end
def link_to_account(acct)
def link_to_account(acct, options = {})
username, domain = acct.split('@')
domain = nil if TagManager.instance.local_domain?(domain)
account = EntityCache.instance.mention(username, domain)
account ? mention_html(account) : "@#{encode(acct)}"
account ? mention_html(account, with_domain: options[:with_domain]) : "@#{encode(acct)}"
end
def link_to_hashtag(entity)
@ -307,7 +321,7 @@ class Formatter
"<a href=\"#{encode(tag_url(tag))}\" class=\"mention hashtag\" rel=\"tag\">#<span>#{encode(tag)}</span></a>"
end
def mention_html(account)
"<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>"
def mention_html(account, with_domain: false)
"<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(with_domain ? account.pretty_acct : account.username)}</span></a></span>"
end
end

2
app/lib/language_detector.rb
View File

@ -69,7 +69,7 @@ class LanguageDetector
def simplify_text(text)
new_text = remove_html(text)
new_text.gsub!(FetchLinkCardService::URL_PATTERN, '')
new_text.gsub!(FetchLinkCardService::URL_PATTERN, '\1')
new_text.gsub!(Account::MENTION_RE, '')
new_text.gsub!(Tag::HASHTAG_RE) { |string| string.gsub(/[#_]/, '#' => '', '_' => ' ').gsub(/[a-z][A-Z]|[a-zA-Z][\d]/) { |s| s.insert(1, ' ') }.downcase }
new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '')

6
app/lib/potential_friendship_tracker.rb
View File

@ -27,11 +27,5 @@ class PotentialFriendshipTracker
def remove(account_id, target_account_id)
redis.zrem("interactions:#{account_id}", target_account_id)
end
def get(account_id, limit: 20, offset: 0)
account_ids = redis.zrevrange("interactions:#{account_id}", offset, limit)
return [] if account_ids.empty?
Account.searchable.where(id: account_ids)
end
end
end

2
app/lib/request.rb
View File

@ -145,7 +145,7 @@ class Request
end
def block_hidden_service?
!Rails.configuration.x.access_to_hidden_service && /\.(onion|i2p)$/.match(@url.host)
!Rails.configuration.x.access_to_hidden_service && /\.(onion|i2p)$/.match?(@url.host)
end
module ClientLimit

128
app/lib/sanitize_config.rb
View File

@ -1,128 +0,0 @@
# frozen_string_literal: true
class Sanitize
module Config
HTTP_PROTOCOLS = %w(
http
https
).freeze
LINK_PROTOCOLS = %w(
http
https
dat
dweb
ipfs
ipns
ssb
gopher
xmpp
magnet
gemini
).freeze
CLASS_WHITELIST_TRANSFORMER = lambda do |env|
node = env[:node]
class_list = node['class']&.split(/[\t\n\f\r ]/)
return unless class_list
class_list.keep_if do |e|
next true if e =~ /^(h|p|u|dt|e)-/ # microformats classes
next true if e =~ /^(mention|hashtag)$/ # semantic classes
next true if e =~ /^(ellipsis|invisible)$/ # link formatting classes
end
node['class'] = class_list.join(' ')
end
UNSUPPORTED_HREF_TRANSFORMER = lambda do |env|
return unless env[:node_name] == 'a'
current_node = env[:node]
scheme = begin
if current_node['href'] =~ Sanitize::REGEX_PROTOCOL
Regexp.last_match(1).downcase
else
:relative
end
end
current_node.replace(current_node.text) unless LINK_PROTOCOLS.include?(scheme)
end
UNSUPPORTED_ELEMENTS_TRANSFORMER = lambda do |env|
return unless %w(h6).include?(env[:node_name])
current_node = env[:node]
case env[:node_name]
when 'li'
current_node.traverse do |node|
next unless %w(p ul ol li).include?(node.name)
node.add_next_sibling('<br>') if node.next_sibling
node.replace(node.children) unless node.text?
end
else
current_node.name = 'p'
end
end
MASTODON_STRICT ||= freeze_config(
elements: %w(p br span a abbr del pre blockquote code b strong i em h1 h2 h3 h4 h5 ul ol li img),
attributes: {
'a' => %w(href rel class title),
'span' => %w(class),
'abbr' => %w(title),
'blockquote' => %w(cite),
'img' => %w(src alt),
},
add_attributes: {
'a' => {
'rel' => 'nofollow noopener noreferrer',
'target' => '_blank',
},
'span' => {
'class' => 'article-type',
},
},
protocols: {
'a' => { 'href' => HTTP_PROTOCOLS },
'blockquote' => { 'cite' => HTTP_PROTOCOLS },
},
transformers: [
CLASS_WHITELIST_TRANSFORMER,
UNSUPPORTED_ELEMENTS_TRANSFORMER,
UNSUPPORTED_HREF_TRANSFORMER,
]
)
MASTODON_OEMBED ||= freeze_config merge(
RELAXED,
elements: RELAXED[:elements] + %w(audio embed iframe source video),
attributes: merge(
RELAXED[:attributes],
'audio' => %w(controls),
'embed' => %w(height src type width),
'iframe' => %w(allowfullscreen frameborder height scrolling src width),
'source' => %w(src type),
'video' => %w(controls height loop width),
'div' => [:data]
),
protocols: merge(
RELAXED[:protocols],
'embed' => { 'src' => HTTP_PROTOCOLS },
'iframe' => { 'src' => HTTP_PROTOCOLS },
'source' => { 'src' => HTTP_PROTOCOLS }
)
)
end
end

4
app/lib/settings/scoped_settings.rb
View File

@ -30,7 +30,7 @@ module Settings
def all_as_records
vars = thing_scoped
records = vars.each_with_object({}) { |r, h| h[r.var] = r }
records = vars.index_by(&:var)
Setting.default_settings.each do |key, default_value|
next if records.key?(key) || default_value.is_a?(Hash)
@ -63,7 +63,7 @@ module Settings
class << self
def default_settings
defaulting = DEFAULTING_TO_UNSCOPED.each_with_object({}) { |k, h| h[k] = Setting[k] }
defaulting = DEFAULTING_TO_UNSCOPED.index_with { |k| Setting[k] }
Setting.default_settings.merge!(defaulting)
end
end

198
app/lib/spam_check.rb
View File

@ -1,198 +0,0 @@
# frozen_string_literal: true
class SpamCheck
include Redisable
include ActionView::Helpers::TextHelper
# Threshold over which two Nilsimsa values are considered
# to refer to the same text
NILSIMSA_COMPARE_THRESHOLD = 95
# Nilsimsa doesn't work well on small inputs, so below
# this size, we check only for exact matches with MD5
NILSIMSA_MIN_SIZE = 10
# How long to keep the trail of digests between updates,
# there is no reason to store it forever
EXPIRE_SET_AFTER = 1.week.seconds
# How many digests to keep in an account's trail. If it's
# too small, spam could rotate around different message templates
MAX_TRAIL_SIZE = 10
# How many detected duplicates to allow through before
# considering the message as spam
THRESHOLD = 5
def initialize(status)
@account = status.account
@status = status
end
def skip?
disabled? || already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply?
end
def spam?
if insufficient_data?
false
elsif nilsimsa?
digests_over_threshold?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD }
else
digests_over_threshold?('md5') { |_, other_digest| other_digest == digest }
end
end
def flag!
auto_report_status!
end
def remember!
# The scores in sorted sets don't actually have enough bits to hold an exact
# value of our snowflake IDs, so we use it only for its ordering property. To
# get the correct status ID back, we have to save it in the string value
redis.zadd(redis_key, @status.id, digest_with_algorithm)
redis.zremrangebyrank(redis_key, 0, -(MAX_TRAIL_SIZE + 1))
redis.expire(redis_key, EXPIRE_SET_AFTER)
end
def reset!
redis.del(redis_key)
end
def hashable_text
return @hashable_text if defined?(@hashable_text)
@hashable_text = @status.text
@hashable_text = remove_mentions(@hashable_text)
@hashable_text = strip_tags(@hashable_text) unless @status.local?
@hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text)
@hashable_text = remove_whitespace(@hashable_text)
end
def insufficient_data?
hashable_text.blank?
end
def digest
@digest ||= begin
if nilsimsa?
Nilsimsa.new(hashable_text).hexdigest
else
Digest::MD5.hexdigest(hashable_text)
end
end
end
def digest_with_algorithm
if nilsimsa?
['nilsimsa', digest, @status.id].join(':')
else
['md5', digest, @status.id].join(':')
end
end
class << self
def perform(status)
spam_check = new(status)
return if spam_check.skip?
if spam_check.spam?
spam_check.flag!
else
spam_check.remember!
end
end
end
private
def disabled?
!Setting.spam_check_enabled
end
def remove_mentions(text)
return text.gsub(Account::MENTION_RE, '') if @status.local?
Nokogiri::HTML.fragment(text).tap do |html|
mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) }
html.traverse do |element|
element.unlink if element.name == 'a' && mentions.include?(element['href'])
end
end.to_s
end
def normalize_unicode(text)
text.unicode_normalize(:nfkc).downcase
end
def remove_whitespace(text)
text.gsub(/\s+/, ' ').strip
end
def auto_report_status!
status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable?
ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected'))
end
def already_flagged?
@account.silenced? || @account.targeted_reports.unresolved.where(account_id: -99).exists?
end
def trusted?
@account.trust_level > Account::TRUST_LEVELS[:untrusted] || (@account.local? && @account.user_staff?)
end
def no_unsolicited_mentions?
@status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) }
end
def solicited_reply?
!@status.thread.nil? && @status.thread.mentions.where(account: @account).exists?
end
def nilsimsa_compare_value(first, second)
first = [first].pack('H*')
second = [second].pack('H*')
bits = 0
0.upto(31) do |i|
bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord
end
128 - bits # -128 <= Nilsimsa Compare Value <= 128
end
def nilsimsa?
hashable_text.size > NILSIMSA_MIN_SIZE
end
def other_digests
redis.zrange(redis_key, 0, -1)
end
def digests_over_threshold?(filter_algorithm)
other_digests.select do |record|
algorithm, other_digest, status_id = record.split(':')
next unless algorithm == filter_algorithm
yield algorithm, other_digest, status_id
end.size >= THRESHOLD
end
def matching_status_ids
if nilsimsa?
other_digests.select { |record| record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD }.map { |record| record.split(':')[2] }.compact
else
other_digests.select { |record| record.start_with?('md5') && record.split(':')[1] == digest }.map { |record| record.split(':')[2] }.compact
end
end
def redis_key
@redis_key ||= "spam_check:#{@account.id}"
end
end

29
app/lib/status_reach_finder.rb
View File

@ -6,11 +6,22 @@ class StatusReachFinder
end
def inboxes
Account.where(id: reached_account_ids).inboxes
(reached_account_inboxes + followers_inboxes + relay_inboxes).uniq
end
private
def reached_account_inboxes
# When the status is a reblog, there are no interactions with it
# directly, we assume all interactions are with the original one
if @status.reblog?
[]
else
Account.where(id: reached_account_ids).inboxes
end
end
def reached_account_ids
[
replied_to_account_id,
@ -49,4 +60,20 @@ class StatusReachFinder
def replies_account_ids
@status.replies.pluck(:account_id)
end
def followers_inboxes
if @status.in_reply_to_local_account? && @status.distributable?
@status.account.followers.or(@status.thread.account.followers).inboxes
else
@status.account.followers.inboxes
end
end
def relay_inboxes
if @status.public_visibility?
Relay.enabled.pluck(:inbox_url)
else
[]
end
end
end

8
app/lib/tag_manager.rb
View File

@ -22,14 +22,6 @@ class TagManager
uri.normalized_host
end
def same_acct?(canonical, needle)
return true if canonical.casecmp(needle).zero?
username, domain = needle.split('@')
local_domain?(domain) && canonical.casecmp(username).zero?
end
def local_url?(url)
uri = Addressable::URI.parse(url).normalize
domain = uri.host + (uri.port ? ":#{uri.port}" : '')

32
app/lib/validation_error_formatter.rb Normal file
View File

@ -0,0 +1,32 @@
# frozen_string_literal: true
class ValidationErrorFormatter
def initialize(error, aliases = {})
@error = error
@aliases = aliases
end
def as_json
{ error: @error.to_s, details: details }
end
private
def details
h = {}
errors.details.each_pair do |attribute_name, attribute_errors|
messages = errors.messages[attribute_name]
h[@aliases[attribute_name] || attribute_name] = attribute_errors.map.with_index do |error, index|
{ error: 'ERR_' + error[:error].to_s.upcase, description: messages[index] }
end
end
h
end
def errors
@errors ||= @error.record.errors
end
end

54
app/lib/video_metadata_extractor.rb Normal file
View File

@ -0,0 +1,54 @@
# frozen_string_literal: true
class VideoMetadataExtractor
attr_reader :duration, :bitrate, :video_codec, :audio_codec,
:colorspace, :width, :height, :frame_rate
def initialize(path)
@path = path
@metadata = Oj.load(ffmpeg_command_output, mode: :strict, symbol_keys: true)
parse_metadata
rescue Terrapin::ExitStatusError, Oj::ParseError
@invalid = true
rescue Terrapin::CommandNotFoundError
raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffprobe` command. Please install ffmpeg.'
end
def valid?
!@invalid
end
private
def ffmpeg_command_output
command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel')
command.run(path: @path, format: 'json', loglevel: 'fatal')
end
def parse_metadata
if @metadata.key?(:format)
@duration = @metadata[:format][:duration].to_f
@bitrate = @metadata[:format][:bit_rate].to_i
end
if @metadata.key?(:streams)
video_streams = @metadata[:streams].select { |stream| stream[:codec_type] == 'video' }
audio_streams = @metadata[:streams].select { |stream| stream[:codec_type] == 'audio' }
if (video_stream = video_streams.first)
@video_codec = video_stream[:codec_name]
@colorspace = video_stream[:pix_fmt]
@width = video_stream[:width]
@height = video_stream[:height]
@frame_rate = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate])
end
if (audio_stream = audio_streams.first)
@audio_codec = audio_stream[:codec_name]
end
end
@invalid = true if @metadata.key?(:error)
end
end

14
app/lib/webfinger.rb
View File

@ -21,7 +21,7 @@ class Webfinger
private
def links
@links ||= @json['links'].map { |link| [link['rel'], link] }.to_h
@links ||= @json['links'].index_by { |link| link['rel'] }
end
end
@ -88,10 +88,18 @@ class Webfinger
end
def standard_url
"https://#{@domain}/.well-known/webfinger?resource=#{@uri}"
if @domain.end_with? ".onion"
"http://#{@domain}/.well-known/webfinger?resource=#{@uri}"
else
"https://#{@domain}/.well-known/webfinger?resource=#{@uri}"
end
end
def host_meta_url
"https://#{@domain}/.well-known/host-meta"
if @domain.end_with? ".onion"
"http://#{@domain}/.well-known/host-meta"
else
"https://#{@domain}/.well-known/host-meta"
end
end
end