Merge tag 'v2.8.0' into instance_only_statuses

This commit is contained in:
Renato "Lond" Cerqueira
2019-04-13 23:47:24 +02:00
689 changed files with 25483 additions and 9047 deletions

View File

@ -1,11 +1,12 @@
# frozen_string_literal: true
class AccountSearchService < BaseService
attr_reader :query, :limit, :options, :account
attr_reader :query, :limit, :offset, :options, :account
def call(query, limit, account = nil, options = {})
def call(query, account = nil, options = {})
@query = query.strip
@limit = limit
@limit = options[:limit].to_i
@offset = options[:offset].to_i
@options = options
@account = account
@ -83,11 +84,11 @@ class AccountSearchService < BaseService
end
def advanced_search_results
Account.advanced_search_for(terms_for_query, account, limit, options[:following])
Account.advanced_search_for(terms_for_query, account, limit, options[:following], offset)
end
def simple_search_results
Account.search_for(terms_for_query, limit)
Account.search_for(terms_for_query, limit, offset)
end
def terms_for_query

View File

@ -0,0 +1,11 @@
# frozen_string_literal: true
class ActivityPub::FetchRemotePollService < BaseService
include JsonLdHelper
def call(poll, on_behalf_of = nil)
json = fetch_resource(poll.status.uri, true, on_behalf_of)
return unless supported_context?(json)
ActivityPub::ProcessPollService.new.call(poll, json)
end
end

View File

@ -0,0 +1,49 @@
# frozen_string_literal: true
class ActivityPub::FetchRepliesService < BaseService
include JsonLdHelper
def call(parent_status, collection_or_uri, allow_synchronous_requests = true)
@account = parent_status.account
@allow_synchronous_requests = allow_synchronous_requests
@items = collection_items(collection_or_uri)
return if @items.nil?
FetchReplyWorker.push_bulk(filtered_replies)
@items
end
private
def collection_items(collection_or_uri)
collection = fetch_collection(collection_or_uri)
return unless collection.is_a?(Hash)
collection = fetch_collection(collection['first']) if collection['first'].present?
return unless collection.is_a?(Hash)
case collection['type']
when 'Collection', 'CollectionPage'
collection['items']
when 'OrderedCollection', 'OrderedCollectionPage'
collection['orderedItems']
end
end
def fetch_collection(collection_or_uri)
return collection_or_uri if collection_or_uri.is_a?(Hash)
return unless @allow_synchronous_requests
return if invalid_origin?(collection_or_uri)
fetch_resource_without_id_validation(collection_or_uri, nil, true)
end
def filtered_replies
# Only fetch replies to the same server as the original status to avoid
# amplification attacks.
# Also limit to 5 fetched replies to limit potential for DoS.
@items.map { |item| value_or_id(item) }.reject { |uri| invalid_origin?(uri) }.take(5)
end
end

View File

@ -24,6 +24,7 @@ class ActivityPub::ProcessAccountService < BaseService
create_account if @account.nil?
update_account
process_tags
process_attachments
else
raise Mastodon::RaceConditionError
end
@ -151,7 +152,7 @@ class ActivityPub::ProcessAccountService < BaseService
def property_values
return unless @json['attachment'].is_a?(Array)
@json['attachment'].select { |attachment| attachment['type'] == 'PropertyValue' }.map { |attachment| attachment.slice('name', 'value') }
as_array(@json['attachment']).select { |attachment| attachment['type'] == 'PropertyValue' }.map { |attachment| attachment.slice('name', 'value') }
end
def mismatching_origin?(url)
@ -231,6 +232,23 @@ class ActivityPub::ProcessAccountService < BaseService
end
end
def process_attachments
return if @json['attachment'].blank?
previous_proofs = @account.identity_proofs.to_a
current_proofs = []
as_array(@json['attachment']).each do |attachment|
next unless equals_or_includes?(attachment['type'], 'IdentityProof')
current_proofs << process_identity_proof(attachment)
end
previous_proofs.each do |previous_proof|
next if current_proofs.any? { |current_proof| current_proof.id == previous_proof.id }
previous_proof.delete
end
end
def process_emoji(tag)
return if skip_download?
return if tag['name'].blank? || tag['icon'].blank? || tag['icon']['url'].blank?
@ -247,4 +265,12 @@ class ActivityPub::ProcessAccountService < BaseService
emoji.image_remote_url = image_url
emoji.save
end
def process_identity_proof(attachment)
provider = attachment['signatureAlgorithm']
provider_username = attachment['name']
token = attachment['signatureValue']
@account.identity_proofs.where(provider: provider, provider_username: provider_username).find_or_create_by(provider: provider, provider_username: provider_username, token: token)
end
end

View File

@ -0,0 +1,60 @@
# frozen_string_literal: true
class ActivityPub::ProcessPollService < BaseService
include JsonLdHelper
def call(poll, json)
@json = json
return unless expected_type?
previous_expires_at = poll.expires_at
expires_at = begin
if @json['closed'].is_a?(String)
@json['closed']
elsif !@json['closed'].nil? && !@json['closed'].is_a?(FalseClass)
Time.now.utc
else
@json['endTime']
end
end
items = begin
if @json['anyOf'].is_a?(Array)
@json['anyOf']
else
@json['oneOf']
end
end
latest_options = items.map { |item| item['name'].presence || item['content'] }
# If for some reasons the options were changed, it invalidates all previous
# votes, so we need to remove them
poll.votes.delete_all if latest_options != poll.options
begin
poll.update!(
last_fetched_at: Time.now.utc,
expires_at: expires_at,
options: latest_options,
cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 }
)
rescue ActiveRecord::StaleObjectError
poll.reload
retry
end
# If the poll had no expiration date set but now has, and people have voted,
# schedule a notification.
if previous_expires_at.nil? && poll.expires_at.present? && poll.votes.exists?
PollExpirationNotifyWorker.perform_at(poll.expires_at + 5.minutes, poll.id)
end
end
private
def expected_type?
equals_or_includes_any?(@json['type'], %w(Question))
end
end

View File

@ -18,6 +18,6 @@ class AppSignUpService < BaseService
private
def allowed_registrations?
Setting.open_registrations && !Rails.configuration.x.single_user_mode
Setting.registrations_mode != 'none' && !Rails.configuration.x.single_user_mode
end
end

View File

@ -35,6 +35,8 @@ class BatchedRemoveStatusService < BaseService
statuses.group_by(&:account_id).each_value do |account_statuses|
account = account_statuses.first.account
next unless account
unpush_from_home_timelines(account, account_statuses)
unpush_from_list_timelines(account, account_statuses)

View File

@ -0,0 +1,101 @@
# frozen_string_literal: true
require 'csv'
class ImportService < BaseService
ROWS_PROCESSING_LIMIT = 20_000
def call(import)
@import = import
@account = @import.account
case @import.type
when 'following'
import_follows!
when 'blocking'
import_blocks!
when 'muting'
import_mutes!
when 'domain_blocking'
import_domain_blocks!
end
end
private
def import_follows!
parse_import_data!(['Account address'])
import_relationships!('follow', 'unfollow', @account.following, follow_limit, reblogs: 'Show boosts')
end
def import_blocks!
parse_import_data!(['Account address'])
import_relationships!('block', 'unblock', @account.blocking, ROWS_PROCESSING_LIMIT)
end
def import_mutes!
parse_import_data!(['Account address'])
import_relationships!('mute', 'unmute', @account.muting, ROWS_PROCESSING_LIMIT, notifications: 'Hide notifications')
end
def import_domain_blocks!
parse_import_data!(['#domain'])
items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#domain'].strip }
if @import.overwrite?
presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true }
@account.domain_blocks.find_each do |domain_block|
if presence_hash[domain_block.domain]
items.delete(domain_block.domain)
else
@account.unblock_domain!(domain_block.domain)
end
end
end
items.each do |domain|
@account.block_domain!(domain)
end
AfterAccountDomainBlockWorker.push_bulk(items) do |domain|
[@account.id, domain]
end
end
def import_relationships!(action, undo_action, overwrite_scope, limit, extra_fields = {})
items = @data.take(limit).map { |row| [row['Account address']&.strip, Hash[extra_fields.map { |key, header| [key, row[header]&.strip] }]] }.reject { |(id, _)| id.blank? }
if @import.overwrite?
presence_hash = items.each_with_object({}) { |(id, extra), mapping| mapping[id] = [true, extra] }
overwrite_scope.find_each do |target_account|
if presence_hash[target_account.acct]
items.delete(target_account.acct)
extra = presence_hash[target_account.acct][1]
Import::RelationshipWorker.perform_async(@account.id, target_account.acct, action, extra)
else
Import::RelationshipWorker.perform_async(@account.id, target_account.acct, undo_action)
end
end
end
Import::RelationshipWorker.push_bulk(items) do |acct, extra|
[@account.id, acct, action, extra]
end
end
def parse_import_data!(default_headers)
data = CSV.parse(import_data, headers: true)
data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(' ')
@data = data.reject(&:blank?)
end
def import_data
Paperclip.io_adapters.for(@import.data).read
end
def follow_limit
FollowLimitValidator.limit_for_account(@account)
end
end

View File

@ -38,6 +38,10 @@ class NotifyService < BaseService
false
end
def blocked_poll?
false
end
def following_sender?
return @following_sender if defined?(@following_sender)
@following_sender = @recipient.following?(@notification.from_account) || @recipient.requested?(@notification.from_account)
@ -88,7 +92,7 @@ class NotifyService < BaseService
def blocked?
blocked = @recipient.suspended? # Skip if the recipient account is suspended anyway
blocked ||= from_self? # Skip for interactions with self
blocked ||= from_self? && @notification.type != :poll # Skip for interactions with self
return blocked if message? && from_staff?

View File

@ -15,6 +15,7 @@ class PostStatusService < BaseService
# @option [String] :spoiler_text
# @option [String] :language
# @option [String] :scheduled_at
# @option [Hash] :poll Optional poll to attach
# @option [Enumerable] :media_ids Optional array of media IDs to attach
# @option [Doorkeeper::Application] :application
# @option [String] :idempotency Optional idempotency key
@ -69,6 +70,7 @@ class PostStatusService < BaseService
def schedule_status!
status_for_validation = @account.statuses.build(status_attributes)
if status_for_validation.valid?
status_for_validation.destroy
@ -92,17 +94,17 @@ class PostStatusService < BaseService
def postprocess_status!
LinkCrawlWorker.perform_async(@status.id) unless @status.spoiler_text?
DistributionWorker.perform_async(@status.id)
unless @status.local_only?
Pubsubhubbub::DistributionWorker.perform_async(@status.stream_entry.id)
ActivityPub::DistributionWorker.perform_async(@status.id)
PollExpirationNotifyWorker.perform_at(@status.poll.expires_at, @status.poll.id) if @status.poll
end
end
def validate_media!
return if @options[:media_ids].blank? || !@options[:media_ids].is_a?(Enumerable)
raise Mastodon::ValidationError, I18n.t('media_attachments.validations.too_many') if @options[:media_ids].size > 4
raise Mastodon::ValidationError, I18n.t('media_attachments.validations.too_many') if @options[:media_ids].size > 4 || @options[:poll].present?
@media = @account.media_attachments.where(status_id: nil).where(id: @options[:media_ids].take(4).map(&:to_i))
@ -161,13 +163,14 @@ class PostStatusService < BaseService
text: @text,
media_attachments: @media || [],
thread: @in_reply_to,
poll_attributes: poll_attributes,
sensitive: (@options[:sensitive].nil? ? @account.user&.setting_default_sensitive : @options[:sensitive]) || @options[:spoiler_text].present?,
spoiler_text: @options[:spoiler_text] || '',
visibility: @visibility,
language: language_from_option(@options[:language]) || @account.user&.setting_default_language&.presence || LanguageDetector.instance.detect(@text, @account),
application: @options[:application],
local_only: local_only_option(@options[:local_only], @in_reply_to, @account.user&.setting_default_federation),
}
}.compact
end
def scheduled_status_attributes
@ -178,6 +181,12 @@ class PostStatusService < BaseService
}
end
def poll_attributes
return if @options[:poll].blank?
@options[:poll].merge(account: @account)
end
def scheduled_options
@options.tap do |options_hash|
options_hash[:in_reply_to_id] = options_hash.delete(:thread)&.id

View File

@ -2,12 +2,22 @@
class ProcessHashtagsService < BaseService
def call(status, tags = [])
tags = Extractor.extract_hashtags(status.text) if status.local?
tags = Extractor.extract_hashtags(status.text) if status.local?
records = []
tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name|
tag = Tag.where(name: name).first_or_create(name: name)
status.tags << tag
records << tag
TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility?
end
return unless status.public_visibility? || status.unlisted_visibility?
status.account.featured_tags.where(tag_id: records.map(&:id)).each do |featured_tag|
featured_tag.increment(status.created_at)
end
end
end

View File

@ -7,8 +7,9 @@ class ReblogService < BaseService
# Reblog a status and notify its remote author
# @param [Account] account Account to reblog from
# @param [Status] reblogged_status Status to be reblogged
# @param [Hash] options
# @return [Status]
def call(account, reblogged_status)
def call(account, reblogged_status, options = {})
reblogged_status = reblogged_status.reblog if reblogged_status.reblog?
authorize_with account, reblogged_status, :reblog?
@ -17,7 +18,7 @@ class ReblogService < BaseService
return reblog unless reblog.nil?
reblog = account.statuses.create!(reblog: reblogged_status, text: '')
reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: options[:visibility] || account.user&.setting_default_privacy)
DistributionWorker.perform_async(reblog.id)
@ -38,7 +39,7 @@ class ReblogService < BaseService
reblogged_status = reblog.reblog
if reblogged_status.account.local?
NotifyService.new.call(reblogged_status.account, reblog)
LocalNotificationWorker.perform_async(reblogged_status.account_id, reblog.id, reblog.class.name)
elsif reblogged_status.account.ostatus?
NotificationWorker.perform_async(stream_entry_to_xml(reblog.stream_entry), reblog.account_id, reblogged_status.account_id)
elsif reblogged_status.account.activitypub? && !reblogged_status.account.following?(reblog.account)

View File

@ -10,20 +10,26 @@ class RemoveStatusService < BaseService
@account = status.account
@tags = status.tags.pluck(:name).to_a
@mentions = status.active_mentions.includes(:account).to_a
@reblogs = status.reblogs.to_a
@reblogs = status.reblogs.includes(:account).to_a
@stream_entry = status.stream_entry
@options = options
remove_from_self if status.account.local?
remove_from_followers
remove_from_lists
remove_from_affected
remove_reblogs
remove_from_hashtags
remove_from_public
remove_from_media if status.media_attachments.any?
RedisLock.acquire(lock_options) do |lock|
if lock.acquired?
remove_from_self if status.account.local?
remove_from_followers
remove_from_lists
remove_from_affected
remove_reblogs
remove_from_hashtags
remove_from_public
remove_from_media if status.media_attachments.any?
@status.destroy!
@status.destroy!
else
raise Mastodon::RaceConditionError
end
end
# There is no reason to send out Undo activities when the
# cause is that the original object has been removed, since
@ -77,8 +83,8 @@ class RemoveStatusService < BaseService
end
# ActivityPub
ActivityPub::DeliveryWorker.push_bulk(target_accounts.select(&:activitypub?).uniq(&:inbox_url)) do |target_account|
[signed_activity_json, @account.id, target_account.inbox_url]
ActivityPub::DeliveryWorker.push_bulk(target_accounts.select(&:activitypub?).uniq(&:preferred_inbox_url)) do |target_account|
[signed_activity_json, @account.id, target_account.preferred_inbox_url]
end
end
@ -131,6 +137,10 @@ class RemoveStatusService < BaseService
end
def remove_from_hashtags
@account.featured_tags.where(tag_id: @status.tags.pluck(:id)).each do |featured_tag|
featured_tag.decrement(@status.id)
end
return unless @status.public_visibility?
@tags.each do |hashtag|
@ -152,4 +162,8 @@ class RemoveStatusService < BaseService
redis.publish('timeline:public:media', @payload)
redis.publish('timeline:public:local:media', @payload) if @status.local?
end
def lock_options
{ redis: Redis.current, key: "distribute:#{@status.id}" }
end
end

View File

@ -21,7 +21,8 @@ class ReportService < BaseService
@report = @source_account.reports.create!(
target_account: @target_account,
status_ids: @status_ids,
comment: @comment
comment: @comment,
uri: @options[:uri]
)
end

View File

@ -20,7 +20,7 @@ class ResolveURLService < BaseService
def process_url
if equals_or_includes_any?(type, %w(Application Group Organization Person Service))
FetchRemoteAccountService.new.call(atom_url, body, protocol)
elsif equals_or_includes_any?(type, %w(Note Article Image Video Page))
elsif equals_or_includes_any?(type, %w(Note Article Image Video Page Question))
FetchRemoteStatusService.new.call(atom_url, body, protocol)
end
end

View File

@ -1,18 +1,18 @@
# frozen_string_literal: true
class SearchService < BaseService
attr_accessor :query, :account, :limit, :resolve
def call(query, limit, resolve = false, account = nil)
def call(query, account, limit, options = {})
@query = query.strip
@account = account
@limit = limit
@resolve = resolve
@options = options
@limit = limit.to_i
@offset = options[:type].blank? ? 0 : options[:offset].to_i
@resolve = options[:resolve] || false
default_results.tap do |results|
if url_query?
results.merge!(url_resource_results) unless url_resource.nil?
elsif query.present?
elsif @query.present?
results[:accounts] = perform_accounts_search! if account_searchable?
results[:statuses] = perform_statuses_search! if full_text_searchable?
results[:hashtags] = perform_hashtags_search! if hashtag_searchable?
@ -23,23 +23,46 @@ class SearchService < BaseService
private
def perform_accounts_search!
AccountSearchService.new.call(query, limit, account, resolve: resolve)
AccountSearchService.new.call(
@query,
@account,
limit: @limit,
resolve: @resolve,
offset: @offset
)
end
def perform_statuses_search!
statuses = StatusesIndex.filter(term: { searchable_by: account.id })
.query(multi_match: { type: 'most_fields', query: query, operator: 'and', fields: %w(text text.stemmed) })
.limit(limit)
.objects
.compact
definition = StatusesIndex.filter(term: { searchable_by: @account.id })
.query(multi_match: { type: 'most_fields', query: @query, operator: 'and', fields: %w(text text.stemmed) })
statuses.reject { |status| StatusFilter.new(status, account).filtered? }
if @options[:account_id].present?
definition = definition.filter(term: { account_id: @options[:account_id] })
end
if @options[:min_id].present? || @options[:max_id].present?
range = {}
range[:gt] = @options[:min_id].to_i if @options[:min_id].present?
range[:lt] = @options[:max_id].to_i if @options[:max_id].present?
definition = definition.filter(range: { id: range })
end
results = definition.limit(@limit).offset(@offset).objects.compact
account_ids = results.map(&:account_id)
account_domains = results.map(&:account_domain)
preloaded_relations = relations_map_for_account(@account, account_ids, account_domains)
results.reject { |status| StatusFilter.new(status, @account, preloaded_relations).filtered? }
rescue Faraday::ConnectionFailed
[]
end
def perform_hashtags_search!
Tag.search_for(query.gsub(/\A#/, ''), limit)
Tag.search_for(
@query.gsub(/\A#/, ''),
@limit,
@offset
)
end
def default_results
@ -47,7 +70,7 @@ class SearchService < BaseService
end
def url_query?
query =~ /\Ahttps?:\/\//
@options[:type].blank? && @query =~ /\Ahttps?:\/\//
end
def url_resource_results
@ -55,7 +78,7 @@ class SearchService < BaseService
end
def url_resource
@_url_resource ||= ResolveURLService.new.call(query, on_behalf_of: @account)
@_url_resource ||= ResolveURLService.new.call(@query, on_behalf_of: @account)
end
def url_resource_symbol
@ -64,14 +87,37 @@ class SearchService < BaseService
def full_text_searchable?
return false unless Chewy.enabled?
!account.nil? && !((query.start_with?('#') || query.include?('@')) && !query.include?(' '))
statuses_search? && !@account.nil? && !((@query.start_with?('#') || @query.include?('@')) && !@query.include?(' '))
end
def account_searchable?
!(query.include?('@') && query.include?(' '))
account_search? && !(@query.include?('@') && @query.include?(' '))
end
def hashtag_searchable?
!query.include?('@')
hashtag_search? && !@query.include?('@')
end
def account_search?
@options[:type].blank? || @options[:type] == 'accounts'
end
def hashtag_search?
@options[:type].blank? || @options[:type] == 'hashtags'
end
def statuses_search?
@options[:type].blank? || @options[:type] == 'statuses'
end
def relations_map_for_account(account, account_ids, domains)
{
blocking: Account.blocking_map(account_ids, account.id),
blocked_by: Account.blocked_by_map(account_ids, account.id),
muting: Account.muting_map(account_ids, account.id),
following: Account.following_map(account_ids, account.id),
domain_blocking_by_domain: Account.domain_blocking_map_by_domain(domains, account.id),
}
end
end

View File

@ -41,6 +41,7 @@ class SuspendAccountService < BaseService
@account = account
@options = options
reject_follows!
purge_user!
purge_profile!
purge_content!
@ -48,6 +49,14 @@ class SuspendAccountService < BaseService
private
def reject_follows!
return if @account.local? || !@account.activitypub?
ActivityPub::DeliveryWorker.push_bulk(Follow.where(account: @account)) do |follow|
[build_reject_json(follow), follow.target_account_id, follow.account.inbox_url]
end
end
def purge_user!
return if !@account.local? || @account.user.nil?
@ -59,7 +68,7 @@ class SuspendAccountService < BaseService
end
def purge_content!
distribute_delete_actor! if @account.local?
distribute_delete_actor! if @account.local? && !@options[:skip_distribution]
@account.statuses.reorder(nil).find_in_batches do |statuses|
BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:destroy])
@ -84,7 +93,7 @@ class SuspendAccountService < BaseService
@account.locked = false
@account.display_name = ''
@account.note = ''
@account.fields = {}
@account.fields = []
@account.statuses_count = 0
@account.followers_count = 0
@account.following_count = 0
@ -120,6 +129,14 @@ class SuspendAccountService < BaseService
@delete_actor_json = Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(@account))
end
def build_reject_json(follow)
ActiveModelSerializers::SerializableResource.new(
follow,
serializer: ActivityPub::RejectFollowSerializer,
adapter: ActivityPub::Adapter
).to_json
end
def delivery_inboxes
@delivery_inboxes ||= @account.followers.inboxes + Relay.enabled.pluck(:inbox_url)
end

View File

@ -0,0 +1,59 @@
# frozen_string_literal: true
class VoteService < BaseService
include Authorization
def call(account, poll, choices)
authorize_with account, poll, :vote?
@account = account
@poll = poll
@choices = choices
@votes = []
ApplicationRecord.transaction do
@choices.each do |choice|
@votes << @poll.votes.create!(account: @account, choice: choice)
end
end
ActivityTracker.increment('activity:interactions')
if @poll.account.local?
distribute_poll!
else
deliver_votes!
queue_final_poll_check!
end
end
private
def distribute_poll!
return if @poll.hide_totals?
ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, @poll.status.id)
end
def queue_final_poll_check!
return unless @poll.expires?
PollExpirationNotifyWorker.perform_at(@poll.expires_at + 5.minutes, @poll.id)
end
def deliver_votes!
@votes.each do |vote|
ActivityPub::DeliveryWorker.perform_async(
build_json(vote),
@account.id,
@poll.account.inbox_url
)
end
end
def build_json(vote)
ActiveModelSerializers::SerializableResource.new(
vote,
serializer: ActivityPub::VoteSerializer,
adapter: ActivityPub::Adapter
).to_json
end
end