Merge tag 'v2.7.0rc1' into instance_only_statuses

This commit is contained in:
Renato "Lond" Cerqueira
2019-01-09 10:47:10 +01:00
585 changed files with 16065 additions and 8146 deletions

View File

@ -5,8 +5,8 @@ class ActivityPub::FetchRemoteAccountService < BaseService
SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
# Does a WebFinger roundtrip on each call
def call(uri, id: true, prefetched_body: nil, break_on_redirect: false)
# Does a WebFinger roundtrip on each call, unless `only_key` is true
def call(uri, id: true, prefetched_body: nil, break_on_redirect: false, only_key: false)
return ActivityPub::TagManager.instance.uri_to_resource(uri, Account) if ActivityPub::TagManager.instance.local_uri?(uri)
@json = if prefetched_body.nil?
@ -21,9 +21,9 @@ class ActivityPub::FetchRemoteAccountService < BaseService
@username = @json['preferredUsername']
@domain = Addressable::URI.parse(@uri).normalized_host
return unless verified_webfinger?
return unless only_key || verified_webfinger?
ActivityPub::ProcessAccountService.new.call(@username, @domain, @json)
ActivityPub::ProcessAccountService.new.call(@username, @domain, @json, only_key: only_key)
rescue Oj::ParseError
nil
end

View File

@ -33,8 +33,10 @@ class ActivityPub::ProcessAccountService < BaseService
after_protocol_change! if protocol_changed?
after_key_change! if key_changed? && !@options[:signed_with_known_key]
check_featured_collection! if @account.featured_collection_url.present?
check_links! unless @account.fields.empty?
unless @options[:only_key]
check_featured_collection! if @account.featured_collection_url.present?
check_links! unless @account.fields.empty?
end
@account
rescue Oj::ParseError
@ -54,11 +56,11 @@ class ActivityPub::ProcessAccountService < BaseService
end
def update_account
@account.last_webfingered_at = Time.now.utc
@account.last_webfingered_at = Time.now.utc unless @options[:only_key]
@account.protocol = :activitypub
set_immediate_attributes!
set_fetchable_attributes!
set_fetchable_attributes! unless @options[:only_keys]
@account.save_with_optional_media!
end
@ -75,6 +77,7 @@ class ActivityPub::ProcessAccountService < BaseService
@account.note = @json['summary'] || ''
@account.locked = @json['manuallyApprovesFollowers'] || false
@account.fields = property_values || {}
@account.also_known_as = as_array(@json['alsoKnownAs'] || []).map { |item| value_or_id(item) }
@account.actor_type = actor_type
end

View File

@ -27,7 +27,7 @@ class ActivityPub::ProcessCollectionService < BaseService
private
def different_actor?
@json['actor'].present? && value_or_id(@json['actor']) != @account.uri && @json['signature'].present?
@json['actor'].present? && value_or_id(@json['actor']) != @account.uri
end
def process_items(items)

View File

@ -31,11 +31,11 @@ class AfterBlockDomainFromAccountService < BaseService
return unless follow.account.activitypub?
json = Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
json = ActiveModelSerializers::SerializableResource.new(
follow,
serializer: ActivityPub::RejectFollowSerializer,
adapter: ActivityPub::Adapter
).as_json).sign!(@account))
).to_json
ActivityPub::DeliveryWorker.perform_async(json, @account.id, follow.account.inbox_url)
end

View File

@ -0,0 +1,23 @@
# frozen_string_literal: true
class AppSignUpService < BaseService
def call(app, params)
return unless allowed_registrations?
user_params = params.slice(:email, :password, :agreement, :locale)
account_params = params.slice(:username)
user = User.create!(user_params.merge(created_by_application: app, password_confirmation: user_params[:password], account_attributes: account_params))
Doorkeeper::AccessToken.create!(application: app,
resource_owner_id: user.id,
scopes: app.scopes,
expires_in: Doorkeeper.configuration.access_token_expires_in,
use_refresh_token: Doorkeeper.configuration.refresh_token_enabled?)
end
private
def allowed_registrations?
Setting.open_registrations && !Rails.configuration.x.single_user_mode
end
end

View File

@ -24,11 +24,11 @@ class AuthorizeFollowService < BaseService
end
def build_json(follow_request)
Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
ActiveModelSerializers::SerializableResource.new(
follow_request,
serializer: ActivityPub::AcceptFollowSerializer,
adapter: ActivityPub::Adapter
).as_json).sign!(follow_request.target_account))
).to_json
end
def build_xml(follow_request)

View File

@ -9,7 +9,9 @@ class BatchedRemoveStatusService < BaseService
# Remove statuses from home feeds
# Push delete events to streaming API for home feeds and public feeds
# @param [Status] statuses A preferably batched array of statuses
def call(statuses)
# @param [Hash] options
# @option [Boolean] :skip_side_effects
def call(statuses, **options)
statuses = Status.where(id: statuses.map(&:id)).includes(:account, :stream_entry).flat_map { |status| [status] + status.reblogs.includes(:account, :stream_entry).to_a }
@mentions = statuses.each_with_object({}) { |s, h| h[s.id] = s.active_mentions.includes(:account).to_a }
@ -26,6 +28,8 @@ class BatchedRemoveStatusService < BaseService
status.destroy
end
return if options[:skip_side_effects]
# Batch by source account
statuses.group_by(&:account_id).each_value do |account_statuses|
account = account_statuses.first.account

View File

@ -1,8 +1,6 @@
# frozen_string_literal: true
class BlockService < BaseService
include StreamEntryRenderer
def call(account, target_account)
return if account.id == target_account.id
@ -27,11 +25,11 @@ class BlockService < BaseService
end
def build_json(block)
Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
ActiveModelSerializers::SerializableResource.new(
block,
serializer: ActivityPub::BlockSerializer,
adapter: ActivityPub::Adapter
).as_json).sign!(block.account))
).to_json
end
def build_xml(block)

View File

@ -137,7 +137,8 @@ class FetchLinkCardService < BaseService
detector.strip_tags = true
guess = detector.detect(@html, @html_charset)
page = Nokogiri::HTML(@html, nil, guess&.fetch(:encoding, nil))
encoding = guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil
page = Nokogiri::HTML(@html, nil, encoding)
player_url = meta_property(page, 'twitter:player')
if player_url && !bad_url?(Addressable::URI.parse(player_url))

View File

@ -1,8 +1,6 @@
# frozen_string_literal: true
class FollowService < BaseService
include StreamEntryRenderer
# Follow a remote user, notify remote user about the follow
# @param [Account] source_account From which to follow
# @param [String, Account] uri User URI to follow in the form of username@domain (or account record)
@ -12,7 +10,7 @@ class FollowService < BaseService
target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true)
raise ActiveRecord::RecordNotFound if target_account.nil? || target_account.id == source_account.id || target_account.suspended?
raise Mastodon::NotPermittedError if target_account.blocking?(source_account) || source_account.blocking?(target_account)
raise Mastodon::NotPermittedError if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved?
if source_account.following?(target_account)
# We're already following this account, but we'll call follow! again to
@ -82,10 +80,10 @@ class FollowService < BaseService
end
def build_json(follow_request)
Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
ActiveModelSerializers::SerializableResource.new(
follow_request,
serializer: ActivityPub::FollowSerializer,
adapter: ActivityPub::Adapter
).as_json).sign!(follow_request.account))
).to_json
end
end

View File

@ -0,0 +1,20 @@
# frozen_string_literal: true
class HashtagQueryService < BaseService
def call(tag, params, account = nil, local = false)
tags = tags_for(Array(tag.name) | Array(params[:any])).pluck(:id)
all = tags_for(params[:all])
none = tags_for(params[:none])
Status.distinct
.as_tag_timeline(tags, account, local)
.tagged_with_all(all)
.tagged_with_none(none)
end
private
def tags_for(tags)
Tag.where(name: tags.map(&:downcase)) if tags.presence
end
end

View File

@ -1,79 +1,105 @@
# frozen_string_literal: true
class PostStatusService < BaseService
MIN_SCHEDULE_OFFSET = 5.minutes.freeze
# Post a text status update, fetch and notify remote users mentioned
# @param [Account] account Account from which to post
# @param [String] text Message
# @param [Status] in_reply_to Optional status to reply to
# @param [Hash] options
# @option [String] :text Message
# @option [Status] :thread Optional status to reply to
# @option [Boolean] :sensitive
# @option [String] :visibility
# @option [String] :spoiler_text
# @option [String] :language
# @option [String] :scheduled_at
# @option [Enumerable] :media_ids Optional array of media IDs to attach
# @option [Doorkeeper::Application] :application
# @option [String] :idempotency Optional idempotency key
# @return [Status]
def call(account, text, in_reply_to = nil, **options)
if options[:idempotency].present?
existing_id = redis.get("idempotency:status:#{account.id}:#{options[:idempotency]}")
return Status.find(existing_id) if existing_id
def call(account, options = {})
@account = account
@options = options
@text = @options[:text] || ''
@in_reply_to = @options[:thread]
return idempotency_duplicate if idempotency_given? && idempotency_duplicate?
validate_media!
preprocess_attributes!
if scheduled?
schedule_status!
else
process_status!
postprocess_status!
bump_potential_friendship!
end
media = validate_media!(options[:media_ids])
status = nil
text = options.delete(:spoiler_text) if text.blank? && options[:spoiler_text].present?
redis.setex(idempotency_key, 3_600, @status.id) if idempotency_given?
ApplicationRecord.transaction do
status = account.statuses.create!(text: text,
media_attachments: media || [],
thread: in_reply_to,
sensitive: (options[:sensitive].nil? ? account.user&.setting_default_sensitive : options[:sensitive]) || options[:spoiler_text].present?,
spoiler_text: options[:spoiler_text] || '',
visibility: options[:visibility] || account.user&.setting_default_privacy,
language: language_from_option(options[:language]) || account.user&.setting_default_language&.presence || LanguageDetector.instance.detect(text, account),
application: options[:application],
local_only: local_only_option(options[:local_only], in_reply_to, account.user&.setting_default_federation))
end
process_hashtags_service.call(status)
process_mentions_service.call(status)
LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
DistributionWorker.perform_async(status.id)
unless status.local_only?
Pubsubhubbub::DistributionWorker.perform_async(status.stream_entry.id)
ActivityPub::DistributionWorker.perform_async(status.id)
ActivityPub::ReplyDistributionWorker.perform_async(status.id) if status.reply? && status.thread.account.local?
end
if options[:idempotency].present?
redis.setex("idempotency:status:#{account.id}:#{options[:idempotency]}", 3_600, status.id)
end
bump_potential_friendship(account, status)
status
@status
end
private
def preprocess_attributes!
@text = @options.delete(:spoiler_text) if @text.blank? && @options[:spoiler_text].present?
@visibility = @options[:visibility] || @account.user&.setting_default_privacy
@visibility = :unlisted if @visibility == :public && @account.silenced
@scheduled_at = @options[:scheduled_at]&.to_datetime
@scheduled_at = nil if scheduled_in_the_past?
end
def process_status!
# The following transaction block is needed to wrap the UPDATEs to
# the media attachments when the status is created
ApplicationRecord.transaction do
@status = @account.statuses.create!(status_attributes)
end
process_hashtags_service.call(@status)
process_mentions_service.call(@status)
end
def schedule_status!
if @account.statuses.build(status_attributes).valid?
# The following transaction block is needed to wrap the UPDATEs to
# the media attachments when the scheduled status is created
ApplicationRecord.transaction do
@status = @account.scheduled_statuses.create!(scheduled_status_attributes)
end
else
raise ActiveRecord::RecordInvalid
end
end
def local_only_option(local_only, in_reply_to, federation_setting)
return in_reply_to&.local_only? if local_only.nil? # XXX temporary, just until clients implement to avoid leaking local_only posts
return federation_setting if local_only.nil?
local_only
end
def validate_media!(media_ids)
return if media_ids.blank? || !media_ids.is_a?(Enumerable)
def postprocess_status!
LinkCrawlWorker.perform_async(@status.id) unless @status.spoiler_text?
DistributionWorker.perform_async(@status.id)
raise Mastodon::ValidationError, I18n.t('media_attachments.validations.too_many') if media_ids.size > 4
unless @status.local_only?
Pubsubhubbub::DistributionWorker.perform_async(@status.stream_entry.id)
ActivityPub::DistributionWorker.perform_async(@status.id)
end
end
media = MediaAttachment.where(status_id: nil).where(id: media_ids.take(4).map(&:to_i))
def validate_media!
return if @options[:media_ids].blank? || !@options[:media_ids].is_a?(Enumerable)
raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if media.size > 1 && media.find(&:video?)
raise Mastodon::ValidationError, I18n.t('media_attachments.validations.too_many') if @options[:media_ids].size > 4
media
@media = MediaAttachment.where(status_id: nil).where(id: @options[:media_ids].take(4).map(&:to_i))
raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if @media.size > 1 && @media.find(&:video?)
end
def language_from_option(str)
@ -92,10 +118,69 @@ class PostStatusService < BaseService
Redis.current
end
def bump_potential_friendship(account, status)
return if !status.reply? || account.id == status.in_reply_to_account_id
def scheduled?
@scheduled_at.present?
end
def idempotency_key
"idempotency:status:#{@account.id}:#{@options[:idempotency]}"
end
def idempotency_given?
@options[:idempotency].present?
end
def idempotency_duplicate
if scheduled?
@account.schedule_statuses.find(@idempotency_duplicate)
else
@account.statuses.find(@idempotency_duplicate)
end
end
def idempotency_duplicate?
@idempotency_duplicate = redis.get(idempotency_key)
end
def scheduled_in_the_past?
@scheduled_at.present? && @scheduled_at <= Time.now.utc + MIN_SCHEDULE_OFFSET
end
def bump_potential_friendship!
return if !@status.reply? || @account.id == @status.in_reply_to_account_id
ActivityTracker.increment('activity:interactions')
return if account.following?(status.in_reply_to_account_id)
PotentialFriendshipTracker.record(account.id, status.in_reply_to_account_id, :reply)
return if @account.following?(@status.in_reply_to_account_id)
PotentialFriendshipTracker.record(@account.id, @status.in_reply_to_account_id, :reply)
end
def status_attributes
{
text: @text,
media_attachments: @media || [],
thread: @in_reply_to,
sensitive: (@options[:sensitive].nil? ? @account.user&.setting_default_sensitive : @options[:sensitive]) || @options[:spoiler_text].present?,
spoiler_text: @options[:spoiler_text] || '',
visibility: @visibility,
language: language_from_option(@options[:language]) || @account.user&.setting_default_language&.presence || LanguageDetector.instance.detect(@text, @account),
application: @options[:application],
local_only: local_only_option(@options[:local_only], @in_reply_to, @account.user&.setting_default_federation),
}
end
def scheduled_status_attributes
{
scheduled_at: @scheduled_at,
media_attachments: @media || [],
params: scheduled_options,
}
end
def scheduled_options
@options.tap do |options_hash|
options_hash[:in_reply_to_id] = options_hash.delete(:thread)&.id
options_hash[:application_id] = options_hash.delete(:application)&.id
options_hash[:scheduled_at] = nil
options_hash[:idempotency] = nil
end
end
end

View File

@ -60,11 +60,13 @@ class ProcessMentionsService < BaseService
end
def activitypub_json
@activitypub_json ||= Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
return @activitypub_json if defined?(@activitypub_json)
payload = ActiveModelSerializers::SerializableResource.new(
@status,
serializer: ActivityPub::ActivitySerializer,
adapter: ActivityPub::Adapter
).as_json).sign!(@status.account))
).as_json
@activitypub_json = Oj.dump(@status.distributable? ? ActivityPub::LinkedDataSignature.new(payload).sign!(@status.account) : payload)
end
def resolve_account_service

View File

@ -19,31 +19,18 @@ class Pubsubhubbub::SubscribeService < BaseService
private
def process_subscribe
case subscribe_status
when :invalid_topic
if account.nil?
['Invalid topic URL', 422]
when :invalid_callback
elsif !valid_callback?
['Invalid callback URL', 422]
when :callback_not_allowed
elsif blocked_domain?
['Callback URL not allowed', 403]
when :valid
else
confirm_subscription
['', 202]
end
end
def subscribe_status
if account.nil?
:invalid_topic
elsif !valid_callback?
:invalid_callback
elsif blocked_domain?
:callback_not_allowed
else
:valid
end
end
def confirm_subscription
subscription = locate_subscription
Pubsubhubbub::ConfirmationWorker.perform_async(subscription.id, 'subscribe', secret, lease_seconds)
@ -58,12 +45,7 @@ class Pubsubhubbub::SubscribeService < BaseService
end
def locate_subscription
subscription = Subscription.find_by(account: account, callback_url: callback)
if subscription.nil?
subscription = Subscription.new(account: account, callback_url: callback)
end
subscription = Subscription.find_or_initialize_by(account: account, callback_url: callback)
subscription.domain = domain
subscription.save!
subscription

View File

@ -19,11 +19,11 @@ class RejectFollowService < BaseService
end
def build_json(follow_request)
Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
ActiveModelSerializers::SerializableResource.new(
follow_request,
serializer: ActivityPub::RejectFollowSerializer,
adapter: ActivityPub::Adapter
).as_json).sign!(follow_request.target_account))
).to_json
end
def build_xml(follow_request)

View File

@ -52,6 +52,6 @@ class ReportService < BaseService
end
def some_local_account
@some_local_account ||= Account.local.where(suspended: false).first
@some_local_account ||= Account.representative
end
end

View File

@ -19,6 +19,7 @@ class ResolveAccountService < BaseService
@account = uri
@username = @account.username
@domain = @account.domain
uri = "#{@username}@#{@domain}"
return @account if @account.local? || !webfinger_update_due?
else

View File

@ -34,6 +34,8 @@ class SearchService < BaseService
.compact
statuses.reject { |status| StatusFilter.new(status, account).filtered? }
rescue Faraday::ConnectionFailed
[]
end
def perform_hashtags_search!

View File

@ -1,6 +1,42 @@
# frozen_string_literal: true
class SuspendAccountService < BaseService
ASSOCIATIONS_ON_SUSPEND = %w(
account_pins
active_relationships
block_relationships
blocked_by_relationships
conversation_mutes
conversations
custom_filters
domain_blocks
favourites
follow_requests
list_accounts
media_attachments
mute_relationships
muted_by_relationships
notifications
owned_lists
passive_relationships
report_notes
scheduled_statuses
status_pins
stream_entries
subscriptions
).freeze
ASSOCIATIONS_ON_DESTROY = %w(
reports
targeted_moderation_notes
targeted_reports
).freeze
# Suspend an account and remove as much of its data as possible
# @param [Account]
# @param [Hash] options
# @option [Boolean] :including_user Remove the user record as well
# @option [Boolean] :destroy Remove the account record instead of suspending
def call(account, **options)
@account = account
@options = options
@ -8,60 +44,66 @@ class SuspendAccountService < BaseService
purge_user!
purge_profile!
purge_content!
unsubscribe_push_subscribers!
end
private
def purge_user!
if @options[:remove_user]
@account.user&.destroy
return if !@account.local? || @account.user.nil?
if @options[:including_user]
@account.user.destroy
else
@account.user&.disable!
@account.user.disable!
end
end
def purge_content!
if @account.local?
ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes) do |inbox_url|
[delete_actor_json, @account.id, inbox_url]
end
end
distribute_delete_actor! if @account.local?
@account.statuses.reorder(nil).find_in_batches do |statuses|
BatchedRemoveStatusService.new.call(statuses)
BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:destroy])
end
[
@account.media_attachments,
@account.stream_entries,
@account.notifications,
@account.favourites,
@account.active_relationships,
@account.passive_relationships,
].each do |association|
destroy_all(association)
associations_for_destruction.each do |association_name|
destroy_all(@account.public_send(association_name))
end
@account.destroy if @options[:destroy]
end
def purge_profile!
@account.suspended = true
@account.display_name = ''
@account.note = ''
@account.statuses_count = 0
# If the account is going to be destroyed
# there is no point wasting time updating
# its values first
return if @options[:destroy]
@account.silenced = false
@account.suspended = true
@account.locked = false
@account.display_name = ''
@account.note = ''
@account.fields = {}
@account.statuses_count = 0
@account.followers_count = 0
@account.following_count = 0
@account.moved_to_account = nil
@account.avatar.destroy
@account.header.destroy
@account.save!
end
def unsubscribe_push_subscribers!
destroy_all(@account.subscriptions)
end
def destroy_all(association)
association.in_batches.destroy_all
end
def distribute_delete_actor!
ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes) do |inbox_url|
[delete_actor_json, @account.id, inbox_url]
end
end
def delete_actor_json
return @delete_actor_json if defined?(@delete_actor_json)
@ -77,4 +119,12 @@ class SuspendAccountService < BaseService
def delivery_inboxes
Account.inboxes + Relay.enabled.pluck(:inbox_url)
end
def associations_for_destruction
if @options[:destroy]
ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY
else
ASSOCIATIONS_ON_SUSPEND
end
end
end

View File

@ -20,11 +20,11 @@ class UnblockService < BaseService
end
def build_json(unblock)
Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
ActiveModelSerializers::SerializableResource.new(
unblock,
serializer: ActivityPub::UndoBlockSerializer,
adapter: ActivityPub::Adapter
).as_json).sign!(unblock.account))
).to_json
end
def build_xml(block)

View File

@ -43,11 +43,11 @@ class UnfollowService < BaseService
end
def build_json(follow)
Oj.dump(ActivityPub::LinkedDataSignature.new(ActiveModelSerializers::SerializableResource.new(
ActiveModelSerializers::SerializableResource.new(
follow,
serializer: ActivityPub::UndoFollowSerializer,
adapter: ActivityPub::Adapter
).as_json).sign!(follow.account))
).to_json
end
def build_xml(follow)

View File

@ -10,7 +10,11 @@ class UpdateAccountService < BaseService
authorize_all_follow_requests(account) if was_locked && !account.locked
check_links(account)
process_hashtags(account)
end
rescue Mastodon::DimensionsValidationError => de
account.errors.add(:avatar, de.message)
false
end
private
@ -24,4 +28,8 @@ class UpdateAccountService < BaseService
def check_links(account)
VerifyAccountLinksWorker.perform_async(account.id)
end
def process_hashtags(account)
account.tags_as_strings = Extractor.extract_hashtags(account.note)
end
end

View File

@ -10,7 +10,6 @@ class VerifyLinkService < BaseService
return unless link_back_present?
field.mark_verified!
field.account.save!
rescue HTTP::Error, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e
Rails.logger.debug "Error fetching link #{@url}: #{e}"
nil