Merge tag 'v2.8.0' into instance_only_statuses

This commit is contained in:
Renato "Lond" Cerqueira
2019-04-13 23:47:24 +02:00
689 changed files with 25483 additions and 9047 deletions

View File

@ -4,7 +4,7 @@ class ActivityPub::Activity
include JsonLdHelper
include Redisable
SUPPORTED_TYPES = %w(Note).freeze
SUPPORTED_TYPES = %w(Note Question).freeze
CONVERTED_TYPES = %w(Image Video Article Page).freeze
def initialize(json, account, **options)

View File

@ -47,6 +47,10 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
followed_by_local_accounts? || requested_through_relay? || reblog_of_local_status?
end
def requested_through_relay?
super || Relay.find_by(inbox_url: @account.inbox_url)&.enabled?
end
def reblog_of_local_status?
status_from_uri(object_uri)&.account&.local?
end

View File

@ -6,7 +6,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
RedisLock.acquire(lock_options) do |lock|
if lock.acquired?
return if delete_arrived_first?(object_uri)
return if delete_arrived_first?(object_uri) || poll_vote?
@status = find_existing_status
@ -40,6 +40,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
end
resolve_thread(@status)
fetch_replies(@status)
distribute(@status)
forward_for_reply if @status.public_visibility? || @status.unlisted_visibility?
end
@ -67,6 +68,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
thread: replied_to_status,
conversation: conversation_from_uri(@object['conversation']),
media_attachment_ids: process_attachments.take(4).map(&:id),
poll: process_poll,
}
end
end
@ -159,7 +161,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
return if tag['href'].blank?
account = account_from_uri(tag['href'])
account = ::FetchRemoteAccountService.new.call(tag['href'], id: false) if account.nil?
account = ::FetchRemoteAccountService.new.call(tag['href']) if account.nil?
return if account.nil?
@ -208,11 +210,60 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
media_attachments
end
def process_poll
return unless @object['type'] == 'Question' && (@object['anyOf'].is_a?(Array) || @object['oneOf'].is_a?(Array))
expires_at = begin
if @object['closed'].is_a?(String)
@object['closed']
elsif !@object['closed'].nil? && !@object['closed'].is_a?(FalseClass)
Time.now.utc
else
@object['endTime']
end
end
if @object['anyOf'].is_a?(Array)
multiple = true
items = @object['anyOf']
else
multiple = false
items = @object['oneOf']
end
@account.polls.new(
multiple: multiple,
expires_at: expires_at,
options: items.map { |item| item['name'].presence || item['content'] },
cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 }
)
end
def poll_vote?
return false if replied_to_status.nil? || replied_to_status.preloadable_poll.nil? || !replied_to_status.local? || !replied_to_status.preloadable_poll.options.include?(@object['name'])
unless replied_to_status.preloadable_poll.expired?
replied_to_status.preloadable_poll.votes.create!(account: @account, choice: replied_to_status.preloadable_poll.options.index(@object['name']), uri: @object['id'])
ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, replied_to_status.id) unless replied_to_status.preloadable_poll.hide_totals?
end
true
end
def resolve_thread(status)
return unless status.reply? && status.thread.nil? && Request.valid_url?(in_reply_to_uri)
ThreadResolveWorker.perform_async(status.id, in_reply_to_uri)
end
def fetch_replies(status)
collection = @object['replies']
return if collection.nil?
replies = ActivityPub::FetchRepliesService.new.call(status, collection, false)
return unless replies.nil?
uri = value_or_id(collection)
ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil?
end
def conversation_from_uri(uri)
return nil if uri.nil?
return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri)
@ -323,15 +374,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
@skip_download ||= DomainBlock.find_by(domain: @account.domain)&.reject_media?
end
def invalid_origin?(url)
return true if unsupported_uri_scheme?(url)
needle = Addressable::URI.parse(url).host
haystack = Addressable::URI.parse(@account.uri).host
!haystack.casecmp(needle).zero?
end
def reply_to_local?
!replied_to_status.nil? && replied_to_status.account.local?
end

View File

@ -75,13 +75,4 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
def lock_options
{ redis: Redis.current, key: "create:#{object_uri}" }
end
def invalid_origin?(url)
return true if unsupported_uri_scheme?(url)
needle = Addressable::URI.parse(url).host
haystack = Addressable::URI.parse(@account.uri).host
!haystack.casecmp(needle).zero?
end
end

View File

@ -14,7 +14,8 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
@account,
target_account,
status_ids: target_statuses.nil? ? [] : target_statuses.map(&:id),
comment: @json['content'] || ''
comment: @json['content'] || '',
uri: report_uri
)
end
end
@ -28,4 +29,8 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity
def object_uris
@object_uris ||= Array(@object.is_a?(Array) ? @object.map { |item| value_or_id(item) } : value_or_id(@object))
end
def report_uri
@json['id'] unless @json['id'].nil? || invalid_origin?(@json['id'])
end
end

View File

@ -4,7 +4,11 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
def perform
update_account if equals_or_includes_any?(@object['type'], SUPPORTED_TYPES)
if equals_or_includes_any?(@object['type'], SUPPORTED_TYPES)
update_account
elsif equals_or_includes_any?(@object['type'], %w(Question))
update_poll
end
end
private
@ -14,4 +18,13 @@ class ActivityPub::Activity::Update < ActivityPub::Activity
ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true)
end
def update_poll
return reject_payload! if invalid_origin?(@object['id'])
status = Status.find_by(uri: object_uri, account_id: @account.id)
return if status.nil? || status.preloadable_poll.nil?
ActivityPub::ProcessPollService.new.call(status.preloadable_poll, @object)
end
end

View File

@ -1,30 +1,24 @@
# frozen_string_literal: true
class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
CONTEXT = {
'@context': [
'https://www.w3.org/ns/activitystreams',
'https://w3id.org/security/v1',
NAMED_CONTEXT_MAP = {
activitystreams: 'https://www.w3.org/ns/activitystreams',
security: 'https://w3id.org/security/v1',
}.freeze
{
'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers',
'sensitive' => 'as:sensitive',
'movedTo' => { '@id' => 'as:movedTo', '@type' => '@id' },
'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' },
'Hashtag' => 'as:Hashtag',
'ostatus' => 'http://ostatus.org#',
'atomUri' => 'ostatus:atomUri',
'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri',
'conversation' => 'ostatus:conversation',
'toot' => 'http://joinmastodon.org/ns#',
'Emoji' => 'toot:Emoji',
'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' },
'featured' => { '@id' => 'toot:featured', '@type' => '@id' },
'schema' => 'http://schema.org#',
'PropertyValue' => 'schema:PropertyValue',
'value' => 'schema:value',
},
],
CONTEXT_EXTENSION_MAP = {
manually_approves_followers: { 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers' },
sensitive: { 'sensitive' => 'as:sensitive' },
hashtag: { 'Hashtag' => 'as:Hashtag' },
moved_to: { 'movedTo' => { '@id' => 'as:movedTo', '@type' => '@id' } },
also_known_as: { 'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' } },
emoji: { 'toot' => 'http://joinmastodon.org/ns#', 'Emoji' => 'toot:Emoji' },
featured: { 'toot' => 'http://joinmastodon.org/ns#', 'featured' => { '@id' => 'toot:featured', '@type' => '@id' } },
property_value: { 'schema' => 'http://schema.org#', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' },
atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' },
conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' },
focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } },
identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
}.freeze
def self.default_key_transform
@ -36,8 +30,36 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
end
def serializable_hash(options = nil)
options = serialization_options(options)
serialized_hash = ActiveModelSerializers::Adapter::Attributes.new(serializer, instance_options).serializable_hash(options)
CONTEXT.merge(self.class.transform_key_casing!(serialized_hash, instance_options))
options = serialization_options(options)
serialized_hash = serializer.serializable_hash(options)
serialized_hash = self.class.transform_key_casing!(serialized_hash, instance_options)
{ '@context' => serialized_context }.merge(serialized_hash)
end
private
def serialized_context
context_array = []
serializer_options = serializer.send(:instance_options) || {}
named_contexts = [:activitystreams] + serializer._named_contexts.keys + serializer_options.fetch(:named_contexts, {}).keys
context_extensions = serializer._context_extensions.keys + serializer_options.fetch(:context_extensions, {}).keys
named_contexts.each do |key|
context_array << NAMED_CONTEXT_MAP[key]
end
extensions = context_extensions.each_with_object({}) do |key, h|
h.merge!(CONTEXT_EXTENSION_MAP[key])
end
context_array << extensions unless extensions.empty?
if context_array.size == 1
context_array.first
else
context_array
end
end
end

View File

@ -0,0 +1,30 @@
# frozen_string_literal: true
class ActivityPub::Serializer < ActiveModel::Serializer
with_options instance_writer: false, instance_reader: true do |serializer|
serializer.class_attribute :_named_contexts
serializer.class_attribute :_context_extensions
self._named_contexts ||= {}
self._context_extensions ||= {}
end
def self.inherited(base)
super
base._named_contexts = _named_contexts.dup
base._context_extensions = _context_extensions.dup
end
def self.context(*named_contexts)
named_contexts.each do |context|
_named_contexts[context] = true
end
end
def self.context_extensions(*extension_names)
extension_names.each do |extension_name|
_context_extensions[extension_name] = true
end
end
end

View File

@ -48,6 +48,12 @@ class ActivityPub::TagManager
activity_account_status_url(target.account, target)
end
def replies_uri_for(target, page_params = nil)
raise ArgumentError, 'target must be a local activity' unless %i(note comment activity).include?(target.object_type) && target.local?
replies_account_status_url(target.account, target, page_params)
end
# Primary audience of a status
# Public statuses go out to primarily the public collection
# Unlisted and private statuses go out primarily to the followers collection

View File

@ -19,6 +19,10 @@ class Formatter
raw_content = status.text
if options[:inline_poll_options] && status.preloadable_poll
raw_content = raw_content + "\n\n" + status.preloadable_poll.options.map { |title| "[ ] #{title}" }.join("\n")
end
return '' if raw_content.blank?
unless status.local?
@ -67,6 +71,12 @@ class Formatter
html.html_safe # rubocop:disable Rails/OutputSafety
end
def format_poll_option(status, option, **options)
html = encode(option.title)
html = encode_custom_emojis(html, status.emojis, options[:autoplay])
html.html_safe # rubocop:disable Rails/OutputSafety
end
def format_display_name(account, **options)
html = encode(account.display_name.presence || account.username)
html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify]

View File

@ -3,7 +3,8 @@
class LanguageDetector
include Singleton
CHARACTER_THRESHOLD = 140
WORDS_THRESHOLD = 4
RELIABLE_CHARACTERS_RE = /[\p{Hebrew}\p{Arabic}\p{Syriac}\p{Thaana}\p{Nko}\p{Han}\p{Katakana}\p{Hiragana}\p{Hangul}]+/m
def initialize
@identifier = CLD3::NNetLanguageIdentifier.new(1, 2048)
@ -11,15 +12,14 @@ class LanguageDetector
def detect(text, account)
input_text = prepare_text(text)
return if input_text.blank?
detect_language_code(input_text) || default_locale(account)
end
def language_names
@language_names =
CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym }
.uniq
@language_names = CLD3::TaskContextParams::LANGUAGE_NAMES.map { |name| iso6391(name.to_s).to_sym }.uniq
end
private
@ -29,12 +29,29 @@ class LanguageDetector
end
def unreliable_input?(text)
text.size < CHARACTER_THRESHOLD
!reliable_input?(text)
end
def reliable_input?(text)
sufficient_text_length?(text) || language_specific_character_set?(text)
end
def sufficient_text_length?(text)
text.split(/\s+/).size >= WORDS_THRESHOLD
end
def language_specific_character_set?(text)
words = text.scan(RELIABLE_CHARACTERS_RE)
if words.present?
words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size.to_f > 0.3
else
false
end
end
def detect_language_code(text)
return if unreliable_input?(text)
result = @identifier.find_language(text)
iso6391(result.language.to_s).to_sym if result.reliable?
end
@ -77,6 +94,6 @@ class LanguageDetector
end
def default_locale(account)
return account.user_locale&.to_sym || I18n.default_locale if account.local?
account.user_locale&.to_sym || I18n.default_locale if account.local?
end
end

View File

@ -352,7 +352,7 @@ class OStatus::AtomSerializer
append_element(entry, 'link', nil, rel: :alternate, type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(status)) if status.account.local?
append_element(entry, 'summary', status.spoiler_text, 'xml:lang': status.language) if status.spoiler_text?
append_element(entry, 'content', Formatter.instance.format(status).to_str || '.', type: 'html', 'xml:lang': status.language)
append_element(entry, 'content', Formatter.instance.format(status, inline_poll_options: true).to_str || '.', type: 'html', 'xml:lang': status.language)
status.active_mentions.sort_by(&:id).each do |mentioned|
append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:person], href: OStatus::TagManager.instance.uri_for(mentioned.account))

12
app/lib/proof_provider.rb Normal file
View File

@ -0,0 +1,12 @@
# frozen_string_literal: true
module ProofProvider
SUPPORTED_PROVIDERS = %w(keybase).freeze
def self.find(identifier, proof = nil)
case identifier
when 'keybase'
ProofProvider::Keybase.new(proof)
end
end
end

View File

@ -0,0 +1,69 @@
# frozen_string_literal: true
class ProofProvider::Keybase
BASE_URL = ENV.fetch('KEYBASE_BASE_URL', 'https://keybase.io')
DOMAIN = ENV.fetch('KEYBASE_DOMAIN', Rails.configuration.x.local_domain)
class Error < StandardError; end
class ExpectedProofLiveError < Error; end
class UnexpectedResponseError < Error; end
def initialize(proof = nil)
@proof = proof
end
def serializer_class
ProofProvider::Keybase::Serializer
end
def worker_class
ProofProvider::Keybase::Worker
end
def validate!
unless @proof.token&.size == 66
@proof.errors.add(:base, I18n.t('identity_proofs.errors.keybase.invalid_token'))
return
end
# Do not perform synchronous validation for remote accounts
return if @proof.provider_username.blank? || !@proof.account.local?
if verifier.valid?
@proof.verified = true
@proof.live = false
else
@proof.errors.add(:base, I18n.t('identity_proofs.errors.keybase.verification_failed', kb_username: @proof.provider_username))
end
end
def refresh!
worker_class.new.perform(@proof)
rescue ProofProvider::Keybase::Error
nil
end
def on_success_path(user_agent = nil)
verifier.on_success_path(user_agent)
end
def badge
@badge ||= ProofProvider::Keybase::Badge.new(@proof.account.username, @proof.provider_username, @proof.token, domain)
end
def verifier
@verifier ||= ProofProvider::Keybase::Verifier.new(@proof.account.username, @proof.provider_username, @proof.token, domain)
end
private
def domain
if @proof.account.local?
DOMAIN
else
@proof.account.domain
end
end
end

View File

@ -0,0 +1,45 @@
# frozen_string_literal: true
class ProofProvider::Keybase::Badge
include RoutingHelper
def initialize(local_username, provider_username, token, domain)
@local_username = local_username
@provider_username = provider_username
@token = token
@domain = domain
end
def proof_url
"#{ProofProvider::Keybase::BASE_URL}/#{@provider_username}/sigchain\##{@token}"
end
def profile_url
"#{ProofProvider::Keybase::BASE_URL}/#{@provider_username}"
end
def icon_url
"#{ProofProvider::Keybase::BASE_URL}/#{@provider_username}/proof_badge/#{@token}?username=#{@local_username}&domain=#{@domain}"
end
def avatar_url
Rails.cache.fetch("proof_providers/keybase/#{@provider_username}/avatar_url", expires_in: 5.minutes) { remote_avatar_url } || default_avatar_url
end
private
def remote_avatar_url
request = Request.new(:get, "#{ProofProvider::Keybase::BASE_URL}/_/api/1.0/user/pic_url.json", params: { username: @provider_username })
request.perform do |res|
json = Oj.load(res.body_with_limit, mode: :strict)
json['pic_url'] if json.is_a?(Hash)
end
rescue Oj::ParseError, HTTP::Error, OpenSSL::SSL::SSLError
nil
end
def default_avatar_url
asset_pack_path('media/images/proof_providers/keybase.png')
end
end

View File

@ -0,0 +1,71 @@
# frozen_string_literal: true
class ProofProvider::Keybase::ConfigSerializer < ActiveModel::Serializer
include RoutingHelper
include ActionView::Helpers::TextHelper
attributes :version, :domain, :display_name, :username,
:brand_color, :logo, :description, :prefill_url,
:profile_url, :check_url, :check_path, :avatar_path,
:contact
def version
1
end
def domain
ProofProvider::Keybase::DOMAIN
end
def display_name
Setting.site_title
end
def logo
{ svg_black: full_asset_url(asset_pack_path('media/images/logo_transparent_black.svg')), svg_full: full_asset_url(asset_pack_path('media/images/logo.svg')) }
end
def brand_color
'#282c37'
end
def description
strip_tags(Setting.site_short_description.presence || I18n.t('about.about_mastodon_html'))
end
def username
{ min: 1, max: 30, re: '[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?' }
end
def prefill_url
params = {
provider: 'keybase',
token: '%{sig_hash}',
provider_username: '%{kb_username}',
username: '%{username}',
user_agent: '%{kb_ua}',
}
CGI.unescape(new_settings_identity_proof_url(params))
end
def profile_url
CGI.unescape(short_account_url('%{username}')) # rubocop:disable Style/FormatStringToken
end
def check_url
CGI.unescape(api_proofs_url(username: '%{username}', provider: 'keybase'))
end
def check_path
['signatures']
end
def avatar_path
['avatar']
end
def contact
[Setting.site_contact_email.presence || 'unknown'].compact
end
end

View File

@ -0,0 +1,25 @@
# frozen_string_literal: true
class ProofProvider::Keybase::Serializer < ActiveModel::Serializer
include RoutingHelper
attribute :avatar
has_many :identity_proofs, key: :signatures
def avatar
full_asset_url(object.avatar_original_url)
end
class AccountIdentityProofSerializer < ActiveModel::Serializer
attributes :sig_hash, :kb_username
def sig_hash
object.token
end
def kb_username
object.provider_username
end
end
end

View File

@ -0,0 +1,59 @@
# frozen_string_literal: true
class ProofProvider::Keybase::Verifier
def initialize(local_username, provider_username, token, domain)
@local_username = local_username
@provider_username = provider_username
@token = token
@domain = domain
end
def valid?
request = Request.new(:get, "#{ProofProvider::Keybase::BASE_URL}/_/api/1.0/sig/proof_valid.json", params: query_params)
request.perform do |res|
json = Oj.load(res.body_with_limit, mode: :strict)
if json.is_a?(Hash)
json.fetch('proof_valid', false)
else
false
end
end
rescue Oj::ParseError, HTTP::Error, OpenSSL::SSL::SSLError
false
end
def on_success_path(user_agent = nil)
url = Addressable::URI.parse("#{ProofProvider::Keybase::BASE_URL}/_/proof_creation_success")
url.query_values = query_params.merge(kb_ua: user_agent || 'unknown')
url.to_s
end
def status
request = Request.new(:get, "#{ProofProvider::Keybase::BASE_URL}/_/api/1.0/sig/proof_live.json", params: query_params)
request.perform do |res|
raise ProofProvider::Keybase::UnexpectedResponseError unless res.code == 200
json = Oj.load(res.body_with_limit, mode: :strict)
raise ProofProvider::Keybase::UnexpectedResponseError unless json.is_a?(Hash) && json.key?('proof_valid') && json.key?('proof_live')
json
end
rescue Oj::ParseError, HTTP::Error, OpenSSL::SSL::SSLError
raise ProofProvider::Keybase::UnexpectedResponseError
end
private
def query_params
{
domain: @domain,
kb_username: @provider_username,
username: @local_username,
sig_hash: @token,
}
end
end

View File

@ -0,0 +1,32 @@
# frozen_string_literal: true
class ProofProvider::Keybase::Worker
include Sidekiq::Worker
sidekiq_options queue: 'pull', retry: 20, unique: :until_executed
sidekiq_retry_in do |count, exception|
# Retry aggressively when the proof is valid but not live in Keybase.
# This is likely because Keybase just hasn't noticed the proof being
# served from here yet.
if exception.class == ProofProvider::Keybase::ExpectedProofLiveError
case count
when 0..2 then 0.seconds
when 2..6 then 1.second
end
end
end
def perform(proof_id)
proof = proof_id.is_a?(AccountIdentityProof) ? proof_id : AccountIdentityProof.find(proof_id)
status = proof.provider_instance.verifier.status
# If Keybase thinks the proof is valid, and it exists here in Mastodon,
# then it should be live. Keybase just has to notice that it's here
# and then update its state. That might take a couple seconds.
raise ProofProvider::Keybase::ExpectedProofLiveError if status['proof_valid'] && !status['proof_live']
proof.update!(verified: status['proof_valid'], live: status['proof_live'])
end
end

View File

@ -171,7 +171,7 @@ class Request
outer_e = nil
Resolv::DNS.open do |dns|
dns.timeouts = 1
dns.timeouts = 5
addresses = dns.getaddresses(host).take(2)
time_slot = 10.0 / addresses.size

View File

@ -33,6 +33,7 @@ class UserSettingsDecorator
user.settings['theme'] = theme_preference if change?('setting_theme')
user.settings['hide_network'] = hide_network_preference if change?('setting_hide_network')
user.settings['aggregate_reblogs'] = aggregate_reblogs_preference if change?('setting_aggregate_reblogs')
user.settings['show_application'] = show_application_preference if change?('setting_show_application')
end
def merged_notification_emails
@ -95,6 +96,10 @@ class UserSettingsDecorator
boolean_cast_setting 'setting_hide_network'
end
def show_application_preference
boolean_cast_setting 'setting_show_application'
end
def theme_preference
settings['setting_theme']
end