autonomic-cooperative/hometown
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge tag 'v3.0.0' into hometown-dev

Browse Source
This commit is contained in:
Darius Kazemi
2019-10-08 13:24:20 -07:00
parent bfd29a34eb 83d3e7733d
commit 228e0f0f6e
1012 changed files with 31176 additions and 15165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/lib/activity_tracker.rb
View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
class ActivityTracker
EXPIRE_AFTER = 90.days.seconds
EXPIRE_AFTER = 6.months.seconds
class << self
include Redisable

2
app/lib/activitypub/activity.rb
View File

@ -5,7 +5,7 @@ class ActivityPub::Activity
include Redisable
SUPPORTED_TYPES = %w(Note Question Article).freeze
CONVERTED_TYPES = %w(Image Video Page).freeze
CONVERTED_TYPES = %w(Image Audio Video Page).freeze
def initialize(json, account, **options)
@json = json

2
app/lib/activitypub/activity/announce.rb
View File

@ -40,7 +40,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
end
def announceable?(status)
status.account_id == @account.id || status.public_visibility? || status.unlisted_visibility?
status.account_id == @account.id || status.distributable?
end
def related_to_local_activity?

77
app/lib/activitypub/activity/create.rb
View File

@ -42,8 +42,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
resolve_thread(@status)
fetch_replies(@status)
check_for_spam
distribute(@status)
forward_for_reply if @status.public_visibility? || @status.unlisted_visibility?
forward_for_reply if @status.distributable?
end
def find_existing_status
@ -199,12 +200,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
def process_hashtag(tag)
return if tag['name'].blank?
hashtag = tag['name'].gsub(/\A#/, '').mb_chars.downcase
hashtag = Tag.where(name: hashtag).first_or_create!(name: hashtag)
return if @tags.include?(hashtag)
@tags << hashtag
Tag.find_or_create_by_names(tag['name']) do |hashtag|
@tags << hashtag unless @tags.include?(hashtag)
end
rescue ActiveRecord::RecordInvalid
nil
end
@ -243,22 +241,25 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
media_attachments = []
as_array(@object['attachment']).each do |attachment|
next if attachment['url'].blank?
next if attachment['url'].blank? || media_attachments.size >= 4
href = Addressable::URI.parse(attachment['url']).normalize.to_s
media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil)
media_attachments << media_attachment
begin
href = Addressable::URI.parse(attachment['url']).normalize.to_s
media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil)
media_attachments << media_attachment
next if unsupported_media_type?(attachment['mediaType']) || skip_download?
next if unsupported_media_type?(attachment['mediaType']) || skip_download?
media_attachment.file_remote_url = href
media_attachment.save
media_attachment.file_remote_url = href
media_attachment.save
rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError
RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id)
end
end
media_attachments
rescue Addressable::URI::InvalidURIError => e
Rails.logger.debug e
Rails.logger.debug "Invalid URL in attachment: #{e}"
media_attachments
end
@ -283,25 +284,40 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
items = @object['oneOf']
end
voters_count = @object['votersCount']
@account.polls.new(
multiple: multiple,
expires_at: expires_at,
options: items.map { |item| item['name'].presence || item['content'] }.compact,
cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 }
cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 },
voters_count: voters_count
)
end
def poll_vote?
return false if replied_to_status.nil? || replied_to_status.preloadable_poll.nil? || !replied_to_status.local? || !replied_to_status.preloadable_poll.options.include?(@object['name'])
unless replied_to_status.preloadable_poll.expired?
replied_to_status.preloadable_poll.votes.create!(account: @account, choice: replied_to_status.preloadable_poll.options.index(@object['name']), uri: @object['id'])
ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, replied_to_status.id) unless replied_to_status.preloadable_poll.hide_totals?
end
poll_vote! unless replied_to_status.preloadable_poll.expired?
true
end
def poll_vote!
poll = replied_to_status.preloadable_poll
already_voted = true
RedisLock.acquire(poll_lock_options) do |lock|
if lock.acquired?
already_voted = poll.votes.where(account: @account).exists?
poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: @object['id'])
else
raise Mastodon::RaceConditionError
end
end
increment_voters_count! unless already_voted
ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, replied_to_status.id) unless replied_to_status.preloadable_poll.hide_totals?
end
def resolve_thread(status)
return unless status.reply? && status.thread.nil? && Request.valid_url?(in_reply_to_uri)
ThreadResolveWorker.perform_async(status.id, in_reply_to_uri)
@ -460,12 +476,31 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
Account.local.where(username: local_usernames).exists?
end
def check_for_spam
SpamCheck.perform(@status)
end
def forward_for_reply
return unless @json['signature'].present? && reply_to_local?
ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url])
end
def increment_voters_count!
poll = replied_to_status.preloadable_poll
unless poll.voters_count.nil?
poll.voters_count = poll.voters_count + 1
poll.save
end
rescue ActiveRecord::StaleObjectError
poll.reload
retry
end
def lock_options
{ redis: Redis.current, key: "create:#{@object['id']}" }
end
def poll_lock_options
{ redis: Redis.current, key: "vote:#{replied_to_status.poll_id}:#{@account.id}" }
end
end

7
app/lib/activitypub/activity/delete.rb
View File

@ -13,8 +13,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
def delete_person
lock_or_return("delete_in_progress:#{@account.id}") do
SuspendAccountService.new.call(@account)
@account.destroy!
SuspendAccountService.new.call(@account, reserve_username: false)
end
end
@ -31,7 +30,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
return if @status.nil?
if @status.public_visibility? || @status.unlisted_visibility?
if @status.distributable?
forward_for_reply
forward_for_reblogs
end
@ -70,7 +69,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity
end
def delete_now!
RemoveStatusService.new.call(@status)
RemoveStatusService.new.call(@status, redraft: false)
end
def payload

4
app/lib/activitypub/activity/follow.rb
View File

@ -8,7 +8,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) || @account.requested?(target_account)
if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved?
if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? || target_account.instance_actor?
reject_follow_request!(target_account)
return
end
@ -21,7 +21,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
follow_request = FollowRequest.create!(account: @account, target_account: target_account, uri: @json['id'])
if target_account.locked?
if target_account.locked? || @account.silenced?
NotifyService.new.call(target_account, follow_request)
else
AuthorizeFollowService.new.call(@account, target_account)

17
app/lib/activitypub/activity/move.rb
View File

@ -10,17 +10,16 @@ class ActivityPub::Activity::Move < ActivityPub::Activity
target_account = ActivityPub::FetchRemoteAccountService.new.call(target_uri)
return if target_account.nil? || !target_account.also_known_as.include?(origin_account.uri)
if target_account.nil? || target_account.suspended? || !target_account.also_known_as.include?(origin_account.uri)
unmark_as_processing!
return
end
# In case for some reason we didn't have a redirect for the profile already, set it
origin_account.update(moved_to_account: target_account) if origin_account.moved_to_account_id.nil?
origin_account.update(moved_to_account: target_account)
# Initiate a re-follow for each follower
origin_account.followers.local.select(:id).find_in_batches do |follower_accounts|
UnfollowFollowWorker.push_bulk(follower_accounts.map(&:id)) do |follower_account_id|
[follower_account_id, origin_account.id, target_account.id]
end
end
MoveWorker.perform_async(origin_account.id, target_account.id)
end
private
@ -40,4 +39,8 @@ class ActivityPub::Activity::Move < ActivityPub::Activity
def mark_as_processing!
redis.setex("move_in_progress:#{@account.id}", PROCESSING_COOLDOWN, true)
end
def unmark_as_processing!
redis.del("move_in_progress:#{@account.id}")
end
end

16
app/lib/activitypub/adapter.rb
View File

@ -20,6 +20,8 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } },
identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
}.freeze
def self.default_key_transform
@ -31,21 +33,23 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
end
def serializable_hash(options = nil)
named_contexts = {}
context_extensions = {}
options = serialization_options(options)
serialized_hash = serializer.serializable_hash(options)
serialized_hash = serializer.serializable_hash(options.merge(named_contexts: named_contexts, context_extensions: context_extensions))
serialized_hash = serialized_hash.select { |k, _| options[:fields].include?(k) } if options[:fields]
serialized_hash = self.class.transform_key_casing!(serialized_hash, instance_options)
{ '@context' => serialized_context }.merge(serialized_hash)
{ '@context' => serialized_context(named_contexts, context_extensions) }.merge(serialized_hash)
end
private
def serialized_context
def serialized_context(named_contexts_map, context_extensions_map)
context_array = []
serializer_options = serializer.send(:instance_options) || {}
named_contexts = [:activitystreams] + serializer._named_contexts.keys + serializer_options.fetch(:named_contexts, {}).keys
context_extensions = serializer._context_extensions.keys + serializer_options.fetch(:context_extensions, {}).keys
named_contexts = [:activitystreams] + named_contexts_map.keys
context_extensions = context_extensions_map.keys
named_contexts.each do |key|
context_array << NAMED_CONTEXT_MAP[key]

8
app/lib/activitypub/serializer.rb
View File

@ -27,4 +27,12 @@ class ActivityPub::Serializer < ActiveModel::Serializer
_context_extensions[extension_name] = true
end
end
def serializable_hash(adapter_options = nil, options = {}, adapter_instance = self.class.serialization_adapter_instance)
unless adapter_options&.fetch(:named_contexts, nil).nil?
adapter_options[:named_contexts].merge!(_named_contexts)
adapter_options[:context_extensions].merge!(_context_extensions)
end
super(adapter_options, options, adapter_instance)
end
end

7
app/lib/activitypub/tag_manager.rb
View File

@ -17,7 +17,7 @@ class ActivityPub::TagManager
case target.object_type
when :person
short_account_url(target)
target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target)
when :note, :comment, :activity
return activity_account_status_url(target.account, target) if target.reblog?
short_account_status_url(target.account, target)
@ -29,7 +29,7 @@ class ActivityPub::TagManager
case target.object_type
when :person
account_url(target)
target.instance_actor? ? instance_actor_url : account_url(target)
when :note, :comment, :activity
return activity_account_status_url(target.account, target) if target.reblog?
account_status_url(target.account, target)
@ -51,7 +51,7 @@ class ActivityPub::TagManager
def replies_uri_for(target, page_params = nil)
raise ArgumentError, 'target must be a local activity' unless %i(note comment activity).include?(target.object_type) && target.local?
replies_account_status_url(target.account, target, page_params)
account_status_replies_url(target.account, target, page_params)
end
# Primary audience of a status
@ -119,6 +119,7 @@ class ActivityPub::TagManager
def uri_to_local_id(uri, param = :id)
path_params = Rails.application.routes.recognize_path(uri)
path_params[:username] = Rails.configuration.x.local_domain if path_params[:controller] == 'instance_actors'
path_params[param]
end

63
app/lib/connection_pool/shared_connection_pool.rb Normal file
View File

@ -0,0 +1,63 @@
# frozen_string_literal: true
require 'connection_pool'
require_relative './shared_timed_stack'
class ConnectionPool::SharedConnectionPool < ConnectionPool
def initialize(options = {}, &block)
super(options, &block)
@available = ConnectionPool::SharedTimedStack.new(@size, &block)
end
delegate :size, :flush, to: :@available
def with(preferred_tag, options = {})
Thread.handle_interrupt(Exception => :never) do
conn = checkout(preferred_tag, options)
begin
Thread.handle_interrupt(Exception => :immediate) do
yield conn
end
ensure
checkin(preferred_tag)
end
end
end
def checkout(preferred_tag, options = {})
if ::Thread.current[key(preferred_tag)]
::Thread.current[key_count(preferred_tag)] += 1
::Thread.current[key(preferred_tag)]
else
::Thread.current[key_count(preferred_tag)] = 1
::Thread.current[key(preferred_tag)] = @available.pop(preferred_tag, options[:timeout] || @timeout)
end
end
def checkin(preferred_tag)
if ::Thread.current[key(preferred_tag)]
if ::Thread.current[key_count(preferred_tag)] == 1
@available.push(::Thread.current[key(preferred_tag)])
::Thread.current[key(preferred_tag)] = nil
else
::Thread.current[key_count(preferred_tag)] -= 1
end
else
raise ConnectionPool::Error, 'no connections are checked out'
end
nil
end
private
def key(tag)
:"#{@key}-#{tag}"
end
def key_count(tag)
:"#{@key_count}-#{tag}"
end
end

95
app/lib/connection_pool/shared_timed_stack.rb Normal file
View File

@ -0,0 +1,95 @@
# frozen_string_literal: true
class ConnectionPool::SharedTimedStack
def initialize(max = 0, &block)
@create_block = block
@max = max
@created = 0
@queue = []
@tagged_queue = Hash.new { |hash, key| hash[key] = [] }
@mutex = Mutex.new
@resource = ConditionVariable.new
end
def push(connection)
@mutex.synchronize do
store_connection(connection)
@resource.broadcast
end
end
alias << push
def pop(preferred_tag, timeout = 5.0)
deadline = current_time + timeout
@mutex.synchronize do
loop do
return fetch_preferred_connection(preferred_tag) unless @tagged_queue[preferred_tag].empty?
connection = try_create(preferred_tag)
return connection if connection
to_wait = deadline - current_time
raise Timeout::Error, "Waited #{timeout} sec" if to_wait <= 0
@resource.wait(@mutex, to_wait)
end
end
end
def empty?
size.zero?
end
def size
@mutex.synchronize do
@queue.size
end
end
def flush
@mutex.synchronize do
@queue.delete_if do |connection|
delete = !connection.in_use && (connection.dead || connection.seconds_idle >= RequestPool::MAX_IDLE_TIME)
if delete
@tagged_queue[connection.site].delete(connection)
connection.close
@created -= 1
end
delete
end
end
end
private
def try_create(preferred_tag)
if @created == @max && !@queue.empty?
throw_away_connection = @queue.pop
@tagged_queue[throw_away_connection.site].delete(throw_away_connection)
@create_block.call(preferred_tag)
elsif @created != @max
connection = @create_block.call(preferred_tag)
@created += 1
connection
end
end
def fetch_preferred_connection(preferred_tag)
connection = @tagged_queue[preferred_tag].pop
@queue.delete(connection)
connection
end
def current_time
Process.clock_gettime(Process::CLOCK_MONOTONIC)
end
def store_connection(connection)
@tagged_queue[connection.site].push(connection)
@queue.push(connection)
end
end

2
app/lib/feed_manager.rb
View File

@ -65,7 +65,7 @@ class FeedManager
reblog_key = key(type, account_id, 'reblogs')
# Remove any items past the MAX_ITEMS'th entry in our feed
redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s)
redis.zremrangebyrank(timeline_key, 0, -(FeedManager::MAX_ITEMS + 1))
# Get the score of the REBLOG_FALLOFF'th item in our feed, and stop
# tracking anything after it for deduplication purposes.

7
app/lib/formatter.rb
View File

@ -84,8 +84,7 @@ class Formatter
end
def format_field(account, str, **options)
return reformat(str).html_safe unless account.local? # rubocop:disable Rails/OutputSafety
html = encode_and_link_urls(str, me: true)
html = account.local? ? encode_and_link_urls(str, me: true) : reformat(str)
html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify]
html.html_safe # rubocop:disable Rails/OutputSafety
end
@ -300,10 +299,10 @@ class Formatter
end
def hashtag_html(tag)
"<a href=\"#{encode(tag_url(tag.downcase))}\" class=\"mention hashtag\" rel=\"tag\">#<span>#{encode(tag)}</span></a>"
"<a href=\"#{encode(tag_url(tag))}\" class=\"mention hashtag\" rel=\"tag\">#<span>#{encode(tag)}</span></a>"
end
def mention_html(account)
"<span class=\"h-card\"><a href=\"#{encode(TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>"
"<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>"
end
end

7
app/lib/nodeinfo/adapter.rb Normal file
View File

@ -0,0 +1,7 @@
# frozen_string_literal: true
class NodeInfo::Adapter < ActiveModelSerializers::Adapter::Attributes
def self.default_key_transform
:camel_lower
end
end

71
app/lib/ostatus/activity/base.rb
View File

@ -1,71 +0,0 @@
# frozen_string_literal: true
class OStatus::Activity::Base
include Redisable
def initialize(xml, account = nil, **options)
@xml = xml
@account = account
@options = options
end
def status?
[:activity, :note, :comment].include?(type)
end
def verb
raw = @xml.at_xpath('./activity:verb', activity: OStatus::TagManager::AS_XMLNS).content
OStatus::TagManager::VERBS.key(raw)
rescue
:post
end
def type
raw = @xml.at_xpath('./activity:object-type', activity: OStatus::TagManager::AS_XMLNS).content
OStatus::TagManager::TYPES.key(raw)
rescue
:activity
end
def id
@xml.at_xpath('./xmlns:id', xmlns: OStatus::TagManager::XMLNS).content
end
def url
link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| link_candidate['type'] == 'text/html' }
link.nil? ? nil : link['href']
end
def activitypub_uri
link = @xml.xpath('./xmlns:link[@rel="alternate"]', xmlns: OStatus::TagManager::XMLNS).find { |link_candidate| ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(link_candidate['type']) }
link.nil? ? nil : link['href']
end
def activitypub_uri?
activitypub_uri.present?
end
private
def find_status(uri)
if OStatus::TagManager.instance.local_id?(uri)
local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Status')
return Status.find_by(id: local_id)
elsif ActivityPub::TagManager.instance.local_uri?(uri)
local_id = ActivityPub::TagManager.instance.uri_to_local_id(uri)
return Status.find_by(id: local_id)
end
Status.find_by(uri: uri)
end
def find_activitypub_status(uri, href)
tag_matches = /tag:([^,:]+)[^:]*:objectId=([\d]+)/.match(uri)
href_matches = %r{/users/([^/]+)}.match(href)
unless tag_matches.nil? || href_matches.nil?
uri = "https://#{tag_matches[1]}/users/#{href_matches[1]}/statuses/#{tag_matches[2]}"
Status.find_by(uri: uri)
end
end
end

219
app/lib/ostatus/activity/creation.rb
View File

@ -1,219 +0,0 @@
# frozen_string_literal: true
class OStatus::Activity::Creation < OStatus::Activity::Base
def perform
if redis.exists("delete_upon_arrival:#{@account.id}:#{id}")
Rails.logger.debug "Delete for status #{id} was queued, ignoring"
return [nil, false]
end
return [nil, false] if @account.suspended? || invalid_origin?
RedisLock.acquire(lock_options) do |lock|
if lock.acquired?
# Return early if status already exists in db
@status = find_status(id)
return [@status, false] unless @status.nil?
@status = process_status
else
raise Mastodon::RaceConditionError
end
end
[@status, true]
end
def process_status
Rails.logger.debug "Creating remote status #{id}"
cached_reblog = reblog
status = nil
# Skip if the reblogged status is not public
return if cached_reblog && !(cached_reblog.public_visibility? || cached_reblog.unlisted_visibility?)
media_attachments = save_media.take(4)
ApplicationRecord.transaction do
status = Status.create!(
uri: id,
url: url,
account: @account,
reblog: cached_reblog,
text: content,
spoiler_text: content_warning,
created_at: published,
override_timestamps: @options[:override_timestamps],
reply: thread?,
language: content_language,
visibility: visibility_scope,
conversation: find_or_create_conversation,
thread: thread? ? find_status(thread.first) || find_activitypub_status(thread.first, thread.second) : nil,
media_attachment_ids: media_attachments.map(&:id),
sensitive: sensitive?
)
save_mentions(status)
save_hashtags(status)
save_emojis(status)
end
if thread? && status.thread.nil? && Request.valid_url?(thread.second)
Rails.logger.debug "Trying to attach #{status.id} (#{id}) to #{thread.first}"
ThreadResolveWorker.perform_async(status.id, thread.second)
end
Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"
LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
# Only continue if the status is supposed to have arrived in real-time.
# Note that if @options[:override_timestamps] isn't set, the status
# may have a lower snowflake id than other existing statuses, potentially
# "hiding" it from paginated API calls
return status unless @options[:override_timestamps] || status.within_realtime_window?
DistributionWorker.perform_async(status.id)
status
end
def content
@xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS).content
end
def content_language
@xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS)['xml:lang']&.presence || 'en'
end
def content_warning
@xml.at_xpath('./xmlns:summary', xmlns: OStatus::TagManager::XMLNS)&.content || ''
end
def visibility_scope
@xml.at_xpath('./mastodon:scope', mastodon: OStatus::TagManager::MTDN_XMLNS)&.content&.to_sym || :public
end
def published
@xml.at_xpath('./xmlns:published', xmlns: OStatus::TagManager::XMLNS).content
end
def thread?
!@xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS).nil?
end
def thread
thr = @xml.at_xpath('./thr:in-reply-to', thr: OStatus::TagManager::THR_XMLNS)
[thr['ref'], thr['href']]
end
private
def sensitive?
# OStatus-specific convention (not standard)
@xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).any? { |category| category['term'] == 'nsfw' }
end
def find_or_create_conversation
uri = @xml.at_xpath('./ostatus:conversation', ostatus: OStatus::TagManager::OS_XMLNS)&.attribute('ref')&.content
return if uri.nil?
if OStatus::TagManager.instance.local_id?(uri)
local_id = OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')
return Conversation.find_by(id: local_id)
end
Conversation.find_by(uri: uri) || Conversation.create!(uri: uri)
end
def save_mentions(parent)
processed_account_ids = []
@xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
next if [OStatus::TagManager::TYPES[:group], OStatus::TagManager::TYPES[:collection]].include? link['ostatus:object-type']
mentioned_account = account_from_href(link['href'])
next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id)
mentioned_account.mentions.where(status: parent).first_or_create(status: parent)
# So we can skip duplicate mentions
processed_account_ids << mentioned_account.id
end
end
def save_hashtags(parent)
tags = @xml.xpath('./xmlns:category', xmlns: OStatus::TagManager::XMLNS).map { |category| category['term'] }.select(&:present?)
ProcessHashtagsService.new.call(parent, tags)
end
def save_media
do_not_download = DomainBlock.reject_media?(@account.domain)
media_attachments = []
@xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
next unless link['href']
media = MediaAttachment.where(status: nil, remote_url: link['href']).first_or_initialize(account: @account, status: nil, remote_url: link['href'])
parsed_url = Addressable::URI.parse(link['href']).normalize
next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty?
media.save
media_attachments << media
next if do_not_download
begin
media.file_remote_url = link['href']
media.save!
rescue ActiveRecord::RecordInvalid
next
end
end
media_attachments
end
def save_emojis(parent)
do_not_download = DomainBlock.reject_media?(parent.account.domain)
return if do_not_download
@xml.xpath('./xmlns:link[@rel="emoji"]', xmlns: OStatus::TagManager::XMLNS).each do |link|
next unless link['href'] && link['name']
shortcode = link['name'].delete(':')
emoji = CustomEmoji.find_by(shortcode: shortcode, domain: parent.account.domain)
next unless emoji.nil?
emoji = CustomEmoji.new(shortcode: shortcode, domain: parent.account.domain)
emoji.image_remote_url = link['href']
emoji.save
end
end
def account_from_href(href)
url = Addressable::URI.parse(href).normalize
if TagManager.instance.web_domain?(url.host)
Account.find_local(url.path.gsub('/users/', ''))
else
Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href)
end
end
def invalid_origin?
return false unless id.start_with?('http') # Legacy IDs cannot be checked
needle = Addressable::URI.parse(id).normalized_host
!(needle.casecmp(@account.domain).zero? ||
needle.casecmp(Addressable::URI.parse(@account.remote_url.presence || @account.uri).normalized_host).zero?)
end
def lock_options
{ redis: Redis.current, key: "create:#{id}" }
end
end

16
app/lib/ostatus/activity/deletion.rb
View File

@ -1,16 +0,0 @@
# frozen_string_literal: true
class OStatus::Activity::Deletion < OStatus::Activity::Base
def perform
Rails.logger.debug "Deleting remote status #{id}"
status = Status.find_by(uri: id, account: @account)
status ||= Status.find_by(uri: activitypub_uri, account: @account) if activitypub_uri?
if status.nil?
redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id)
else
RemoveStatusService.new.call(status)
end
end
end

20
app/lib/ostatus/activity/general.rb
View File

@ -1,20 +0,0 @@
# frozen_string_literal: true
class OStatus::Activity::General < OStatus::Activity::Base
def specialize
special_class&.new(@xml, @account, @options)
end
private
def special_class
case verb
when :post
OStatus::Activity::Post
when :share
OStatus::Activity::Share
when :delete
OStatus::Activity::Deletion
end
end
end

23
app/lib/ostatus/activity/post.rb
View File

@ -1,23 +0,0 @@
# frozen_string_literal: true
class OStatus::Activity::Post < OStatus::Activity::Creation
def perform
status, just_created = super
if just_created
status.mentions.includes(:account).each do |mention|
mentioned_account = mention.account
next unless mentioned_account.local?
NotifyService.new.call(mentioned_account, mention)
end
end
status
end
private
def reblog
nil
end
end

11
app/lib/ostatus/activity/remote.rb
View File

@ -1,11 +0,0 @@
# frozen_string_literal: true
class OStatus::Activity::Remote < OStatus::Activity::Base
def perform
if activitypub_uri?
find_status(activitypub_uri) || FetchRemoteStatusService.new.call(url)
else
find_status(id) || FetchRemoteStatusService.new.call(url)
end
end
end

26
app/lib/ostatus/activity/share.rb
View File

@ -1,26 +0,0 @@
# frozen_string_literal: true
class OStatus::Activity::Share < OStatus::Activity::Creation
def perform
return if reblog.nil?
status, just_created = super
NotifyService.new.call(reblog.account, status) if reblog.account.local? && just_created
status
end
def object
@xml.at_xpath('.//activity:object', activity: OStatus::TagManager::AS_XMLNS)
end
private
def reblog
return @reblog if defined? @reblog
original_status = OStatus::Activity::Remote.new(object).perform
return if original_status.nil?
@reblog = original_status.reblog? ? original_status.reblog : original_status
end
end

378
app/lib/ostatus/atom_serializer.rb
View File

@ -1,378 +0,0 @@
# frozen_string_literal: true
class OStatus::AtomSerializer
include RoutingHelper
include ActionView::Helpers::SanitizeHelper
class << self
def render(element)
document = Ox::Document.new(version: '1.0')
document << element
('<?xml version="1.0"?>' + Ox.dump(element, effort: :tolerant)).force_encoding('UTF-8')
end
end
def author(account)
author = Ox::Element.new('author')
uri = OStatus::TagManager.instance.uri_for(account)
append_element(author, 'id', uri)
append_element(author, 'activity:object-type', OStatus::TagManager::TYPES[:person])
append_element(author, 'uri', uri)
append_element(author, 'name', account.username)
append_element(author, 'email', account.local? ? account.local_username_and_domain : account.acct)
append_element(author, 'summary', Formatter.instance.simplified_format(account).to_str, type: :html) if account.note?
append_element(author, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account))
append_element(author, 'link', nil, rel: :avatar, type: account.avatar_content_type, 'media:width': 120, 'media:height': 120, href: full_asset_url(account.avatar.url(:original))) if account.avatar?
append_element(author, 'link', nil, rel: :header, type: account.header_content_type, 'media:width': 700, 'media:height': 335, href: full_asset_url(account.header.url(:original))) if account.header?
account.emojis.each do |emoji|
append_element(author, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode)
end
append_element(author, 'poco:preferredUsername', account.username)
append_element(author, 'poco:displayName', account.display_name) if account.display_name?
append_element(author, 'poco:note', account.local? ? account.note : strip_tags(account.note)) if account.note?
append_element(author, 'mastodon:scope', account.locked? ? :private : :public)
author
end
def feed(account, stream_entries)
feed = Ox::Element.new('feed')
add_namespaces(feed)
append_element(feed, 'id', account_url(account, format: 'atom'))
append_element(feed, 'title', account.display_name.presence || account.username)
append_element(feed, 'subtitle', account.note)
append_element(feed, 'updated', account.updated_at.iso8601)
append_element(feed, 'logo', full_asset_url(account.avatar.url(:original)))
feed << author(account)
append_element(feed, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(account))
append_element(feed, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_url(account, format: 'atom'))
append_element(feed, 'link', nil, rel: :next, type: 'application/atom+xml', href: account_url(account, format: 'atom', max_id: stream_entries.last.id)) if stream_entries.size == 20
append_element(feed, 'link', nil, rel: :hub, href: api_push_url)
append_element(feed, 'link', nil, rel: :salmon, href: api_salmon_url(account.id))
stream_entries.each do |stream_entry|
feed << entry(stream_entry)
end
feed
end
def entry(stream_entry, root = false)
entry = Ox::Element.new('entry')
add_namespaces(entry) if root
append_element(entry, 'id', OStatus::TagManager.instance.uri_for(stream_entry.status))
append_element(entry, 'published', stream_entry.created_at.iso8601)
append_element(entry, 'updated', stream_entry.updated_at.iso8601)
append_element(entry, 'title', stream_entry&.status&.title || "#{stream_entry.account.acct} deleted status")
entry << author(stream_entry.account) if root
append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[stream_entry.object_type])
append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[stream_entry.verb])
entry << object(stream_entry.target) if stream_entry.targeted?
if stream_entry.status.nil?
append_element(entry, 'content', 'Deleted status')
elsif stream_entry.status.destroyed?
append_element(entry, 'content', 'Deleted status')
append_element(entry, 'link', nil, rel: :alternate, type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(stream_entry.status)) if stream_entry.account.local?
else
serialize_status_attributes(entry, stream_entry.status)
end
append_element(entry, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(stream_entry.status))
append_element(entry, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_stream_entry_url(stream_entry.account, stream_entry, format: 'atom'))
append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(stream_entry.thread), href: ::TagManager.instance.url_for(stream_entry.thread)) if stream_entry.threaded?
append_element(entry, 'ostatus:conversation', nil, ref: conversation_uri(stream_entry.status.conversation)) unless stream_entry&.status&.conversation_id.nil?
entry
end
def object(status)
object = Ox::Element.new('activity:object')
append_element(object, 'id', OStatus::TagManager.instance.uri_for(status))
append_element(object, 'published', status.created_at.iso8601)
append_element(object, 'updated', status.updated_at.iso8601)
append_element(object, 'title', status.title)
object << author(status.account)
append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[status.object_type])
append_element(object, 'activity:verb', OStatus::TagManager::VERBS[status.verb])
serialize_status_attributes(object, status)
append_element(object, 'link', nil, rel: :alternate, type: 'text/html', href: ::TagManager.instance.url_for(status))
append_element(object, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(status.thread), href: ::TagManager.instance.url_for(status.thread)) unless status.thread.nil?
append_element(object, 'ostatus:conversation', nil, ref: conversation_uri(status.conversation)) unless status.conversation_id.nil?
object
end
def follow_salmon(follow)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{follow.account.acct} started following #{follow.target_account.acct}"
append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(follow.created_at, follow.id, 'Follow'))
append_element(entry, 'title', description)
append_element(entry, 'content', description, type: :html)
entry << author(follow.account)
append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:follow])
object = author(follow.target_account)
object.value = 'activity:object'
entry << object
entry
end
def follow_request_salmon(follow_request)
entry = Ox::Element.new('entry')
add_namespaces(entry)
append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(follow_request.created_at, follow_request.id, 'FollowRequest'))
append_element(entry, 'title', "#{follow_request.account.acct} requested to follow #{follow_request.target_account.acct}")
entry << author(follow_request.account)
append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
object = author(follow_request.target_account)
object.value = 'activity:object'
entry << object
entry
end
def authorize_follow_request_salmon(follow_request)
entry = Ox::Element.new('entry')
add_namespaces(entry)
append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
append_element(entry, 'title', "#{follow_request.target_account.acct} authorizes follow request by #{follow_request.account.acct}")
entry << author(follow_request.target_account)
append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:authorize])
object = Ox::Element.new('activity:object')
object << author(follow_request.account)
append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
append_element(object, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
inner_object = author(follow_request.target_account)
inner_object.value = 'activity:object'
object << inner_object
entry << object
entry
end
def reject_follow_request_salmon(follow_request)
entry = Ox::Element.new('entry')
add_namespaces(entry)
append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow_request.id, 'FollowRequest'))
append_element(entry, 'title', "#{follow_request.target_account.acct} rejects follow request by #{follow_request.account.acct}")
entry << author(follow_request.target_account)
append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:reject])
object = Ox::Element.new('activity:object')
object << author(follow_request.account)
append_element(object, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
append_element(object, 'activity:verb', OStatus::TagManager::VERBS[:request_friend])
inner_object = author(follow_request.target_account)
inner_object.value = 'activity:object'
object << inner_object
entry << object
entry
end
def unfollow_salmon(follow)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{follow.account.acct} is no longer following #{follow.target_account.acct}"
append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, follow.id, 'Follow'))
append_element(entry, 'title', description)
append_element(entry, 'content', description, type: :html)
entry << author(follow.account)
append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unfollow])
object = author(follow.target_account)
object.value = 'activity:object'
entry << object
entry
end
def block_salmon(block)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{block.account.acct} no longer wishes to interact with #{block.target_account.acct}"
append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
append_element(entry, 'title', description)
entry << author(block.account)
append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:block])
object = author(block.target_account)
object.value = 'activity:object'
entry << object
entry
end
def unblock_salmon(block)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{block.account.acct} no longer blocks #{block.target_account.acct}"
append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, block.id, 'Block'))
append_element(entry, 'title', description)
entry << author(block.account)
append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unblock])
object = author(block.target_account)
object.value = 'activity:object'
entry << object
entry
end
def favourite_salmon(favourite)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{favourite.account.acct} favourited a status by #{favourite.status.account.acct}"
append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(favourite.created_at, favourite.id, 'Favourite'))
append_element(entry, 'title', description)
append_element(entry, 'content', description, type: :html)
entry << author(favourite.account)
append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:favorite])
entry << object(favourite.status)
append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(favourite.status), href: ::TagManager.instance.url_for(favourite.status))
entry
end
def unfavourite_salmon(favourite)
entry = Ox::Element.new('entry')
add_namespaces(entry)
description = "#{favourite.account.acct} no longer favourites a status by #{favourite.status.account.acct}"
append_element(entry, 'id', OStatus::TagManager.instance.unique_tag(Time.now.utc, favourite.id, 'Favourite'))
append_element(entry, 'title', description)
append_element(entry, 'content', description, type: :html)
entry << author(favourite.account)
append_element(entry, 'activity:object-type', OStatus::TagManager::TYPES[:activity])
append_element(entry, 'activity:verb', OStatus::TagManager::VERBS[:unfavorite])
entry << object(favourite.status)
append_element(entry, 'thr:in-reply-to', nil, ref: OStatus::TagManager.instance.uri_for(favourite.status), href: ::TagManager.instance.url_for(favourite.status))
entry
end
private
def append_element(parent, name, content = nil, **attributes)
element = Ox::Element.new(name)
attributes.each { |k, v| element[k] = sanitize_str(v) }
element << sanitize_str(content) unless content.nil?
parent << element
end
def sanitize_str(raw_str)
raw_str.to_s
end
def conversation_uri(conversation)
return conversation.uri if conversation.uri?
OStatus::TagManager.instance.unique_tag(conversation.created_at, conversation.id, 'Conversation')
end
def add_namespaces(parent)
parent['xmlns'] = OStatus::TagManager::XMLNS
parent['xmlns:thr'] = OStatus::TagManager::THR_XMLNS
parent['xmlns:activity'] = OStatus::TagManager::AS_XMLNS
parent['xmlns:poco'] = OStatus::TagManager::POCO_XMLNS
parent['xmlns:media'] = OStatus::TagManager::MEDIA_XMLNS
parent['xmlns:ostatus'] = OStatus::TagManager::OS_XMLNS
parent['xmlns:mastodon'] = OStatus::TagManager::MTDN_XMLNS
end
def serialize_status_attributes(entry, status)
append_element(entry, 'link', nil, rel: :alternate, type: 'application/activity+json', href: ActivityPub::TagManager.instance.uri_for(status)) if status.account.local?
append_element(entry, 'summary', status.spoiler_text, 'xml:lang': status.language) if status.spoiler_text?
append_element(entry, 'content', Formatter.instance.format(status, inline_poll_options: true).to_str || '.', type: 'html', 'xml:lang': status.language)
status.active_mentions.sort_by(&:id).each do |mentioned|
append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:person], href: OStatus::TagManager.instance.uri_for(mentioned.account))
end
append_element(entry, 'link', nil, rel: :mentioned, 'ostatus:object-type': OStatus::TagManager::TYPES[:collection], href: OStatus::TagManager::COLLECTIONS[:public]) if status.public_visibility?
status.tags.each do |tag|
append_element(entry, 'category', nil, term: tag.name)
end
status.media_attachments.each do |media|
append_element(entry, 'link', nil, rel: :enclosure, type: media.file_content_type, length: media.file_file_size, href: full_asset_url(media.file.url(:original, false)))
end
append_element(entry, 'category', nil, term: 'nsfw') if status.sensitive? && status.media_attachments.any?
append_element(entry, 'mastodon:scope', status.visibility)
status.emojis.each do |emoji|
append_element(entry, 'link', nil, rel: :emoji, href: full_asset_url(emoji.image.url), name: emoji.shortcode)
end
end
end

131
app/lib/request.rb
View File

@ -17,15 +17,22 @@ end
class Request
REQUEST_TARGET = '(request-target)'
# We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening
# and 5s timeout on the TLS handshake, meaning the worst case should take
# about 15s in total
TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze
include RoutingHelper
def initialize(verb, url, **options)
raise ArgumentError if url.blank?
@verb = verb
@url = Addressable::URI.parse(url).normalize
@options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket })
@headers = {}
@verb = verb
@url = Addressable::URI.parse(url).normalize
@http_client = options.delete(:http_client)
@options = options.merge(socket_class: use_proxy? ? ProxySocket : Socket)
@options = @options.merge(Rails.configuration.x.http_client_proxy) if use_proxy?
@headers = {}
raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service?
@ -33,8 +40,8 @@ class Request
set_digest! if options.key?(:body)
end
def on_behalf_of(account, key_id_format = :acct, sign_with: nil)
raise ArgumentError unless account.local?
def on_behalf_of(account, key_id_format = :uri, sign_with: nil)
raise ArgumentError, 'account must not be nil' if account.nil?
@account = account
@keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair
@ -50,15 +57,24 @@ class Request
def perform
begin
response = http_client.headers(headers).public_send(@verb, @url.to_s, @options)
response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers))
rescue => e
raise e.class, "#{e.message} on #{@url}", e.backtrace[0]
raise e.class, "#{e.message} on #{@url}", e.backtrace
end
begin
yield response.extend(ClientLimit) if block_given?
response = response.extend(ClientLimit)
# If we are using a persistent connection, we have to
# read every response to be able to move forward at all.
# However, simply calling #to_s or #flush may not be safe,
# as the response body, if malicious, could be too big
# for our memory. So we use the #body_with_limit method
response.body_with_limit if http_client.persistent?
yield response if block_given?
ensure
http_client.close
http_client.close unless http_client.persistent?
end
end
@ -76,6 +92,10 @@ class Request
%w(http https).include?(parsed_url.scheme) && parsed_url.host.present?
end
def http_client
HTTP.use(:auto_inflate).timeout(:per_operation, TIMEOUT.dup).follow(max_hops: 2)
end
end
private
@ -116,16 +136,8 @@ class Request
end
end
def timeout
# We enforce a 1s timeout on DNS resolving, 10s timeout on socket opening
# and 5s timeout on the TLS handshake, meaning the worst case should take
# about 16s in total
{ connect: 5, read: 10, write: 10 }
end
def http_client
@http_client ||= HTTP.use(:auto_inflate).timeout(:per_operation, timeout).follow(max_hops: 2)
@http_client ||= Request.http_client
end
def use_proxy?
@ -166,26 +178,67 @@ class Request
class Socket < TCPSocket
class << self
def open(host, *args)
return super(host, *args) if thru_hidden_service?(host)
outer_e = nil
port = args.first
Resolv::DNS.open do |dns|
dns.timeouts = 5
addresses = []
begin
addresses = [IPAddr.new(host)]
rescue IPAddr::InvalidAddressError
Resolv::DNS.open do |dns|
dns.timeouts = 5
addresses = dns.getaddresses(host).take(2)
end
end
addresses = dns.getaddresses(host).take(2)
time_slot = 10.0 / addresses.size
socks = []
addr_by_socket = {}
addresses.each do |address|
begin
check_private_address(address)
sock = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0)
sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s)
sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1)
sock.connect_nonblock(sockaddr)
# If that hasn't raised an exception, we somehow managed to connect
# immediately, close pending sockets and return immediately
socks.each(&:close)
return sock
rescue IO::WaitWritable
socks << sock
addr_by_socket[sock] = sockaddr
rescue => e
outer_e = e
end
end
until socks.empty?
_, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect])
if available_socks.nil?
socks.each(&:close)
raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds"
end
available_socks.each do |sock|
socks.delete(sock)
addresses.each do |address|
begin
raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(IPAddr.new(address.to_s))
::Timeout.timeout(time_slot, HTTP::TimeoutError) do
return super(address.to_s, *args)
end
sock.connect_nonblock(addr_by_socket[sock])
rescue Errno::EISCONN
rescue => e
sock.close
outer_e = e
next
end
socks.each(&:close)
return sock
end
end
@ -198,11 +251,21 @@ class Request
alias new open
def thru_hidden_service?(host)
Rails.configuration.x.access_to_hidden_service && /\.(onion|i2p)$/.match(host)
def check_private_address(address)
raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(IPAddr.new(address.to_s))
end
end
end
private_constant :ClientLimit, :Socket
class ProxySocket < Socket
class << self
def check_private_address(_address)
# Accept connections to private addresses as HTTP proxies will usually
# be on local addresses
nil
end
end
end
private_constant :ClientLimit, :Socket, :ProxySocket
end

114
app/lib/request_pool.rb Normal file
View File

@ -0,0 +1,114 @@
# frozen_string_literal: true
require_relative './connection_pool/shared_connection_pool'
class RequestPool
def self.current
@current ||= RequestPool.new
end
class Reaper
attr_reader :pool, :frequency
def initialize(pool, frequency)
@pool = pool
@frequency = frequency
end
def run
return unless frequency&.positive?
Thread.new(frequency, pool) do |t, p|
loop do
sleep t
p.flush
end
end
end
end
MAX_IDLE_TIME = 30
WAIT_TIMEOUT = 5
MAX_POOL_SIZE = ENV.fetch('MAX_REQUEST_POOL_SIZE', 512).to_i
class Connection
attr_reader :site, :last_used_at, :created_at, :in_use, :dead, :fresh
def initialize(site)
@site = site
@http_client = http_client
@last_used_at = nil
@created_at = current_time
@dead = false
@fresh = true
end
def use
@last_used_at = current_time
@in_use = true
retries = 0
begin
yield @http_client
rescue HTTP::ConnectionError
# It's possible the connection was closed, so let's
# try re-opening it once
close
if @fresh || retries.positive?
raise
else
@http_client = http_client
retries += 1
retry
end
rescue StandardError
# If this connection raises errors of any kind, it's
# better if it gets reaped as soon as possible
close
@dead = true
raise
end
ensure
@fresh = false
@in_use = false
end
def seconds_idle
current_time - (@last_used_at || @created_at)
end
def close
@http_client.close
end
private
def http_client
Request.http_client.persistent(@site, timeout: MAX_IDLE_TIME)
end
def current_time
Process.clock_gettime(Process::CLOCK_MONOTONIC)
end
end
def initialize
@pool = ConnectionPool::SharedConnectionPool.new(size: MAX_POOL_SIZE, timeout: WAIT_TIMEOUT) { |site| Connection.new(site) }
@reaper = Reaper.new(self, 30)
@reaper.run
end
def with(site, &block)
@pool.with(site) do |connection|
ActiveSupport::Notifications.instrument('with.request_pool', miss: connection.fresh, host: connection.site) do
connection.use(&block)
end
end
end
delegate :size, :flush, to: :@pool
end

15
app/lib/search_query_parser.rb Normal file
View File

@ -0,0 +1,15 @@
# frozen_string_literal: true
class SearchQueryParser < Parslet::Parser
rule(:term) { match('[^\s":]').repeat(1).as(:term) }
rule(:quote) { str('"') }
rule(:colon) { str(':') }
rule(:space) { match('\s').repeat(1) }
rule(:operator) { (str('+') | str('-')).as(:operator) }
rule(:prefix) { (term >> colon).as(:prefix) }
rule(:shortcode) { (colon >> term >> colon.maybe).as(:shortcode) }
rule(:phrase) { (quote >> (term >> space.maybe).repeat >> quote).as(:phrase) }
rule(:clause) { (prefix.maybe >> operator.maybe >> (phrase | term | shortcode)).as(:clause) }
rule(:query) { (clause >> space.maybe).repeat.as(:query) }
root(:query)
end

88
app/lib/search_query_transformer.rb Normal file
View File

@ -0,0 +1,88 @@
# frozen_string_literal: true
class SearchQueryTransformer < Parslet::Transform
class Query
attr_reader :should_clauses, :must_not_clauses, :must_clauses
def initialize(clauses)
grouped = clauses.chunk(&:operator).to_h
@should_clauses = grouped.fetch(:should, [])
@must_not_clauses = grouped.fetch(:must_not, [])
@must_clauses = grouped.fetch(:must, [])
end
def apply(search)
should_clauses.each { |clause| search = search.query.should(clause_to_query(clause)) }
must_clauses.each { |clause| search = search.query.must(clause_to_query(clause)) }
must_not_clauses.each { |clause| search = search.query.must_not(clause_to_query(clause)) }
search.query.minimum_should_match(1)
end
private
def clause_to_query(clause)
case clause
when TermClause
{ multi_match: { type: 'most_fields', query: clause.term, fields: ['text', 'text.stemmed'] } }
when PhraseClause
{ match_phrase: { text: { query: clause.phrase } } }
else
raise "Unexpected clause type: #{clause}"
end
end
end
class Operator
class << self
def symbol(str)
case str
when '+'
:must
when '-'
:must_not
when nil
:should
else
raise "Unknown operator: #{str}"
end
end
end
end
class TermClause
attr_reader :prefix, :operator, :term
def initialize(prefix, operator, term)
@prefix = prefix
@operator = Operator.symbol(operator)
@term = term
end
end
class PhraseClause
attr_reader :prefix, :operator, :phrase
def initialize(prefix, operator, phrase)
@prefix = prefix
@operator = Operator.symbol(operator)
@phrase = phrase
end
end
rule(clause: subtree(:clause)) do
prefix = clause[:prefix][:term].to_s if clause[:prefix]
operator = clause[:operator]&.to_s
if clause[:term]
TermClause.new(prefix, operator, clause[:term].to_s)
elsif clause[:shortcode]
TermClause.new(prefix, operator, ":#{clause[:term]}:")
elsif clause[:phrase]
PhraseClause.new(prefix, operator, clause[:phrase].map { |p| p[:term].to_s }.join(' '))
else
raise "Unexpected clause type: #{clause}"
end
end
rule(query: sequence(:clauses)) { Query.new(clauses) }
end

1
app/lib/settings/scoped_settings.rb
View File

@ -4,6 +4,7 @@ module Settings
class ScopedSettings
DEFAULTING_TO_UNSCOPED = %w(
theme
noindex
).freeze
def initialize(object)

203
app/lib/spam_check.rb Normal file
View File

@ -0,0 +1,203 @@
# frozen_string_literal: true
class SpamCheck
include Redisable
include ActionView::Helpers::TextHelper
# Threshold over which two Nilsimsa values are considered
# to refer to the same text
NILSIMSA_COMPARE_THRESHOLD = 95
# Nilsimsa doesn't work well on small inputs, so below
# this size, we check only for exact matches with MD5
NILSIMSA_MIN_SIZE = 10
# How long to keep the trail of digests between updates,
# there is no reason to store it forever
EXPIRE_SET_AFTER = 1.week.seconds
# How many digests to keep in an account's trail. If it's
# too small, spam could rotate around different message templates
MAX_TRAIL_SIZE = 10
# How many detected duplicates to allow through before
# considering the message as spam
THRESHOLD = 5
def initialize(status)
@account = status.account
@status = status
end
def skip?
disabled? || already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply?
end
def spam?
if insufficient_data?
false
elsif nilsimsa?
digests_over_threshold?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD }
else
digests_over_threshold?('md5') { |_, other_digest| other_digest == digest }
end
end
def flag!
auto_silence_account!
auto_report_status!
end
def remember!
# The scores in sorted sets don't actually have enough bits to hold an exact
# value of our snowflake IDs, so we use it only for its ordering property. To
# get the correct status ID back, we have to save it in the string value
redis.zadd(redis_key, @status.id, digest_with_algorithm)
redis.zremrangebyrank(redis_key, 0, -(MAX_TRAIL_SIZE + 1))
redis.expire(redis_key, EXPIRE_SET_AFTER)
end
def reset!
redis.del(redis_key)
end
def hashable_text
return @hashable_text if defined?(@hashable_text)
@hashable_text = @status.text
@hashable_text = remove_mentions(@hashable_text)
@hashable_text = strip_tags(@hashable_text) unless @status.local?
@hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text)
@hashable_text = remove_whitespace(@hashable_text)
end
def insufficient_data?
hashable_text.blank?
end
def digest
@digest ||= begin
if nilsimsa?
Nilsimsa.new(hashable_text).hexdigest
else
Digest::MD5.hexdigest(hashable_text)
end
end
end
def digest_with_algorithm
if nilsimsa?
['nilsimsa', digest, @status.id].join(':')
else
['md5', digest, @status.id].join(':')
end
end
class << self
def perform(status)
spam_check = new(status)
return if spam_check.skip?
if spam_check.spam?
spam_check.flag!
else
spam_check.remember!
end
end
end
private
def disabled?
!Setting.spam_check_enabled
end
def remove_mentions(text)
return text.gsub(Account::MENTION_RE, '') if @status.local?
Nokogiri::HTML.fragment(text).tap do |html|
mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) }
html.traverse do |element|
element.unlink if element.name == 'a' && mentions.include?(element['href'])
end
end.to_s
end
def normalize_unicode(text)
text.unicode_normalize(:nfkc).downcase
end
def remove_whitespace(text)
text.gsub(/\s+/, ' ').strip
end
def auto_silence_account!
@account.silence!
end
def auto_report_status!
status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable?
ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected_and_silenced'))
end
def already_flagged?
@account.silenced?
end
def trusted?
@account.trust_level > Account::TRUST_LEVELS[:untrusted]
end
def no_unsolicited_mentions?
@status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) }
end
def solicited_reply?
!@status.thread.nil? && @status.thread.mentions.where(account: @account).exists?
end
def nilsimsa_compare_value(first, second)
first = [first].pack('H*')
second = [second].pack('H*')
bits = 0
0.upto(31) do |i|
bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord
end
128 - bits # -128 <= Nilsimsa Compare Value <= 128
end
def nilsimsa?
hashable_text.size > NILSIMSA_MIN_SIZE
end
def other_digests
redis.zrange(redis_key, 0, -1)
end
def digests_over_threshold?(filter_algorithm)
other_digests.select do |record|
algorithm, other_digest, status_id = record.split(':')
next unless algorithm == filter_algorithm
yield algorithm, other_digest, status_id
end.size >= THRESHOLD
end
def matching_status_ids
if nilsimsa?
other_digests.select { |record| record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD }.map { |record| record.split(':')[2] }.compact
else
other_digests.select { |record| record.start_with?('md5') && record.split(':')[1] == digest }.map { |record| record.split(':')[2] }.compact
end
end
def redis_key
@redis_key ||= "spam_check:#{@account.id}"
end
end

2
app/lib/status_finder.rb
View File

@ -13,8 +13,6 @@ class StatusFinder
raise ActiveRecord::RecordNotFound unless TagManager.instance.local_url?(url)
case recognized_params[:controller]
when 'stream_entries'
StreamEntry.find(recognized_params[:id]).status
when 'statuses'
Status.find(recognized_params[:id])
else

14
app/lib/tag_manager.rb
View File

@ -24,24 +24,16 @@ class TagManager
def same_acct?(canonical, needle)
return true if canonical.casecmp(needle).zero?
username, domain = needle.split('@')
local_domain?(domain) && canonical.casecmp(username).zero?
end
def local_url?(url)
uri = Addressable::URI.parse(url).normalize
domain = uri.host + (uri.port ? ":#{uri.port}" : '')
TagManager.instance.web_domain?(domain)
end
def url_for(target)
return target.url if target.respond_to?(:local?) && !target.local?
case target.object_type
when :person
short_account_url(target)
when :note, :comment, :activity
short_account_status_url(target.account, target)
end
end
end

69
app/lib/toc_generator.rb Normal file
View File

@ -0,0 +1,69 @@
# frozen_string_literal: true
class TOCGenerator
TARGET_ELEMENTS = %w(h1 h2 h3 h4 h5 h6).freeze
LISTED_ELEMENTS = %w(h2 h3).freeze
class Section
attr_accessor :depth, :title, :children, :anchor
def initialize(depth, title, anchor)
@depth = depth
@title = title
@children = []
@anchor = anchor
end
delegate :<<, to: :children
end
def initialize(source_html)
@source_html = source_html
@processed = false
@target_html = ''
@headers = []
@slugs = Hash.new { |h, k| h[k] = 0 }
end
def html
parse_and_transform unless @processed
@target_html
end
def toc
parse_and_transform unless @processed
@headers
end
private
def parse_and_transform
return if @source_html.blank?
parsed_html = Nokogiri::HTML.fragment(@source_html)
parsed_html.traverse do |node|
next unless TARGET_ELEMENTS.include?(node.name)
anchor = node['id'] || node.text.parameterize.presence || 'sec'
@slugs[anchor] += 1
anchor = "#{anchor}-#{@slugs[anchor]}" if @slugs[anchor] > 1
node['id'] = anchor
next unless LISTED_ELEMENTS.include?(node.name)
depth = node.name[1..-1]
latest_section = @headers.last
if latest_section.nil? || latest_section.depth >= depth
@headers << Section.new(depth, node.text, anchor)
else
latest_section << Section.new(depth, node.text, anchor)
end
end
@target_html = parsed_html.to_s
@processed = true
end
end

15
app/lib/user_settings_decorator.rb
View File

@ -35,6 +35,9 @@ class UserSettingsDecorator
user.settings['aggregate_reblogs'] = aggregate_reblogs_preference if change?('setting_aggregate_reblogs')
user.settings['show_application'] = show_application_preference if change?('setting_show_application')
user.settings['advanced_layout'] = advanced_layout_preference if change?('setting_advanced_layout')
user.settings['use_blurhash'] = use_blurhash_preference if change?('setting_use_blurhash')
user.settings['use_pending_items'] = use_pending_items_preference if change?('setting_use_pending_items')
user.settings['trends'] = trends_preference if change?('setting_trends')
end
def merged_notification_emails
@ -117,6 +120,18 @@ class UserSettingsDecorator
boolean_cast_setting 'setting_advanced_layout'
end
def use_blurhash_preference
boolean_cast_setting 'setting_use_blurhash'
end
def use_pending_items_preference
boolean_cast_setting 'setting_use_pending_items'
end
def trends_preference
boolean_cast_setting 'setting_trends'
end
def boolean_cast_setting(key)
ActiveModel::Type::Boolean.new.cast(settings[key])
end

6
app/lib/webfinger_resource.rb
View File

@ -23,11 +23,17 @@ class WebfingerResource
def username_from_url
if account_show_page?
path_params[:username]
elsif instance_actor_page?
Rails.configuration.x.local_domain
else
raise ActiveRecord::RecordNotFound
end
end
def instance_actor_page?
path_params[:controller] == 'instance_actors'
end
def account_show_page?
path_params[:controller] == 'accounts' && path_params[:action] == 'show'
end