Merge tag 'v3.1.4' into hometown-dev

app/services/account_search_service.rb

@@ -171,7 +171,7 @@ class AccountSearchService < BaseService
   end
 
   def username_complete?
-    query.include?('@') && "@#{query}" =~ Account::MENTION_RE
+    query.include?('@') && "@#{query}" =~ /\A#{Account::MENTION_RE}\Z/
   end
 
   def likely_acct?

app/services/activitypub/fetch_remote_account_service.rb

@@ -3,6 +3,7 @@
 class ActivityPub::FetchRemoteAccountService < BaseService
   include JsonLdHelper
   include DomainControlHelper
+  include WebfingerHelper
 
   SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze
 
@@ -35,12 +36,12 @@ class ActivityPub::FetchRemoteAccountService < BaseService
   private
 
   def verified_webfinger?
-    webfinger                            = Goldfinger.finger("acct:#{@username}@#{@domain}")
+    webfinger                            = webfinger!("acct:#{@username}@#{@domain}")
     confirmed_username, confirmed_domain = split_acct(webfinger.subject)
 
     return webfinger.link('self')&.href == @uri if @username.casecmp(confirmed_username).zero? && @domain.casecmp(confirmed_domain).zero?
 
-    webfinger                            = Goldfinger.finger("acct:#{confirmed_username}@#{confirmed_domain}")
+    webfinger                            = webfinger!("acct:#{confirmed_username}@#{confirmed_domain}")
     @username, @domain                   = split_acct(webfinger.subject)
     self_reference                       = webfinger.link('self')
 

app/services/activitypub/process_account_service.rb

@@ -96,6 +96,7 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.statuses_count    = outbox_total_items    if outbox_total_items.present?
     @account.following_count   = following_total_items if following_total_items.present?
     @account.followers_count   = followers_total_items if followers_total_items.present?
+    @account.hide_collections  = following_private? || followers_private?
     @account.moved_to_account  = @json['movedTo'].present? ? moved_account : nil
   end
 
@@ -168,26 +169,36 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def outbox_total_items
-    collection_total_items('outbox')
+    collection_info('outbox').first
   end
 
   def following_total_items
-    collection_total_items('following')
+    collection_info('following').first
   end
 
   def followers_total_items
-    collection_total_items('followers')
+    collection_info('followers').first
   end
 
-  def collection_total_items(type)
-    return if @json[type].blank?
+  def following_private?
+    !collection_info('following').last
+  end
+
+  def followers_private?
+    !collection_info('followers').last
+  end
+
+  def collection_info(type)
+    return [nil, nil] if @json[type].blank?
     return @collections[type] if @collections.key?(type)
 
     collection = fetch_resource_without_id_validation(@json[type])
 
-    @collections[type] = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil
+    total_items = collection.is_a?(Hash) && collection['totalItems'].present? && collection['totalItems'].is_a?(Numeric) ? collection['totalItems'] : nil
+    has_first_page = collection.is_a?(Hash) && collection['first'].present?
+    @collections[type] = [total_items, has_first_page]
   rescue HTTP::Error, OpenSSL::SSL::SSLError
-    @collections[type] = nil
+    @collections[type] = [nil, nil]
   end
 
   def moved_account

app/services/batched_remove_status_service.rb

@@ -72,11 +72,18 @@ class BatchedRemoveStatusService < BaseService
 
     redis.pipelined do
       redis.publish('timeline:public', payload)
-      redis.publish('timeline:public:local', payload) if status.local?
-
+      if status.local?
+        redis.publish('timeline:public:local', payload)
+      else
+        redis.publish('timeline:public:remote', payload)
+      end
       if status.media_attachments.any?
         redis.publish('timeline:public:media', payload)
-        redis.publish('timeline:public:local:media', payload) if status.local?
+        if status.local?
+          redis.publish('timeline:public:local:media', payload)
+        else
+          redis.publish('timeline:public:remote:media', payload)
+        end
       end
 
       @tags[status.id].each do |hashtag|

app/services/fan_out_on_write_service.rb

@@ -81,14 +81,22 @@ class FanOutOnWriteService < BaseService
     Rails.logger.debug "Delivering status #{status.id} to public timeline"
 
     Redis.current.publish('timeline:public', @payload)
-    Redis.current.publish('timeline:public:local', @payload) if status.local?
+    if status.local?
+      Redis.current.publish('timeline:public:local', @payload)
+    else
+      Redis.current.publish('timeline:public:remote', @payload)
+    end
   end
 
   def deliver_to_media(status)
     Rails.logger.debug "Delivering status #{status.id} to media timeline"
 
     Redis.current.publish('timeline:public:media', @payload)
-    Redis.current.publish('timeline:public:local:media', @payload) if status.local?
+    if status.local?
+      Redis.current.publish('timeline:public:local:media', @payload)
+    else
+      Redis.current.publish('timeline:public:remote:media', @payload)
+    end
   end
 
   def deliver_to_own_conversation(status)

app/services/fetch_resource_service.rb

@@ -5,6 +5,8 @@ class FetchResourceService < BaseService
 
   ACCEPT_HEADER = 'application/activity+json, application/ld+json; profile="https://www.w3.org/ns/activitystreams", text/html;q=0.1'
 
+  attr_reader :response_code
+
   def call(url)
     return if url.blank?
 
@@ -23,10 +25,22 @@ class FetchResourceService < BaseService
   end
 
   def perform_request(&block)
-    Request.new(:get, @url).add_headers('Accept' => ACCEPT_HEADER).on_behalf_of(Account.representative).perform(&block)
+    Request.new(:get, @url).tap do |request|
+      request.add_headers('Accept' => ACCEPT_HEADER)
+
+      # In a real setting we want to sign all outgoing requests,
+      # in case the remote server has secure mode enabled and requires
+      # authentication on all resources. However, during development,
+      # sending request signatures with an inaccessible host is useless
+      # and prevents even public resources from being fetched, so
+      # don't do it
+
+      request.on_behalf_of(Account.representative) unless Rails.env.development?
+    end.perform(&block)
   end
 
   def process_response(response, terminal = false)
+    @response_code = response.code
     return nil if response.code != 200
 
     if ['application/activity+json', 'application/ld+json'].include?(response.mime_type)

app/services/follow_service.rb

@@ -7,54 +7,68 @@ class FollowService < BaseService
   # Follow a remote user, notify remote user about the follow
   # @param [Account] source_account From which to follow
   # @param [String, Account] uri User URI to follow in the form of username@domain (or account record)
-  # @param [true, false, nil] reblogs Whether or not to show reblogs, defaults to true
-  def call(source_account, target_account, reblogs: nil, bypass_locked: false)
-    reblogs = true if reblogs.nil?
-    target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true)
+  # @param [Hash] options
+  # @option [Boolean] :reblogs Whether or not to show reblogs, defaults to true
+  # @option [Boolean] :bypass_locked
+  # @option [Boolean] :with_rate_limit
+  def call(source_account, target_account, options = {})
+    @source_account = source_account
+    @target_account = ResolveAccountService.new.call(target_account, skip_webfinger: true)
+    @options        = { reblogs: true, bypass_locked: false, with_rate_limit: false }.merge(options)
 
-    raise ActiveRecord::RecordNotFound if target_account.nil? || target_account.id == source_account.id || target_account.suspended?
-    raise Mastodon::NotPermittedError  if target_account.blocking?(source_account) || source_account.blocking?(target_account) || target_account.moved? || (!target_account.local? && target_account.ostatus?) || source_account.domain_blocking?(target_account.domain)
+    raise ActiveRecord::RecordNotFound if following_not_possible?
+    raise Mastodon::NotPermittedError  if following_not_allowed?
 
-    if source_account.following?(target_account)
-      # We're already following this account, but we'll call follow! again to
-      # make sure the reblogs status is set correctly.
-      return source_account.follow!(target_account, reblogs: reblogs)
-    elsif source_account.requested?(target_account)
-      # This isn't managed by a method in AccountInteractions, so we modify it
-      # ourselves if necessary.
-      req = source_account.follow_requests.find_by(target_account: target_account)
-      req.update!(show_reblogs: reblogs)
-      return req
+    if @source_account.following?(@target_account)
+      return change_follow_options!
+    elsif @source_account.requested?(@target_account)
+      return change_follow_request_options!
     end
 
     ActivityTracker.increment('activity:interactions')
 
-    if (target_account.locked? && !bypass_locked) || source_account.silenced? || target_account.activitypub?
-      request_follow(source_account, target_account, reblogs: reblogs)
-    elsif target_account.local?
-      direct_follow(source_account, target_account, reblogs: reblogs)
+    if (@target_account.locked? && !@options[:bypass_locked]) || @source_account.silenced? || @target_account.activitypub?
+      request_follow!
+    elsif @target_account.local?
+      direct_follow!
     end
   end
 
   private
 
-  def request_follow(source_account, target_account, reblogs: true)
-    follow_request = FollowRequest.create!(account: source_account, target_account: target_account, show_reblogs: reblogs)
+  def following_not_possible?
+    @target_account.nil? || @target_account.id == @source_account.id || @target_account.suspended?
+  end
 
-    if target_account.local?
-      LocalNotificationWorker.perform_async(target_account.id, follow_request.id, follow_request.class.name)
-    elsif target_account.activitypub?
-      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), source_account.id, target_account.inbox_url)
+  def following_not_allowed?
+    @target_account.blocking?(@source_account) || @source_account.blocking?(@target_account) || @target_account.moved? || (!@target_account.local? && @target_account.ostatus?) || @source_account.domain_blocking?(@target_account.domain)
+  end
+
+  def change_follow_options!
+    @source_account.follow!(@target_account, reblogs: @options[:reblogs])
+  end
+
+  def change_follow_request_options!
+    @source_account.request_follow!(@target_account, reblogs: @options[:reblogs])
+  end
+
+  def request_follow!
+    follow_request = @source_account.request_follow!(@target_account, reblogs: @options[:reblogs], rate_limit: @options[:with_rate_limit])
+
+    if @target_account.local?
+      LocalNotificationWorker.perform_async(@target_account.id, follow_request.id, follow_request.class.name)
+    elsif @target_account.activitypub?
+      ActivityPub::DeliveryWorker.perform_async(build_json(follow_request), @source_account.id, @target_account.inbox_url)
     end
 
     follow_request
   end
 
-  def direct_follow(source_account, target_account, reblogs: true)
-    follow = source_account.follow!(target_account, reblogs: reblogs)
+  def direct_follow!
+    follow = @source_account.follow!(@target_account, reblogs: @options[:reblogs], rate_limit: @options[:with_rate_limit])
 
-    LocalNotificationWorker.perform_async(target_account.id, follow.id, follow.class.name)
-    MergeWorker.perform_async(target_account.id, source_account.id)
+    LocalNotificationWorker.perform_async(@target_account.id, follow.id, follow.class.name)
+    MergeWorker.perform_async(@target_account.id, @source_account.id)
 
     follow
   end

app/services/import_service.rb

@@ -64,7 +64,8 @@ class ImportService < BaseService
   end
 
   def import_relationships!(action, undo_action, overwrite_scope, limit, extra_fields = {})
-    items = @data.take(limit).map { |row| [row['Account address']&.strip, Hash[extra_fields.map { |key, header| [key, row[header]&.strip] }]] }.reject { |(id, _)| id.blank? }
+    local_domain_suffix = "@#{Rails.configuration.x.local_domain}"
+    items = @data.take(limit).map { |row| [row['Account address']&.strip&.delete_suffix(local_domain_suffix), Hash[extra_fields.map { |key, header| [key, row[header]&.strip] }]] }.reject { |(id, _)| id.blank? }
 
     if @import.overwrite?
       presence_hash = items.each_with_object({}) { |(id, extra), mapping| mapping[id] = [true, extra] }

app/services/post_status_service.rb

@@ -19,6 +19,7 @@ class PostStatusService < BaseService
   # @option [Enumerable] :media_ids Optional array of media IDs to attach
   # @option [Doorkeeper::Application] :application
   # @option [String] :idempotency Optional idempotency key
+  # @option [Boolean] :with_rate_limit
   # @return [Status]
   def call(account, options = {})
     @account     = account
@@ -47,9 +48,10 @@ class PostStatusService < BaseService
   private
 
   def preprocess_attributes!
+    @sensitive    = (@options[:sensitive].nil? ? @account.user&.setting_default_sensitive : @options[:sensitive]) || @options[:spoiler_text].present?
     @text         = @options.delete(:spoiler_text) if @text.blank? && @options[:spoiler_text].present?
     @visibility   = @options[:visibility] || @account.user&.setting_default_privacy
-    @visibility   = :unlisted if @visibility == :public && @account.silenced?
+    @visibility   = :unlisted if @visibility&.to_sym == :public && @account.silenced?
     @scheduled_at = @options[:scheduled_at]&.to_datetime
     @scheduled_at = nil if scheduled_in_the_past?
   rescue ArgumentError
@@ -108,6 +110,7 @@ class PostStatusService < BaseService
     @media = @account.media_attachments.where(status_id: nil).where(id: @options[:media_ids].take(4).map(&:to_i))
 
     raise Mastodon::ValidationError, I18n.t('media_attachments.validations.images_and_video') if @media.size > 1 && @media.find(&:audio_or_video?)
+    raise Mastodon::ValidationError, I18n.t('media_attachments.validations.not_ready') if @media.any?(&:not_processed?)
   end
 
   def language_from_option(str)
@@ -163,12 +166,13 @@ class PostStatusService < BaseService
       media_attachments: @media || [],
       thread: @in_reply_to,
       poll_attributes: poll_attributes,
-      sensitive: (@options[:sensitive].nil? ? @account.user&.setting_default_sensitive : @options[:sensitive]) || @options[:spoiler_text].present?,
+      sensitive: @sensitive,
       spoiler_text: @options[:spoiler_text] || '',
       visibility: @visibility,
       language: language_from_option(@options[:language]) || @account.user&.setting_default_language&.presence || LanguageDetector.instance.detect(@text, @account),
       application: @options[:application],
       local_only: local_only_option(@options[:local_only], @in_reply_to, @account.user&.setting_default_federation),
+      rate_limit: @options[:with_rate_limit],
     }.compact
   end
 
@@ -188,10 +192,11 @@ class PostStatusService < BaseService
 
   def scheduled_options
     @options.tap do |options_hash|
-      options_hash[:in_reply_to_id] = options_hash.delete(:thread)&.id
-      options_hash[:application_id] = options_hash.delete(:application)&.id
-      options_hash[:scheduled_at]   = nil
-      options_hash[:idempotency]    = nil
+      options_hash[:in_reply_to_id]  = options_hash.delete(:thread)&.id
+      options_hash[:application_id]  = options_hash.delete(:application)&.id
+      options_hash[:scheduled_at]    = nil
+      options_hash[:idempotency]     = nil
+      options_hash[:with_rate_limit] = false
     end
   end
 end

app/services/reblog_service.rb

@@ -8,6 +8,8 @@ class ReblogService < BaseService
   # @param [Account] account Account to reblog from
   # @param [Status] reblogged_status Status to be reblogged
   # @param [Hash] options
+  # @option [String]  :visibility
+  # @option [Boolean] :with_rate_limit
   # @return [Status]
   def call(account, reblogged_status, options = {})
     reblogged_status = reblogged_status.reblog if reblogged_status.reblog?
@@ -18,9 +20,15 @@ class ReblogService < BaseService
 
     return reblog unless reblog.nil?
 
-    visibility = options[:visibility] || account.user&.setting_default_privacy
-    visibility = reblogged_status.visibility if reblogged_status.hidden?
-    reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility)
+    visibility = begin
+      if reblogged_status.hidden?
+        reblogged_status.visibility
+      else
+        options[:visibility] || account.user&.setting_default_privacy
+      end
+    end
+
+    reblog = account.statuses.create!(reblog: reblogged_status, text: '', visibility: visibility, rate_limit: options[:with_rate_limit])
 
     DistributionWorker.perform_async(reblog.id)
     unless reblogged_status.local_only?
@@ -47,7 +55,9 @@ class ReblogService < BaseService
 
   def bump_potential_friendship(account, reblog)
     ActivityTracker.increment('activity:interactions')
+
     return if account.following?(reblog.reblog.account_id)
+
     PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog)
   end
 

app/services/remove_status_service.rb

@@ -140,14 +140,22 @@ class RemoveStatusService < BaseService
     return unless @status.public_visibility?
 
     redis.publish('timeline:public', @payload)
-    redis.publish('timeline:public:local', @payload) if @status.local?
+    if @status.local?
+      redis.publish('timeline:public:local', @payload)
+    else
+      redis.publish('timeline:public:remote', @payload)
+    end
   end
 
   def remove_from_media
     return unless @status.public_visibility?
 
     redis.publish('timeline:public:media', @payload)
-    redis.publish('timeline:public:local:media', @payload) if @status.local?
+    if @status.local?
+      redis.publish('timeline:public:local:media', @payload)
+    else
+      redis.publish('timeline:public:remote:media', @payload)
+    end
   end
 
   def remove_media

app/services/resolve_account_service.rb

@@ -3,6 +3,7 @@
 class ResolveAccountService < BaseService
   include JsonLdHelper
   include DomainControlHelper
+  include WebfingerHelper
 
   class WebfingerRedirectError < StandardError; end
 
@@ -76,7 +77,7 @@ class ResolveAccountService < BaseService
   end
 
   def process_webfinger!(uri, redirected = false)
-    @webfinger                           = Goldfinger.finger("acct:#{uri}")
+    @webfinger                           = webfinger!("acct:#{uri}")
     confirmed_username, confirmed_domain = @webfinger.subject.gsub(/\Aacct:/, '').split('@')
 
     if confirmed_username.casecmp(@username).zero? && confirmed_domain.casecmp(@domain).zero?

app/services/resolve_url_service.rb

@@ -12,7 +12,7 @@ class ResolveURLService < BaseService
       process_local_url
     elsif !fetched_resource.nil?
       process_url
-    elsif @on_behalf_of.present?
+    else
       process_url_from_db
     end
   end
@@ -30,6 +30,8 @@ class ResolveURLService < BaseService
   end
 
   def process_url_from_db
+    return unless @on_behalf_of.present? && [401, 403, 404].include?(fetch_resource_service.response_code)
+
     # It may happen that the resource is a private toot, and thus not fetchable,
     # but we can return the toot if we already know about it.
     status = Status.find_by(uri: @url) || Status.find_by(url: @url)
@@ -40,7 +42,11 @@ class ResolveURLService < BaseService
   end
 
   def fetched_resource
-    @fetched_resource ||= FetchResourceService.new.call(@url)
+    @fetched_resource ||= fetch_resource_service.call(@url)
+  end
+
+  def fetch_resource_service
+    @_fetch_resource_service ||= FetchResourceService.new
   end
 
   def resource_url

app/services/search_service.rb

@@ -10,10 +10,10 @@ class SearchService < BaseService
     @resolve = options[:resolve] || false
 
     default_results.tap do |results|
-      next if @query.blank?
+      next if @query.blank? || @limit.zero?
 
       if url_query?
-        results.merge!(url_resource_results) unless url_resource.nil? || (@options[:type].present? && url_resource_symbol != @options[:type].to_sym)
+        results.merge!(url_resource_results) unless url_resource.nil? || @offset.positive? || (@options[:type].present? && url_resource_symbol != @options[:type].to_sym)
       elsif @query.present?
         results[:accounts] = perform_accounts_search! if account_searchable?
         results[:statuses] = perform_statuses_search! if full_text_searchable?