Merge tag 'v3.2.0' into hometown-dev

app/services/account_search_service.rb

@@ -27,7 +27,7 @@ class AccountSearchService < BaseService
 
     return @exact_match if defined?(@exact_match)
 
-    @exact_match = begin
+    match = begin
       if options[:resolve]
         ResolveAccountService.new.call(query)
       elsif domain_is_local?
@@ -36,6 +36,10 @@ class AccountSearchService < BaseService
         Account.find_remote(query_username, query_domain)
       end
     end
+
+    match = nil if !match.nil? && !account.nil? && options[:following] && !account.following?(match)
+
+    @exact_match = match
   end
 
   def search_results

app/services/activitypub/process_account_service.rb

@@ -78,6 +78,7 @@ class ActivityPub::ProcessAccountService < BaseService
     @account.shared_inbox_url        = (@json['endpoints'].is_a?(Hash) ? @json['endpoints']['sharedInbox'] : @json['sharedInbox']) || ''
     @account.followers_url           = @json['followers'] || ''
     @account.featured_collection_url = @json['featured'] || ''
+    @account.devices_url             = @json['devices'] || ''
     @account.url                     = url || @uri
     @account.uri                     = @uri
     @account.display_name            = @json['name'] || ''
@@ -90,8 +91,8 @@ class ActivityPub::ProcessAccountService < BaseService
   end
 
   def set_fetchable_attributes!
-    @account.avatar_remote_url = image_url('icon')  unless skip_download?
-    @account.header_remote_url = image_url('image') unless skip_download?
+    @account.avatar_remote_url = image_url('icon')  || '' unless skip_download?
+    @account.header_remote_url = image_url('image') || '' unless skip_download?
     @account.public_key        = public_key || ''
     @account.statuses_count    = outbox_total_items    if outbox_total_items.present?
     @account.following_count   = following_total_items if following_total_items.present?

app/services/after_block_domain_from_account_service.rb

@@ -19,7 +19,7 @@ class AfterBlockDomainFromAccountService < BaseService
   private
 
   def remove_follows!
-    @account.active_relationships.where(account: Account.where(domain: @domain)).includes(:target_account).reorder(nil).find_each do |follow|
+    @account.active_relationships.where(target_account: Account.where(domain: @domain)).includes(:target_account).reorder(nil).find_each do |follow|
       UnfollowService.new.call(@account, follow.target_account)
     end
   end

app/services/after_unallow_domain_service.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AfterUnallowDomainService < BaseService
+  def call(domain)
+    Account.where(domain: domain).find_each do |account|
+      SuspendAccountService.new.call(account, reserve_username: false)
+    end
+  end
+end

app/services/backup_service.rb

@@ -22,7 +22,7 @@ class BackupService < BaseService
 
     account.statuses.with_includes.reorder(nil).find_in_batches do |statuses|
       statuses.each do |status|
-        item = serialize_payload(status, ActivityPub::ActivitySerializer, signer: @account)
+        item = serialize_payload(ActivityPub::ActivityPresenter.from_status(status), ActivityPub::ActivitySerializer, signer: @account)
         item.delete(:'@context')
 
         unless item[:type] == 'Announce' || item[:object][:attachment].blank?

app/services/block_domain_service.rb

@@ -26,59 +26,20 @@ class BlockDomainService < BaseService
       suspend_accounts!
     end
 
-    clear_media! if domain_block.reject_media?
-  end
-
-  def invalidate_association_caches!
-    # Normally, associated models of a status are immutable (except for accounts)
-    # so they are aggressively cached. After updating the media attachments to no
-    # longer point to a local file, we need to clear the cache to make those
-    # changes appear in the API and UI
-    @affected_status_ids.each { |id| Rails.cache.delete_matched("statuses/#{id}-*") }
+    DomainClearMediaWorker.perform_async(domain_block.id) if domain_block.reject_media?
   end
 
   def silence_accounts!
     blocked_domain_accounts.without_silenced.in_batches.update_all(silenced_at: @domain_block.created_at)
   end
 
-  def clear_media!
-    @affected_status_ids = []
-
-    clear_account_images!
-    clear_account_attachments!
-    clear_emojos!
-
-    invalidate_association_caches!
-  end
-
   def suspend_accounts!
-    blocked_domain_accounts.without_suspended.reorder(nil).find_each do |account|
+    blocked_domain_accounts.without_suspended.in_batches.update_all(suspended_at: @domain_block.created_at)
+    blocked_domain_accounts.where(suspended_at: @domain_block.created_at).reorder(nil).find_each do |account|
       SuspendAccountService.new.call(account, reserve_username: true, suspended_at: @domain_block.created_at)
     end
   end
 
-  def clear_account_images!
-    blocked_domain_accounts.reorder(nil).find_each do |account|
-      account.avatar.destroy if account.avatar.exists?
-      account.header.destroy if account.header.exists?
-      account.save
-    end
-  end
-
-  def clear_account_attachments!
-    media_from_blocked_domain.reorder(nil).find_each do |attachment|
-      @affected_status_ids << attachment.status_id if attachment.status_id.present?
-
-      attachment.file.destroy if attachment.file.exists?
-      attachment.type = :unknown
-      attachment.save
-    end
-  end
-
-  def clear_emojos!
-    emojis_from_blocked_domains.destroy_all
-  end
-
   def blocked_domain
     domain_block.domain
   end
@@ -86,12 +47,4 @@ class BlockDomainService < BaseService
   def blocked_domain_accounts
     Account.by_domain_and_subdomains(blocked_domain)
   end
-
-  def media_from_blocked_domain
-    MediaAttachment.joins(:account).merge(blocked_domain_accounts).reorder(nil)
-  end
-
-  def emojis_from_blocked_domains
-    CustomEmoji.by_domain_and_subdomains(blocked_domain)
-  end
 end

app/services/clear_domain_media_service.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+class ClearDomainMediaService < BaseService
+  attr_reader :domain_block
+
+  def call(domain_block)
+    @domain_block = domain_block
+    clear_media! if domain_block.reject_media?
+  end
+
+  private
+
+  def invalidate_association_caches!
+    # Normally, associated models of a status are immutable (except for accounts)
+    # so they are aggressively cached. After updating the media attachments to no
+    # longer point to a local file, we need to clear the cache to make those
+    # changes appear in the API and UI
+    @affected_status_ids.each { |id| Rails.cache.delete_matched("statuses/#{id}-*") }
+  end
+
+  def clear_media!
+    @affected_status_ids = []
+
+    begin
+      clear_account_images!
+      clear_account_attachments!
+      clear_emojos!
+    ensure
+      invalidate_association_caches!
+    end
+  end
+
+  def clear_account_images!
+    blocked_domain_accounts.reorder(nil).find_each do |account|
+      account.avatar.destroy if account.avatar&.exists?
+      account.header.destroy if account.header&.exists?
+      account.save
+    end
+  end
+
+  def clear_account_attachments!
+    media_from_blocked_domain.reorder(nil).find_each do |attachment|
+      @affected_status_ids << attachment.status_id if attachment.status_id.present?
+
+      attachment.file.destroy if attachment.file&.exists?
+      attachment.type = :unknown
+      attachment.save
+    end
+  end
+
+  def clear_emojos!
+    emojis_from_blocked_domains.destroy_all
+  end
+
+  def blocked_domain
+    domain_block.domain
+  end
+
+  def blocked_domain_accounts
+    Account.by_domain_and_subdomains(blocked_domain)
+  end
+
+  def media_from_blocked_domain
+    MediaAttachment.joins(:account).merge(blocked_domain_accounts).reorder(nil)
+  end
+
+  def emojis_from_blocked_domains
+    CustomEmoji.by_domain_and_subdomains(blocked_domain)
+  end
+end

app/services/concerns/payloadable.rb

@@ -5,8 +5,9 @@ module Payloadable
     signer    = options.delete(:signer)
     sign_with = options.delete(:sign_with)
     payload   = ActiveModelSerializers::SerializableResource.new(record, options.merge(serializer: serializer, adapter: ActivityPub::Adapter)).as_json
+    object    = record.respond_to?(:virtual_object) ? record.virtual_object : record
 
-    if (record.respond_to?(:sign?) && record.sign?) && signer && signing_enabled?
+    if (object.respond_to?(:sign?) && object.sign?) && signer && signing_enabled?
       ActivityPub::LinkedDataSignature.new(payload).sign!(signer, sign_with: sign_with)
     else
       payload

app/services/deliver_to_device_service.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+class DeliverToDeviceService < BaseService
+  include Payloadable
+
+  class EncryptedMessage < ActiveModelSerializers::Model
+    attributes :source_account, :target_account, :source_device,
+               :target_device_id, :type, :body, :digest,
+               :message_franking
+  end
+
+  def call(source_account, source_device, options = {})
+    @source_account   = source_account
+    @source_device    = source_device
+    @target_account   = Account.find(options[:account_id])
+    @target_device_id = options[:device_id]
+    @body             = options[:body]
+    @type             = options[:type]
+    @hmac             = options[:hmac]
+
+    set_message_franking!
+
+    if @target_account.local?
+      deliver_to_local!
+    else
+      deliver_to_remote!
+    end
+  end
+
+  private
+
+  def set_message_franking!
+    @message_franking = message_franking.to_token
+  end
+
+  def deliver_to_local!
+    target_device = @target_account.devices.find_by!(device_id: @target_device_id)
+
+    target_device.encrypted_messages.create!(
+      from_account: @source_account,
+      from_device_id: @source_device.device_id,
+      type: @type,
+      body: @body,
+      digest: @hmac,
+      message_franking: @message_franking
+    )
+  end
+
+  def deliver_to_remote!
+    ActivityPub::DeliveryWorker.perform_async(
+      Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_encrypted_message(encrypted_message), ActivityPub::ActivitySerializer)),
+      @source_account.id,
+      @target_account.inbox_url
+    )
+  end
+
+  def message_franking
+    MessageFranking.new(
+      source_account_id: @source_account.id,
+      target_account_id: @target_account.id,
+      hmac: @hmac,
+      timestamp: Time.now.utc
+    )
+  end
+
+  def encrypted_message
+    EncryptedMessage.new(
+      source_account: @source_account,
+      target_account: @target_account,
+      source_device: @source_device,
+      target_device_id: @target_device_id,
+      type: @type,
+      body: @body,
+      digest: @hmac,
+      message_franking: @message_franking
+    )
+  end
+end

app/services/fetch_link_card_service.rb

@@ -45,7 +45,7 @@ class FetchLinkCardService < BaseService
   def html
     return @html if defined?(@html)
 
-    Request.new(:get, @url).add_headers('Accept' => 'text/html').perform do |res|
+    Request.new(:get, @url).add_headers('Accept' => 'text/html', 'User-Agent' => Mastodon::Version.user_agent + ' Bot').perform do |res|
       if res.code == 200 && res.mime_type == 'text/html'
         @html = res.body_with_limit
         @html_charset = res.charset

app/services/import_service.rb

@@ -81,7 +81,9 @@ class ImportService < BaseService
       end
     end
 
-    Import::RelationshipWorker.push_bulk(items) do |acct, extra|
+    head_items = items.uniq { |acct, _| acct.split('@')[1] }
+    tail_items = items - head_items
+    Import::RelationshipWorker.push_bulk(head_items + tail_items) do |acct, extra|
       [@account.id, acct, action, extra]
     end
   end

app/services/keys/claim_service.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+class Keys::ClaimService < BaseService
+  HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze
+
+  class Result < ActiveModelSerializers::Model
+    attributes :account, :device_id, :key_id,
+               :key, :signature
+
+    def initialize(account, device_id, key_attributes = {})
+      @account   = account
+      @device_id = device_id
+      @key_id    = key_attributes[:key_id]
+      @key       = key_attributes[:key]
+      @signature = key_attributes[:signature]
+    end
+  end
+
+  def call(source_account, target_account_id, device_id)
+    @source_account = source_account
+    @target_account = Account.find(target_account_id)
+    @device_id      = device_id
+
+    if @target_account.local?
+      claim_local_key!
+    else
+      claim_remote_key!
+    end
+  rescue ActiveRecord::RecordNotFound
+    nil
+  end
+
+  private
+
+  def claim_local_key!
+    device = @target_account.devices.find_by(device_id: @device_id)
+    key    = nil
+
+    ApplicationRecord.transaction do
+      key = device.one_time_keys.order(Arel.sql('random()')).first!
+      key.destroy!
+    end
+
+    @result = Result.new(@target_account, @device_id, key)
+  end
+
+  def claim_remote_key!
+    query_result = QueryService.new.call(@target_account)
+    device       = query_result.find(@device_id)
+
+    return unless device.present? && device.valid_claim_url?
+
+    json = fetch_resource_with_post(device.claim_url)
+
+    return unless json.present? && json['publicKeyBase64'].present?
+
+    @result = Result.new(@target_account, @device_id, key_id: json['id'], key: json['publicKeyBase64'], signature: json.dig('signature', 'signatureValue'))
+  rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error => e
+    Rails.logger.debug "Claiming one-time key for #{@target_account.acct}:#{@device_id} failed: #{e}"
+    nil
+  end
+
+  def fetch_resource_with_post(uri)
+    build_post_request(uri).perform do |response|
+      raise Mastodon::UnexpectedResponseError, response unless response_successful?(response) || response_error_unsalvageable?(response)
+
+      body_to_json(response.body_with_limit) if response.code == 200
+    end
+  end
+
+  def build_post_request(uri)
+    Request.new(:post, uri).tap do |request|
+      request.on_behalf_of(@source_account, :uri)
+      request.add_headers(HEADERS)
+    end
+  end
+end

app/services/keys/query_service.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+class Keys::QueryService < BaseService
+  include JsonLdHelper
+
+  class Result < ActiveModelSerializers::Model
+    attributes :account, :devices
+
+    def initialize(account, devices)
+      @account = account
+      @devices = devices || []
+    end
+
+    def find(device_id)
+      @devices.find { |device| device.device_id == device_id }
+    end
+  end
+
+  class Device < ActiveModelSerializers::Model
+    attributes :device_id, :name, :identity_key, :fingerprint_key
+
+    def initialize(attributes = {})
+      @device_id       = attributes[:device_id]
+      @name            = attributes[:name]
+      @identity_key    = attributes[:identity_key]
+      @fingerprint_key = attributes[:fingerprint_key]
+      @claim_url       = attributes[:claim_url]
+    end
+
+    def valid_claim_url?
+      return false if @claim_url.blank?
+
+      begin
+        parsed_url = Addressable::URI.parse(@claim_url).normalize
+      rescue Addressable::URI::InvalidURIError
+        return false
+      end
+
+      %w(http https).include?(parsed_url.scheme) && parsed_url.host.present?
+    end
+  end
+
+  def call(account)
+    @account = account
+
+    if @account.local?
+      query_local_devices!
+    else
+      query_remote_devices!
+    end
+
+    Result.new(@account, @devices)
+  end
+
+  private
+
+  def query_local_devices!
+    @devices = @account.devices.map { |device| Device.new(device) }
+  end
+
+  def query_remote_devices!
+    return if @account.devices_url.blank?
+
+    json = fetch_resource(@account.devices_url)
+
+    return if json['items'].blank?
+
+    @devices = json['items'].map do |device|
+      Device.new(device_id: device['id'], name: device['name'], identity_key: device.dig('identityKey', 'publicKeyBase64'), fingerprint_key: device.dig('fingerprintKey', 'publicKeyBase64'), claim_url: device['claim'])
+    end
+  rescue HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error => e
+    Rails.logger.debug "Querying devices for #{@account.acct} failed: #{e}"
+    nil
+  end
+end

app/services/process_mentions_service.rb

@@ -36,7 +36,8 @@ class ProcessMentionsService < BaseService
 
       next match if mention_undeliverable?(mentioned_account) || mentioned_account&.suspended?
 
-      mentions << mentioned_account.mentions.where(status: status).first_or_create(status: status)
+      mention = mentioned_account.mentions.new(status: status)
+      mentions << mention if mention.save
 
       "@#{mentioned_account.acct}"
     end
@@ -65,7 +66,7 @@ class ProcessMentionsService < BaseService
 
   def activitypub_json
     return @activitypub_json if defined?(@activitypub_json)
-    @activitypub_json = Oj.dump(serialize_payload(@status, ActivityPub::ActivitySerializer, signer: @status.account))
+    @activitypub_json = Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(@status), ActivityPub::ActivitySerializer, signer: @status.account))
   end
 
   def resolve_account_service

app/services/reblog_service.rb

@@ -62,6 +62,6 @@ class ReblogService < BaseService
   end
 
   def build_json(reblog)
-    Oj.dump(serialize_payload(reblog, ActivityPub::ActivitySerializer, signer: reblog.account))
+    Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account))
   end
 end

app/services/resolve_account_service.rb

@@ -112,6 +112,8 @@ class ResolveAccountService < BaseService
   end
 
   def webfinger_update_due?
+    return false if @options[:check_delivery_availability] && !DeliveryFailureTracker.available?(@domain)
+
     @account.nil? || ((!@options[:skip_webfinger] || @account.ostatus?) && @account.possibly_stale?)
   end
 

app/services/search_service.rb

@@ -94,7 +94,7 @@ class SearchService < BaseService
   end
 
   def account_searchable?
-    account_search? && !(@query.include?('@') && @query.include?(' '))
+    account_search? && !(@query.start_with?('#') || (@query.include?('@') && @query.include?(' ')))
   end
 
   def hashtag_searchable?

app/services/unallow_domain_service.rb

@@ -1,11 +1,18 @@
 # frozen_string_literal: true
 
 class UnallowDomainService < BaseService
+  include DomainControlHelper
+
   def call(domain_allow)
-    Account.where(domain: domain_allow.domain).find_each do |account|
-      SuspendAccountService.new.call(account, reserve_username: false)
-    end
+    suspend_accounts!(domain_allow.domain) if whitelist_mode?
 
     domain_allow.destroy
   end
+
+  private
+
+  def suspend_accounts!(domain)
+    Account.where(domain: domain).in_batches.update_all(suspended_at: Time.now.utc)
+    AfterUnallowDomainWorker.perform_async(domain)
+  end
 end

app/services/update_account_service.rb

@@ -12,7 +12,7 @@ class UpdateAccountService < BaseService
       check_links(account)
       process_hashtags(account)
     end
-  rescue Mastodon::DimensionsValidationError => de
+  rescue Mastodon::DimensionsValidationError, Mastodon::StreamValidationError => de
     account.errors.add(:avatar, de.message)
     false
   end