Merge tag 'v3.4.0' into hometown-dev

app/models/account.rb

@@ -6,12 +6,8 @@
 #  id                            :bigint(8)        not null, primary key
 #  username                      :string           default(""), not null
 #  domain                        :string
-#  secret                        :string           default(""), not null
 #  private_key                   :text
 #  public_key                    :text             default(""), not null
-#  remote_url                    :string           default(""), not null
-#  salmon_url                    :string           default(""), not null
-#  hub_url                       :string           default(""), not null
 #  created_at                    :datetime         not null
 #  updated_at                    :datetime         not null
 #  note                          :text             default(""), not null
@@ -27,7 +23,6 @@
 #  header_file_size              :integer
 #  header_updated_at             :datetime
 #  avatar_remote_url             :string
-#  subscription_expires_at       :datetime
 #  locked                        :boolean          default(FALSE), not null
 #  header_remote_url             :string           default(""), not null
 #  last_webfingered_at           :datetime
@@ -50,11 +45,19 @@
 #  avatar_storage_schema_version :integer
 #  header_storage_schema_version :integer
 #  devices_url                   :string
-#  sensitized_at                 :datetime
 #  suspension_origin             :integer
+#  sensitized_at                 :datetime
 #
 
 class Account < ApplicationRecord
+  self.ignored_columns = %w(
+    subscription_expires_at
+    secret
+    remote_url
+    salmon_url
+    hub_url
+  )
+
   USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
   MENTION_RE  = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[a-z0-9]+)?)/i
 
@@ -93,7 +96,6 @@ class Account < ApplicationRecord
 
   scope :remote, -> { where.not(domain: nil) }
   scope :local, -> { where(domain: nil) }
-  scope :expiring, ->(time) { remote.where.not(subscription_expires_at: nil).where('subscription_expires_at < ?', time) }
   scope :partitioned, -> { order(Arel.sql('row_number() over (partition by domain)')) }
   scope :silenced, -> { where.not(silenced_at: nil) }
   scope :suspended, -> { where.not(suspended_at: nil) }
@@ -110,7 +112,7 @@ class Account < ApplicationRecord
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
   scope :searchable, -> { without_suspended.where(moved_to_account_id: nil) }
   scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) }
-  scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) }
+  scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) }
   scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) }
   scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) }
   scope :popular, -> { order('account_stats.followers_count desc') }
@@ -190,10 +192,6 @@ class Account < ApplicationRecord
     "acct:#{local_username_and_domain}"
   end
 
-  def subscribed?
-    subscription_expires_at.present?
-  end
-
   def searchable?
     !(suspended? || moved?)
   end
@@ -238,6 +236,7 @@ class Account < ApplicationRecord
     transaction do
       create_deletion_request!
       update!(suspended_at: date, suspension_origin: origin)
+      create_canonical_email_block!
     end
   end
 
@@ -245,6 +244,7 @@ class Account < ApplicationRecord
     transaction do
       deletion_request&.destroy!
       update!(suspended_at: nil, suspension_origin: nil)
+      destroy_canonical_email_block!
     end
   end
 
@@ -273,26 +273,20 @@ class Account < ApplicationRecord
   end
 
   def tags_as_strings=(tag_names)
-    hashtags_map = Tag.find_or_create_by_names(tag_names).each_with_object({}) { |tag, h| h[tag.name] = tag }
+    hashtags_map = Tag.find_or_create_by_names(tag_names).index_by(&:name)
 
     # Remove hashtags that are to be deleted
     tags.each do |tag|
       if hashtags_map.key?(tag.name)
         hashtags_map.delete(tag.name)
       else
-        transaction do
-          tags.delete(tag)
-          tag.decrement_count!(:accounts_count)
-        end
+        tags.delete(tag)
       end
     end
 
     # Add hashtags that were so far missing
     hashtags_map.each_value do |tag|
-      transaction do
-        tags << tag
-        tag.increment_count!(:accounts_count)
-      end
+      tags << tag
     end
   end
 
@@ -367,7 +361,7 @@ class Account < ApplicationRecord
   end
 
   def excluded_from_timeline_account_ids
-    Rails.cache.fetch("exclude_account_ids_for:#{id}") { blocking.pluck(:target_account_id) + blocked_by.pluck(:account_id) + muting.pluck(:target_account_id) }
+    Rails.cache.fetch("exclude_account_ids_for:#{id}") { block_relationships.pluck(:target_account_id) + blocked_by_relationships.pluck(:account_id) + mute_relationships.pluck(:target_account_id) }
   end
 
   def excluded_from_timeline_domains
@@ -385,15 +379,17 @@ class Account < ApplicationRecord
   end
 
   class Field < ActiveModelSerializers::Model
-    attributes :name, :value, :verified_at, :account, :errors
+    attributes :name, :value, :verified_at, :account
 
     def initialize(account, attributes)
-      @account     = account
-      @attributes  = attributes
-      @name        = attributes['name'].strip[0, string_limit]
-      @value       = attributes['value'].strip[0, string_limit]
-      @verified_at = attributes['verified_at']&.to_datetime
-      @errors      = {}
+      @original_field = attributes
+      string_limit = account.local? ? 255 : 2047
+      super(
+        account:     account,
+        name:        attributes['name'].strip[0, string_limit],
+        value:       attributes['value'].strip[0, string_limit],
+        verified_at: attributes['verified_at']&.to_datetime,
+      )
     end
 
     def verified?
@@ -415,22 +411,12 @@ class Account < ApplicationRecord
     end
 
     def mark_verified!
-      @verified_at = Time.now.utc
-      @attributes['verified_at'] = @verified_at
+      self.verified_at = Time.now.utc
+      @original_field['verified_at'] = verified_at
     end
 
     def to_h
-      { name: @name, value: @value, verified_at: @verified_at }
-    end
-
-    private
-
-    def string_limit
-      if account.local?
-        255
-      else
-        2047
-      end
+      { name: name, value: value, verified_at: verified_at }
     end
   end
 
@@ -516,7 +502,7 @@ class Account < ApplicationRecord
     def from_text(text)
       return [] if text.blank?
 
-      text.scan(MENTION_RE).map { |match| match.first.split('@', 2) }.uniq.map do |(username, domain)|
+      text.scan(MENTION_RE).map { |match| match.first.split('@', 2) }.uniq.filter_map do |(username, domain)|
         domain = begin
           if TagManager.instance.local_domain?(domain)
             nil
@@ -525,7 +511,7 @@ class Account < ApplicationRecord
           end
         end
         EntityCache.instance.mention(username, domain)
-      end.compact
+      end
     end
 
     private
@@ -580,4 +566,16 @@ class Account < ApplicationRecord
   def clean_feed_manager
     FeedManager.instance.clean_feeds!(:home, [id])
   end
+
+  def create_canonical_email_block!
+    return unless local? && user_email.present?
+
+    CanonicalEmailBlock.create(reference_account: self, email: user_email)
+  end
+
+  def destroy_canonical_email_block!
+    return unless local?
+
+    CanonicalEmailBlock.where(reference_account: self).delete_all
+  end
 end

app/models/account_migration.rb

@@ -14,6 +14,8 @@
 #
 
 class AccountMigration < ApplicationRecord
+  include Redisable
+
   COOLDOWN_PERIOD = 30.days.freeze
 
   belongs_to :account
@@ -39,7 +41,13 @@ class AccountMigration < ApplicationRecord
 
     return false unless errors.empty?
 
-    save
+    RedisLock.acquire(lock_options) do |lock|
+      if lock.acquired?
+        save
+      else
+        raise Mastodon::RaceConditionError
+      end
+    end
   end
 
   def cooldown_at
@@ -75,4 +83,8 @@ class AccountMigration < ApplicationRecord
   def validate_migration_cooldown
     errors.add(:base, I18n.t('migrations.errors.on_cooldown')) if account.migrations.within_cooldown.exists?
   end
+
+  def lock_options
+    { redis: redis, key: "account_migration:#{account.id}" }
+  end
 end

app/models/account_stat.rb

@@ -18,46 +18,4 @@ class AccountStat < ApplicationRecord
   belongs_to :account, inverse_of: :account_stat
 
   update_index('accounts#account', :account)
-
-  def increment_count!(key)
-    update(attributes_for_increment(key))
-  rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique
-    begin
-      reload_with_id
-    rescue ActiveRecord::RecordNotFound
-      return
-    end
-
-    retry
-  end
-
-  def decrement_count!(key)
-    update(attributes_for_decrement(key))
-  rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique
-    begin
-      reload_with_id
-    rescue ActiveRecord::RecordNotFound
-      return
-    end
-
-    retry
-  end
-
-  private
-
-  def attributes_for_increment(key)
-    attrs = { key => public_send(key) + 1 }
-    attrs[:last_status_at] = Time.now.utc if key == :statuses_count
-    attrs
-  end
-
-  def attributes_for_decrement(key)
-    attrs = { key => [public_send(key) - 1, 0].max }
-    attrs
-  end
-
-  def reload_with_id
-    self.id = self.class.find_by!(account: account).id if new_record?
-    reload
-  end
 end

app/models/account_suggestions.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class AccountSuggestions
+  SOURCES = [
+    AccountSuggestions::SettingSource,
+    AccountSuggestions::PastInteractionsSource,
+    AccountSuggestions::GlobalSource,
+  ].freeze
+
+  def self.get(account, limit)
+    SOURCES.each_with_object([]) do |source_class, suggestions|
+      source_suggestions = source_class.new.get(
+        account,
+        skip_account_ids: suggestions.map(&:account_id),
+        limit: limit - suggestions.size
+      )
+
+      suggestions.concat(source_suggestions)
+    end
+  end
+
+  def self.remove(account, target_account_id)
+    SOURCES.each do |source_class|
+      source = source_class.new
+      source.remove(account, target_account_id)
+    end
+  end
+end

app/models/account_suggestions/global_source.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::GlobalSource < AccountSuggestions::Source
+  def key
+    :global
+  end
+
+  def get(account, skip_account_ids: [], limit: 40)
+    account_ids = account_ids_for_locale(account.user_locale) - [account.id] - skip_account_ids
+
+    as_ordered_suggestions(
+      scope(account).where(id: account_ids),
+      account_ids
+    ).take(limit)
+  end
+
+  def remove(_account, _target_account_id)
+    nil
+  end
+
+  private
+
+  def scope(account)
+    Account.searchable
+           .followable_by(account)
+           .not_excluded_by_account(account)
+           .not_domain_blocked_by_account(account)
+  end
+
+  def account_ids_for_locale(locale)
+    Redis.current.zrevrange("follow_recommendations:#{locale}", 0, -1).map(&:to_i)
+  end
+
+  def to_ordered_list_key(account)
+    account.id
+  end
+end

app/models/account_suggestions/past_interactions_source.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::PastInteractionsSource < AccountSuggestions::Source
+  include Redisable
+
+  def key
+    :past_interactions
+  end
+
+  def get(account, skip_account_ids: [], limit: 40)
+    account_ids = account_ids_for_account(account.id, limit + skip_account_ids.size) - skip_account_ids
+
+    as_ordered_suggestions(
+      scope.where(id: account_ids),
+      account_ids
+    ).take(limit)
+  end
+
+  def remove(account, target_account_id)
+    redis.zrem("interactions:#{account.id}", target_account_id)
+  end
+
+  private
+
+  def scope
+    Account.searchable
+  end
+
+  def account_ids_for_account(account_id, limit)
+    redis.zrevrange("interactions:#{account_id}", 0, limit).map(&:to_i)
+  end
+
+  def to_ordered_list_key(account)
+    account.id
+  end
+end

app/models/account_suggestions/setting_source.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::SettingSource < AccountSuggestions::Source
+  def key
+    :staff
+  end
+
+  def get(account, skip_account_ids: [], limit: 40)
+    return [] unless setting_enabled?
+
+    as_ordered_suggestions(
+      scope(account).where(setting_to_where_condition).where.not(id: skip_account_ids),
+      usernames_and_domains
+    ).take(limit)
+  end
+
+  def remove(_account, _target_account_id)
+    nil
+  end
+
+  private
+
+  def scope(account)
+    Account.searchable
+           .followable_by(account)
+           .not_excluded_by_account(account)
+           .not_domain_blocked_by_account(account)
+           .where(locked: false)
+           .where.not(id: account.id)
+  end
+
+  def usernames_and_domains
+    @usernames_and_domains ||= setting_to_usernames_and_domains
+  end
+
+  def setting_enabled?
+    setting.present?
+  end
+
+  def setting_to_where_condition
+    usernames_and_domains.map do |(username, domain)|
+      Arel::Nodes::Grouping.new(
+        Account.arel_table[:username].lower.eq(username.downcase).and(
+          Account.arel_table[:domain].lower.eq(domain&.downcase)
+        )
+      )
+    end.reduce(:or)
+  end
+
+  def setting_to_usernames_and_domains
+    setting.split(',').map do |str|
+      username, domain = str.strip.gsub(/\A@/, '').split('@', 2)
+      domain           = nil if TagManager.instance.local_domain?(domain)
+
+      next if username.blank?
+
+      [username, domain]
+    end.compact
+  end
+
+  def setting
+    Setting.bootstrap_timeline_accounts
+  end
+
+  def to_ordered_list_key(account)
+    [account.username, account.domain]
+  end
+end

app/models/account_suggestions/source.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::Source
+  def key
+    raise NotImplementedError
+  end
+
+  def get(_account, **kwargs)
+    raise NotImplementedError
+  end
+
+  def remove(_account, target_account_id)
+    raise NotImplementedError
+  end
+
+  protected
+
+  def as_ordered_suggestions(scope, ordered_list)
+    return [] if ordered_list.empty?
+
+    map = scope.index_by(&method(:to_ordered_list_key))
+
+    ordered_list.map { |ordered_list_key| map[ordered_list_key] }.compact.map do |account|
+      AccountSuggestions::Suggestion.new(
+        account: account,
+        source: key
+      )
+    end
+  end
+
+  def to_ordered_list_key(_account)
+    raise NotImplementedError
+  end
+end

app/models/account_suggestions/suggestion.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AccountSuggestions::Suggestion < ActiveModelSerializers::Model
+  attributes :account, :source
+
+  delegate :id, to: :account, prefix: true
+end

app/models/account_summary.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: account_summaries
+#
+#  account_id :bigint(8)        primary key
+#  language   :string
+#  sensitive  :boolean
+#
+
+class AccountSummary < ApplicationRecord
+  self.primary_key = :account_id
+
+  scope :safe, -> { where(sensitive: false) }
+  scope :localized, ->(locale) { where(language: locale) }
+  scope :filtered, -> { joins(arel_table.join(FollowRecommendationSuppression.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:account_id].eq(FollowRecommendationSuppression.arel_table[:account_id])).join_sources).where(FollowRecommendationSuppression.arel_table[:id].eq(nil)) }
+
+  def self.refresh
+    Scenic.database.refresh_materialized_view(table_name, concurrently: false, cascade: false)
+  end
+
+  def readonly?
+    true
+  end
+end

app/models/account_tag_stat.rb

@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-# == Schema Information
-#
-# Table name: account_tag_stats
-#
-#  id             :bigint(8)        not null, primary key
-#  tag_id         :bigint(8)        not null
-#  accounts_count :bigint(8)        default(0), not null
-#  hidden         :boolean          default(FALSE), not null
-#  created_at     :datetime         not null
-#  updated_at     :datetime         not null
-#
-
-class AccountTagStat < ApplicationRecord
-  belongs_to :tag, inverse_of: :account_tag_stat
-
-  def increment_count!(key)
-    update(key => public_send(key) + 1)
-  end
-
-  def decrement_count!(key)
-    update(key => [public_send(key) - 1, 0].max)
-  end
-end

app/models/admin/action_log_filter.rb

@@ -17,12 +17,14 @@ class Admin::ActionLogFilter
     create_domain_allow: { target_type: 'DomainAllow', action: 'create' }.freeze,
     create_domain_block: { target_type: 'DomainBlock', action: 'create' }.freeze,
     create_email_domain_block: { target_type: 'EmailDomainBlock', action: 'create' }.freeze,
+    create_unavailable_domain: { target_type: 'UnavailableDomain', action: 'create' }.freeze,
     demote_user: { target_type: 'User', action: 'demote' }.freeze,
     destroy_announcement: { target_type: 'Announcement', action: 'destroy' }.freeze,
     destroy_custom_emoji: { target_type: 'CustomEmoji', action: 'destroy' }.freeze,
     destroy_domain_allow: { target_type: 'DomainAllow', action: 'destroy' }.freeze,
     destroy_domain_block: { target_type: 'DomainBlock', action: 'destroy' }.freeze,
     destroy_email_domain_block: { target_type: 'EmailDomainBlock', action: 'destroy' }.freeze,
+    destroy_unavailable_domain: { target_type: 'UnavailableDomain', action: 'destroy' }.freeze,
     destroy_status: { target_type: 'Status', action: 'destroy' }.freeze,
     disable_2fa_user: { target_type: 'User', action: 'disable' }.freeze,
     disable_custom_emoji: { target_type: 'CustomEmoji', action: 'disable' }.freeze,

app/models/canonical_email_block.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: canonical_email_blocks
+#
+#  id                   :bigint(8)        not null, primary key
+#  canonical_email_hash :string           default(""), not null
+#  reference_account_id :bigint(8)        not null
+#  created_at           :datetime         not null
+#  updated_at           :datetime         not null
+#
+
+class CanonicalEmailBlock < ApplicationRecord
+  include EmailHelper
+
+  belongs_to :reference_account, class_name: 'Account'
+
+  validates :canonical_email_hash, presence: true
+
+  def email=(email)
+    self.canonical_email_hash = email_to_canonical_email_hash(email)
+  end
+
+  def self.block?(email)
+    where(canonical_email_hash: email_to_canonical_email_hash(email)).exists?
+  end
+end

app/models/concerns/account_associations.rb

@@ -63,5 +63,8 @@ module AccountAssociations
 
     # Account deletion requests
     has_one :deletion_request, class_name: 'AccountDeletionRequest', inverse_of: :account, dependent: :destroy
+
+    # Follow recommendations
+    has_one :follow_recommendation_suppression, inverse_of: :account, dependent: :destroy
   end
 end

app/models/concerns/account_avatar.rb

@@ -21,7 +21,7 @@ module AccountAvatar
     has_attached_file :avatar, styles: ->(f) { avatar_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
     validates_attachment_content_type :avatar, content_type: IMAGE_MIME_TYPES
     validates_attachment_size :avatar, less_than: LIMIT
-    remotable_attachment :avatar, LIMIT
+    remotable_attachment :avatar, LIMIT, suppress_errors: false
   end
 
   def avatar_original_url

app/models/concerns/account_counters.rb

@@ -3,6 +3,8 @@
 module AccountCounters
   extend ActiveSupport::Concern
 
+  ALLOWED_COUNTER_KEYS = %i(statuses_count following_count followers_count).freeze
+
   included do
     has_one :account_stat, inverse_of: :account
     after_save :save_account_stat
@@ -14,11 +16,65 @@ module AccountCounters
            :following_count=,
            :followers_count,
            :followers_count=,
-           :increment_count!,
-           :decrement_count!,
            :last_status_at,
            to: :account_stat
 
+  # @param [Symbol] key
+  def increment_count!(key)
+    update_count!(key, 1)
+  end
+
+  # @param [Symbol] key
+  def decrement_count!(key)
+    update_count!(key, -1)
+  end
+
+  # @param [Symbol] key
+  # @param [Integer] value
+  def update_count!(key, value)
+    raise ArgumentError, "Invalid key #{key}" unless ALLOWED_COUNTER_KEYS.include?(key)
+    raise ArgumentError, 'Do not call update_count! on dirty objects' if association(:account_stat).loaded? && account_stat&.changed? && account_stat.changed_attribute_names_to_save == %w(id)
+
+    value = value.to_i
+    default_value = value.positive? ? value : 0
+
+    # We do an upsert using manually written SQL, as Rails' upsert method does
+    # not seem to support writing expressions in the UPDATE clause, but only
+    # re-insert the provided values instead.
+    # Even ARel seem to be missing proper handling of upserts.
+    sql = if value.positive? && key == :statuses_count
+            <<-SQL.squish
+              INSERT INTO account_stats(account_id, #{key}, created_at, updated_at, last_status_at)
+                VALUES (:account_id, :default_value, now(), now(), now())
+              ON CONFLICT (account_id) DO UPDATE
+              SET #{key} = account_stats.#{key} + :value,
+                  last_status_at = now(),
+                  lock_version = account_stats.lock_version + 1,
+                  updated_at = now()
+              RETURNING id;
+            SQL
+          else
+            <<-SQL.squish
+              INSERT INTO account_stats(account_id, #{key}, created_at, updated_at)
+                VALUES (:account_id, :default_value, now(), now())
+              ON CONFLICT (account_id) DO UPDATE
+              SET #{key} = account_stats.#{key} + :value,
+                  lock_version = account_stats.lock_version + 1,
+                  updated_at = now()
+              RETURNING id;
+            SQL
+          end
+
+    sql = AccountStat.sanitize_sql([sql, account_id: id, default_value: default_value, value: value])
+    account_stat_id = AccountStat.connection.exec_query(sql)[0]['id']
+
+    # Reload account_stat if it was loaded, taking into account newly-created unsaved records
+    if association(:account_stat).loaded?
+      account_stat.id = account_stat_id if account_stat.new_record?
+      account_stat.reload
+    end
+  end
+
   def account_stat
     super || build_account_stat
   end

app/models/concerns/account_finder_concern.rb

@@ -14,6 +14,8 @@ module AccountFinderConcern
 
     def representative
       Account.find(-99)
+    rescue ActiveRecord::RecordNotFound
+      Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain)
     end
 
     def find_local(username)

app/models/concerns/account_header.rb

@@ -22,7 +22,7 @@ module AccountHeader
     has_attached_file :header, styles: ->(f) { header_styles(f) }, convert_options: { all: '-strip' }, processors: [:lazy_thumbnail]
     validates_attachment_content_type :header, content_type: IMAGE_MIME_TYPES
     validates_attachment_size :header, less_than: LIMIT
-    remotable_attachment :header, LIMIT
+    remotable_attachment :header, LIMIT, suppress_errors: false
   end
 
   def header_original_url

app/models/concerns/account_interactions.rb

@@ -67,7 +67,7 @@ module AccountInteractions
     private
 
     def follow_mapping(query, field)
-      query.pluck(field).each_with_object({}) { |id, mapping| mapping[id] = true }
+      query.pluck(field).index_with(true)
     end
   end
 
@@ -184,6 +184,14 @@ module AccountInteractions
     active_relationships.where(target_account: other_account).exists?
   end
 
+  def following_anyone?
+    active_relationships.exists?
+  end
+
+  def not_following_anyone?
+    !following_anyone?
+  end
+
   def blocking?(other_account)
     block_relationships.where(target_account: other_account).exists?
   end

app/models/concerns/account_merging.rb

@@ -15,7 +15,7 @@ module AccountMerging
       Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite,
       Follow, FollowRequest, Block, Mute, AccountIdentityProof,
       AccountModerationNote, AccountPin, AccountStat, ListAccount,
-      PollVote, Mention, AccountDeletionRequest, AccountNote
+      PollVote, Mention, AccountDeletionRequest, AccountNote, FollowRecommendationSuppression
     ]
 
     owned_classes.each do |klass|
@@ -43,6 +43,10 @@ module AccountMerging
       end
     end
 
+    CanonicalEmailBlock.where(reference_account_id: other_account.id).find_each do |record|
+      record.update_attribute(:reference_account_id, id)
+    end
+
     # Some follow relationships have moved, so the cache is stale
     Rails.cache.delete_matched("followers_hash:#{id}:*")
     Rails.cache.delete_matched("relationships:#{id}:*")

app/models/concerns/expireable.rb

@@ -17,7 +17,7 @@ module Expireable
     end
 
     def expires_in=(interval)
-      self.expires_at = interval.to_i.seconds.from_now if interval.present?
+      self.expires_at = interval.present? ? interval.to_i.seconds.from_now : nil 
       @expires_in     = interval
     end
 

app/models/concerns/omniauthable.rb

@@ -57,7 +57,7 @@ module Omniauthable
 
       user = User.new(user_params_from_auth(email, auth))
 
-      user.account.avatar_remote_url = auth.info.image if auth.info.image =~ /\A#{URI::DEFAULT_PARSER.make_regexp(%w(http https))}\z/
+      user.account.avatar_remote_url = auth.info.image if /\A#{URI::DEFAULT_PARSER.make_regexp(%w(http https))}\z/.match?(auth.info.image)
       user.skip_confirmation!
       user.save!
       user
@@ -68,7 +68,6 @@ module Omniauthable
     def user_params_from_auth(email, auth)
       {
         email: email || "#{TEMP_EMAIL_PREFIX}-#{auth.uid}-#{auth.provider}.com",
-        password: Devise.friendly_token[0, 20],
         agreement: true,
         external: true,
         account_attributes: {

app/models/concerns/remotable.rb

@@ -28,9 +28,11 @@ module Remotable
           end
         rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e
           Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}"
+          public_send("#{attachment_name}=", nil) if public_send("#{attachment_name}_file_name").present?
           raise e unless suppress_errors
         rescue Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, Paperclip::Error, Mastodon::DimensionsValidationError, Mastodon::StreamValidationError => e
           Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}"
+          public_send("#{attachment_name}=", nil) if public_send("#{attachment_name}_file_name").present?
         end
 
         nil

app/models/concerns/status_threading_concern.rb

@@ -83,7 +83,7 @@ module StatusThreadingConcern
   def find_statuses_from_tree_path(ids, account, promote: false)
     statuses    = Status.with_accounts(ids).to_a
     account_ids = statuses.map(&:account_id).uniq
-    domains     = statuses.map(&:account_domain).compact.uniq
+    domains     = statuses.filter_map(&:account_domain).uniq
     relations   = relations_map_for_account(account, account_ids, domains)
 
     statuses.reject! { |status| StatusFilter.new(status, account, relations).filtered? }

app/models/custom_filter.rb

@@ -46,7 +46,7 @@ class CustomFilter < ApplicationRecord
   private
 
   def clean_up_contexts
-    self.context = Array(context).map(&:strip).map(&:presence).compact
+    self.context = Array(context).map(&:strip).filter_map(&:presence)
   end
 
   def remove_cache

app/models/follow_recommendation.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: follow_recommendations
+#
+#  account_id :bigint(8)        primary key
+#  rank       :decimal(, )
+#  reason     :text             is an Array
+#
+
+class FollowRecommendation < ApplicationRecord
+  self.primary_key = :account_id
+
+  belongs_to :account_summary, foreign_key: :account_id
+  belongs_to :account, foreign_key: :account_id
+
+  scope :localized, ->(locale) { joins(:account_summary).merge(AccountSummary.localized(locale)) }
+
+  def self.refresh
+    Scenic.database.refresh_materialized_view(table_name, concurrently: false, cascade: false)
+  end
+
+  def readonly?
+    true
+  end
+end

app/models/follow_recommendation_filter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class FollowRecommendationFilter
+  KEYS = %i(
+    language
+    status
+  ).freeze
+
+  attr_reader :params, :language
+
+  def initialize(params)
+    @language = params.delete('language') || I18n.locale
+    @params   = params
+  end
+
+  def results
+    if params['status'] == 'suppressed'
+      Account.joins(:follow_recommendation_suppression).order(FollowRecommendationSuppression.arel_table[:id].desc).to_a
+    else
+      account_ids = Redis.current.zrevrange("follow_recommendations:#{@language}", 0, -1).map(&:to_i)
+      accounts    = Account.where(id: account_ids).index_by(&:id)
+
+      account_ids.map { |id| accounts[id] }.compact
+    end
+  end
+end

app/models/follow_recommendation_suppression.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: follow_recommendation_suppressions
+#
+#  id         :bigint(8)        not null, primary key
+#  account_id :bigint(8)        not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class FollowRecommendationSuppression < ApplicationRecord
+  include Redisable
+
+  belongs_to :account
+
+  after_commit :remove_follow_recommendations, on: :create
+
+  private
+
+  def remove_follow_recommendations
+    redis.pipelined do
+      I18n.available_locales.each do |locale|
+        redis.zrem("follow_recommendations:#{locale}", account_id)
+      end
+    end
+  end
+end

app/models/follow_request.rb

@@ -29,7 +29,7 @@ class FollowRequest < ApplicationRecord
   validates :account_id, uniqueness: { scope: :target_account_id }
 
   def authorize!
-    account.follow!(target_account, reblogs: show_reblogs, notify: notify, uri: uri)
+    account.follow!(target_account, reblogs: show_reblogs, notify: notify, uri: uri, bypass_limit: true)
     MergeWorker.perform_async(target_account.id, account.id) if account.local?
     destroy!
   end

app/models/form/account_batch.rb

@@ -21,6 +21,10 @@ class Form::AccountBatch
       approve!
     when 'reject'
       reject!
+    when 'suppress_follow_recommendation'
+      suppress_follow_recommendation!
+    when 'unsuppress_follow_recommendation'
+      unsuppress_follow_recommendation!
     end
   end
 
@@ -79,4 +83,18 @@ class Form::AccountBatch
     records.each { |account| authorize(account.user, :reject?) }
            .each { |account| DeleteAccountService.new.call(account, reserve_email: false, reserve_username: false) }
   end
+
+  def suppress_follow_recommendation!
+    authorize(:follow_recommendation, :suppress?)
+
+    accounts.each do |account|
+      FollowRecommendationSuppression.create(account: account)
+    end
+  end
+
+  def unsuppress_follow_recommendation!
+    authorize(:follow_recommendation, :unsuppress?)
+
+    FollowRecommendationSuppression.where(account_id: account_ids).destroy_all
+  end
 end

app/models/form/admin_settings.rb

@@ -16,7 +16,6 @@ class Form::AdminSettings
     open_deletion
     timeline_preview
     show_staff_badge
-    enable_bootstrap_timeline_accounts
     bootstrap_timeline_accounts
     theme
     min_invite_role
@@ -29,7 +28,6 @@ class Form::AdminSettings
     thumbnail
     hero
     mascot
-    spam_check_enabled
     trends
     trendable_by_default
     show_domain_blocks
@@ -42,13 +40,11 @@ class Form::AdminSettings
     open_deletion
     timeline_preview
     show_staff_badge
-    enable_bootstrap_timeline_accounts
     activity_api_enabled
     peers_api_enabled
     show_known_fediverse_at_about_page
     preview_sensitive_media
     profile_directory
-    spam_check_enabled
     trends
     trendable_by_default
     noindex

app/models/home_feed.rb

@@ -2,12 +2,11 @@
 
 class HomeFeed < Feed
   def initialize(account)
-    @type    = :home
-    @id      = account.id
     @account = account
+    super(:home, account.id)
   end
 
   def regenerating?
-    redis.exists?("account:#{@id}:regeneration")
+    redis.exists?("account:#{@account.id}:regeneration")
   end
 end

app/models/instance.rb

@@ -10,10 +10,13 @@
 class Instance < ApplicationRecord
   self.primary_key = :domain
 
+  attr_accessor :failure_days
+
   has_many :accounts, foreign_key: :domain, primary_key: :domain
 
   belongs_to :domain_block, foreign_key: :domain, primary_key: :domain
   belongs_to :domain_allow, foreign_key: :domain, primary_key: :domain
+  belongs_to :unavailable_domain, foreign_key: :domain, primary_key: :domain # skipcq: RB-RL1031
 
   scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
 

app/models/instance_filter.rb

@@ -4,6 +4,8 @@ class InstanceFilter
   KEYS = %i(
     limited
     by_domain
+    warning
+    unavailable
   ).freeze
 
   attr_reader :params
@@ -13,7 +15,7 @@ class InstanceFilter
   end
 
   def results
-    scope = Instance.includes(:domain_block, :domain_allow).order(accounts_count: :desc)
+    scope = Instance.includes(:domain_block, :domain_allow, :unavailable_domain).order(accounts_count: :desc)
 
     params.each do |key, value|
       scope.merge!(scope_for(key, value.to_s.strip)) if value.present?
@@ -32,6 +34,10 @@ class InstanceFilter
       Instance.joins(:domain_allow).reorder(Arel.sql('domain_allows.id desc'))
     when 'by_domain'
       Instance.matches_domain(value)
+    when 'warning'
+      Instance.where(domain: DeliveryFailureTracker.warning_domains)
+    when 'unavailable'
+      Instance.joins(:unavailable_domain)
     else
       raise "Unknown filter: #{key}"
     end

app/models/list_feed.rb

@@ -2,7 +2,6 @@
 
 class ListFeed < Feed
   def initialize(list)
-    @type    = :list
-    @id      = list.id
+    super(:list, list.id)
   end
 end

app/models/media_attachment.rb

@@ -59,7 +59,7 @@ class MediaAttachment < ApplicationRecord
 
   IMAGE_STYLES = {
     original: {
-      pixels: 1_638_400, # 1280x1280px
+      pixels: 2_073_600, # 1920x1080px
       file_geometry_parser: FastGeometryParser,
     }.freeze,
 
@@ -287,7 +287,7 @@ class MediaAttachment < ApplicationRecord
       if instance.file_content_type == 'image/gif'
         [:gif_transcoder, :blurhash_transcoder]
       elsif VIDEO_MIME_TYPES.include?(instance.file_content_type)
-        [:video_transcoder, :blurhash_transcoder, :type_corrector]
+        [:transcoder, :blurhash_transcoder, :type_corrector]
       elsif AUDIO_MIME_TYPES.include?(instance.file_content_type)
         [:image_extractor, :transcoder, :type_corrector]
       else
@@ -388,7 +388,7 @@ class MediaAttachment < ApplicationRecord
   # paths but ultimately the same file, so it makes sense to memoize the
   # result while disregarding the path
   def ffmpeg_data(path = nil)
-    @ffmpeg_data ||= FFMPEG::Movie.new(path)
+    @ffmpeg_data ||= VideoMetadataExtractor.new(path)
   end
 
   def enqueue_processing

app/models/notification.rb

@@ -17,7 +17,6 @@ class Notification < ApplicationRecord
   self.inheritance_column = nil
 
   include Paginable
-  include Cacheable
 
   LEGACY_TYPE_CLASS_MAP = {
     'Mention'       => :mention,
@@ -38,18 +37,24 @@ class Notification < ApplicationRecord
     poll
   ).freeze
 
-  STATUS_INCLUDES = [:account, :application, :preloadable_poll, :media_attachments, :tags, active_mentions: :account, reblog: [:account, :application, :preloadable_poll, :media_attachments, :tags, active_mentions: :account]].freeze
+  TARGET_STATUS_INCLUDES_BY_TYPE = {
+    status: :status,
+    reblog: [status: :reblog],
+    mention: [mention: :status],
+    favourite: [favourite: :status],
+    poll: [poll: :status],
+  }.freeze
 
   belongs_to :account, optional: true
   belongs_to :from_account, class_name: 'Account', optional: true
   belongs_to :activity, polymorphic: true, optional: true
 
-  belongs_to :mention,        foreign_type: 'Mention',       foreign_key: 'activity_id', optional: true
-  belongs_to :status,         foreign_type: 'Status',        foreign_key: 'activity_id', optional: true
-  belongs_to :follow,         foreign_type: 'Follow',        foreign_key: 'activity_id', optional: true
-  belongs_to :follow_request, foreign_type: 'FollowRequest', foreign_key: 'activity_id', optional: true
-  belongs_to :favourite,      foreign_type: 'Favourite',     foreign_key: 'activity_id', optional: true
-  belongs_to :poll,           foreign_type: 'Poll',          foreign_key: 'activity_id', optional: true
+  belongs_to :mention,        foreign_key: 'activity_id', optional: true
+  belongs_to :status,         foreign_key: 'activity_id', optional: true
+  belongs_to :follow,         foreign_key: 'activity_id', optional: true
+  belongs_to :follow_request, foreign_key: 'activity_id', optional: true
+  belongs_to :favourite,      foreign_key: 'activity_id', optional: true
+  belongs_to :poll,           foreign_key: 'activity_id', optional: true
 
   validates :type, inclusion: { in: TYPES }
 
@@ -65,8 +70,6 @@ class Notification < ApplicationRecord
     end
   }
 
-  cache_associated :from_account, status: STATUS_INCLUDES, mention: [status: STATUS_INCLUDES], favourite: [:account, status: STATUS_INCLUDES], follow: :account, follow_request: :account, poll: [status: STATUS_INCLUDES]
-
   def type
     @type ||= (super || LEGACY_TYPE_CLASS_MAP[activity_type]).to_sym
   end
@@ -87,21 +90,40 @@ class Notification < ApplicationRecord
   end
 
   class << self
-    def cache_ids
-      select(:id, :updated_at, :activity_type, :activity_id)
-    end
+    def preload_cache_collection_target_statuses(notifications, &_block)
+      notifications.group_by(&:type).each do |type, grouped_notifications|
+        associations = TARGET_STATUS_INCLUDES_BY_TYPE[type]
+        next unless associations
 
-    def reload_stale_associations!(cached_items)
-      account_ids = (cached_items.map(&:from_account_id) + cached_items.map { |item| item.target_status&.account_id }.compact).uniq
-
-      return if account_ids.empty?
-
-      accounts = Account.where(id: account_ids).includes(:account_stat).each_with_object({}) { |a, h| h[a.id] = a }
-
-      cached_items.each do |item|
-        item.from_account = accounts[item.from_account_id]
-        item.target_status.account = accounts[item.target_status.account_id] if item.target_status
+        # Instead of using the usual `includes`, manually preload each type.
+        # If polymorphic associations are loaded with the usual `includes`, other types of associations will be loaded more.
+        ActiveRecord::Associations::Preloader.new.preload(grouped_notifications, associations)
       end
+
+      unique_target_statuses = notifications.map(&:target_status).compact.uniq
+      # Call cache_collection in block
+      cached_statuses_by_id = yield(unique_target_statuses).index_by(&:id)
+
+      notifications.each do |notification|
+        next if notification.target_status.nil?
+
+        cached_status = cached_statuses_by_id[notification.target_status.id]
+
+        case notification.type
+        when :status
+          notification.status = cached_status
+        when :reblog
+          notification.status.reblog = cached_status
+        when :favourite
+          notification.favourite.status = cached_status
+        when :mention
+          notification.mention.status = cached_status
+        when :poll
+          notification.poll.status = cached_status
+        end
+      end
+
+      notifications
     end
   end
 

app/models/poll.rb

@@ -73,10 +73,12 @@ class Poll < ApplicationRecord
     attributes :id, :title, :votes_count, :poll
 
     def initialize(poll, id, title, votes_count)
-      @poll        = poll
-      @id          = id
-      @title       = title
-      @votes_count = votes_count
+      super(
+        poll:        poll,
+        id:          id,
+        title:       title,
+        votes_count: votes_count,
+      )
     end
   end
 

app/models/public_feed.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class PublicFeed < Feed
+class PublicFeed
   # @param [Account] account
   # @param [Hash] options
   # @option [Boolean] :with_replies
@@ -37,28 +37,30 @@ class PublicFeed < Feed
 
   private
 
+  attr_reader :account, :options
+
   def with_reblogs?
-    @options[:with_reblogs]
+    options[:with_reblogs]
   end
 
   def with_replies?
-    @options[:with_replies]
+    options[:with_replies]
   end
 
   def local_only?
-    @options[:local]
+    options[:local]
   end
 
   def remote_only?
-    @options[:remote]
+    options[:remote]
   end
 
   def account?
-    @account.present?
+    account.present?
   end
 
   def media_only?
-    @options[:only_media]
+    options[:only_media]
   end
 
   def public_scope
@@ -90,9 +92,9 @@ class PublicFeed < Feed
   end
 
   def account_filters_scope
-    Status.not_excluded_by_account(@account).tap do |scope|
-      scope.merge!(Status.not_domain_blocked_by_account(@account)) unless local_only?
-      scope.merge!(Status.in_chosen_languages(@account)) if @account.chosen_languages.present?
+    Status.not_excluded_by_account(account).tap do |scope|
+      scope.merge!(Status.not_domain_blocked_by_account(account)) unless local_only?
+      scope.merge!(Status.in_chosen_languages(account)) if account.chosen_languages.present?
     end
   end
 end

app/models/report.rb

@@ -32,7 +32,7 @@ class Report < ApplicationRecord
 
   scope :unresolved, -> { where(action_taken: false) }
   scope :resolved,   -> { where(action_taken: true) }
-  scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].each_with_object({}) { |k, h| h[k] = { user: [:invite_request, :invite] } }) }
+  scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].index_with({ user: [:invite_request, :invite] })) }
 
   validates :comment, length: { maximum: 1000 }
 

app/models/rule.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: rules
+#
+#  id         :bigint(8)        not null, primary key
+#  priority   :integer          default(0), not null
+#  deleted_at :datetime
+#  text       :text             default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+class Rule < ApplicationRecord
+  include Discard::Model
+
+  self.discard_column = :deleted_at
+
+  validates :text, presence: true, length: { maximum: 300 }
+
+  scope :ordered, -> { kept.order(priority: :asc) }
+end

app/models/session_activation.rb

@@ -44,7 +44,7 @@ class SessionActivation < ApplicationRecord
     end
 
     def activate(**options)
-      activation = create!(options)
+      activation = create!(**options)
       purge_old
       activation
     end

app/models/setting.rb

@@ -40,7 +40,7 @@ class Setting < RailsSettings::Base
 
     def all_as_records
       vars    = thing_scoped
-      records = vars.each_with_object({}) { |r, h| h[r.var] = r }
+      records = vars.index_by(&:var)
 
       default_settings.each do |key, default_value|
         next if records.key?(key) || default_value.is_a?(Hash)

app/models/status.rb

@@ -117,7 +117,7 @@ class Status < ApplicationRecord
                    :tags,
                    :preview_cards,
                    :preloadable_poll,
-                   account: :account_stat,
+                   account: [:account_stat, :user],
                    active_mentions: { account: :account_stat },
                    reblog: [
                      :application,
@@ -127,7 +127,7 @@ class Status < ApplicationRecord
                      :conversation,
                      :status_stat,
                      :preloadable_poll,
-                     account: :account_stat,
+                     account: [:account_stat, :user],
                      active_mentions: { account: :account_stat },
                    ],
                    thread: { account: :account_stat }
@@ -168,6 +168,10 @@ class Status < ApplicationRecord
     local_only
   end
 
+  def in_reply_to_local_account?
+    reply? && thread&.account&.local?
+  end
+
   def reblog?
     !reblog_of_id.nil?
   end
@@ -310,7 +314,7 @@ class Status < ApplicationRecord
 
       return if account_ids.empty?
 
-      accounts = Account.where(id: account_ids).includes(:account_stat).each_with_object({}) { |a, h| h[a.id] = a }
+      accounts = Account.where(id: account_ids).includes(:account_stat, :user).index_by(&:id)
 
       cached_items.each do |item|
         item.account = accounts[item.account_id]
@@ -343,7 +347,7 @@ class Status < ApplicationRecord
     def from_text(text)
       return [] if text.blank?
 
-      text.scan(FetchLinkCardService::URL_PATTERN).map(&:first).uniq.map do |url|
+      text.scan(FetchLinkCardService::URL_PATTERN).map(&:first).uniq.filter_map do |url|
         status = begin
           if TagManager.instance.local_url?(url)
             ActivityPub::TagManager.instance.uri_to_resource(url, Status)
@@ -352,7 +356,7 @@ class Status < ApplicationRecord
           end
         end
         status&.distributable? ? status : nil
-      end.compact
+      end
     end
   end
 

app/models/tag.rb

@@ -20,10 +20,8 @@
 class Tag < ApplicationRecord
   has_and_belongs_to_many :statuses
   has_and_belongs_to_many :accounts
-  has_and_belongs_to_many :sample_accounts, -> { local.discoverable.popular.limit(3) }, class_name: 'Account'
 
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
-  has_one :account_tag_stat, dependent: :destroy
 
   HASHTAG_SEPARATORS = "_\u00B7\u200c"
   HASHTAG_NAME_RE    = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)"
@@ -38,28 +36,11 @@ class Tag < ApplicationRecord
   scope :usable, -> { where(usable: [true, nil]) }
   scope :listable, -> { where(listable: [true, nil]) }
   scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) }
-  scope :discoverable, -> { listable.joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) }
-  scope :matches_name, ->(value) { where(arel_table[:name].matches("#{value}%")) }
-
-  delegate :accounts_count,
-           :accounts_count=,
-           :increment_count!,
-           :decrement_count!,
-           to: :account_tag_stat
-
-  after_save :save_account_tag_stat
+  scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index
 
   update_index('tags#tag', :self)
 
-  def account_tag_stat
-    super || build_account_tag_stat
-  end
-
-  def cached_sample_accounts
-    Rails.cache.fetch("#{cache_key}/sample_accounts", expires_in: 12.hours) { sample_accounts }
-  end
-
   def to_param
     name
   end
@@ -94,6 +75,10 @@ class Tag < ApplicationRecord
     requested_review_at.present?
   end
 
+  def use!(account, status: nil, at_time: Time.now.utc)
+    TrendingTags.record_use!(self, account, status: status, at_time: at_time)
+  end
+
   def trending?
     TrendingTags.trending?(self)
   end
@@ -126,10 +111,10 @@ class Tag < ApplicationRecord
     end
 
     def search_for(term, limit = 5, offset = 0, options = {})
-      normalized_term = normalize(term.strip)
-      pattern         = sanitize_sql_like(normalized_term) + '%'
-      query           = Tag.listable.where(arel_table[:name].lower.matches(pattern))
-      query           = query.where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil))) if options[:exclude_unreviewed]
+      stripped_term = term.strip
+
+      query = Tag.listable.matches_name(stripped_term)
+      query = query.merge(matching_name(stripped_term).or(where.not(reviewed_at: nil))) if options[:exclude_unreviewed]
 
       query.order(Arel.sql('length(name) ASC, name ASC'))
            .limit(limit)
@@ -145,7 +130,7 @@ class Tag < ApplicationRecord
     end
 
     def matching_name(name_or_names)
-      names = Array(name_or_names).map { |name| normalize(name).mb_chars.downcase.to_s }
+      names = Array(name_or_names).map { |name| arel_table.lower(normalize(name)) }
 
       if names.size == 1
         where(arel_table[:name].lower.eq(names.first))
@@ -154,8 +139,6 @@ class Tag < ApplicationRecord
       end
     end
 
-    private
-
     def normalize(str)
       str.gsub(/\A#/, '')
     end
@@ -163,11 +146,6 @@ class Tag < ApplicationRecord
 
   private
 
-  def save_account_tag_stat
-    return unless account_tag_stat&.changed?
-    account_tag_stat.save
-  end
-
   def validate_name_change
     errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero?
   end

app/models/tag_feed.rb

@@ -13,9 +13,8 @@ class TagFeed < PublicFeed
   # @option [Boolean] :remote
   # @option [Boolean] :only_media
   def initialize(tag, account, options = {})
-    @tag     = tag
-    @account = account
-    @options = options
+    @tag = tag
+    super(account, options)
   end
 
   # @param [Integer] limit
@@ -40,15 +39,15 @@ class TagFeed < PublicFeed
   private
 
   def tagged_with_any_scope
-    Status.group(:id).tagged_with(tags_for(Array(@tag.name) | Array(@options[:any])))
+    Status.group(:id).tagged_with(tags_for(Array(@tag.name) | Array(options[:any])))
   end
 
   def tagged_with_all_scope
-    Status.group(:id).tagged_with_all(tags_for(@options[:all]))
+    Status.group(:id).tagged_with_all(tags_for(options[:all]))
   end
 
   def tagged_with_none_scope
-    Status.group(:id).tagged_with_none(tags_for(@options[:none]))
+    Status.group(:id).tagged_with_none(tags_for(options[:none]))
   end
 
   def tags_for(names)

app/models/tag_filter.rb

@@ -33,8 +33,6 @@ class TagFilter
 
   def scope_for(key, value)
     case key.to_s
-    when 'directory'
-      Tag.discoverable
     when 'reviewed'
       Tag.reviewed.order(reviewed_at: :desc)
     when 'unreviewed'

app/models/trending_tags.rb

@@ -13,19 +13,23 @@ class TrendingTags
   class << self
     include Redisable
 
-    def record_use!(tag, account, at_time = Time.now.utc)
-      return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?)
+    def record_use!(tag, account, status: nil, at_time: Time.now.utc)
+      return unless tag.usable? && !account.silenced?
 
+      # Even if a tag is not allowed to trend, we still need to
+      # record the stats since they can be displayed in other places
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
       increment_use!(tag.id, at_time)
 
-      tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago
+      # Only update when the tag was last used once every 12 hours
+      # and only if a status is given (lets use ignore reblogs)
+      tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago))
     end
 
     def update!(at_time = Time.now.utc)
       tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
-      tags    = Tag.where(id: tag_ids.uniq)
+      tags    = Tag.trendable.where(id: tag_ids.uniq)
 
       # First pass to calculate scores and update the set
 
@@ -91,7 +95,7 @@ class TrendingTags
 
       tags = Tag.where(id: tag_ids)
       tags = tags.trendable if filtered
-      tags = tags.each_with_object({}) { |tag, h| h[tag.id] = tag }
+      tags = tags.index_by(&:id)
 
       tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit)
     end

app/models/user.rb

@@ -152,7 +152,7 @@ class User < ApplicationRecord
 
   def confirm
     new_user      = !confirmed?
-    self.approved = true if open_registrations?
+    self.approved = true if open_registrations? && !sign_up_from_ip_requires_approval?
 
     super
 
@@ -370,15 +370,20 @@ class User < ApplicationRecord
 
   protected
 
-  def send_devise_notification(notification, *args)
+  def send_devise_notification(notification, *args, **kwargs)
     # This method can be called in `after_update` and `after_commit` hooks,
     # but we must make sure the mailer is actually called *after* commit,
     # otherwise it may work on stale data. To do this, figure out if we are
     # within a transaction.
+
+    # It seems like devise sends keyword arguments as a hash in the last
+    # positional argument
+    kwargs = args.pop if args.last.is_a?(Hash) && kwargs.empty?
+
     if ActiveRecord::Base.connection.current_transaction.try(:records)&.include?(self)
-      pending_devise_notifications << [notification, args]
+      pending_devise_notifications << [notification, args, kwargs]
     else
-      render_and_send_devise_message(notification, *args)
+      render_and_send_devise_message(notification, *args, **kwargs)
     end
   end
 
@@ -389,8 +394,8 @@ class User < ApplicationRecord
   end
 
   def send_pending_devise_notifications
-    pending_devise_notifications.each do |notification, args|
-      render_and_send_devise_message(notification, *args)
+    pending_devise_notifications.each do |notification, args, kwargs|
+      render_and_send_devise_message(notification, *args, **kwargs)
     end
 
     # Empty the pending notifications array because the
@@ -403,8 +408,8 @@ class User < ApplicationRecord
     @pending_devise_notifications ||= []
   end
 
-  def render_and_send_devise_message(notification, *args)
-    devise_mailer.send(notification, self, *args).deliver_later
+  def render_and_send_devise_message(notification, *args, **kwargs)
+    devise_mailer.send(notification, self, *args, **kwargs).deliver_later
   end
 
   def set_approved
@@ -458,9 +463,7 @@ class User < ApplicationRecord
   end
 
   def regenerate_feed!
-    return unless Redis.current.setnx("account:#{account_id}:regeneration", true)
-    Redis.current.expire("account:#{account_id}:regeneration", 1.day.seconds)
-    RegenerationWorker.perform_async(account_id)
+    RegenerationWorker.perform_async(account_id) if Redis.current.set("account:#{account_id}:regeneration", true, nx: true, ex: 1.day.seconds)
   end
 
   def needs_feed_update?
@@ -468,7 +471,7 @@ class User < ApplicationRecord
   end
 
   def validate_email_dns?
-    email_changed? && !(Rails.env.test? || Rails.env.development?)
+    email_changed? && !external? && !(Rails.env.test? || Rails.env.development?)
   end
 
   def invite_text_required?

app/models/web/push_subscription.rb

@@ -24,81 +24,101 @@ class Web::PushSubscription < ApplicationRecord
   validates :key_p256dh, presence: true
   validates :key_auth, presence: true
 
-  def push(notification)
-    I18n.with_locale(associated_user&.locale || I18n.default_locale) do
-      push_payload(payload_for_notification(notification), 48.hours.seconds)
-    end
+  delegate :locale, to: :associated_user
+
+  def encrypt(payload)
+    Webpush::Encryption.encrypt(payload, key_p256dh, key_auth)
+  end
+
+  def audience
+    @audience ||= Addressable::URI.parse(endpoint).normalized_site
+  end
+
+  def crypto_key_header
+    p256ecdsa = vapid_key.public_key_for_push_header
+
+    "p256ecdsa=#{p256ecdsa}"
+  end
+
+  def authorization_header
+    jwt = JWT.encode({ aud: audience, exp: 24.hours.from_now.to_i, sub: "mailto:#{contact_email}" }, vapid_key.curve, 'ES256', typ: 'JWT')
+
+    "WebPush #{jwt}"
   end
 
   def pushable?(notification)
-    data&.key?('alerts') && ActiveModel::Type::Boolean.new.cast(data['alerts'][notification.type.to_s])
+    policy_allows_notification?(notification) && alert_enabled_for_notification_type?(notification)
   end
 
   def associated_user
     return @associated_user if defined?(@associated_user)
 
-    @associated_user = if user_id.nil?
-                         session_activation.user
-                       else
-                         user
-                       end
+    @associated_user = begin
+      if user_id.nil?
+        session_activation.user
+      else
+        user
+      end
+    end
   end
 
   def associated_access_token
     return @associated_access_token if defined?(@associated_access_token)
 
-    @associated_access_token = if access_token_id.nil?
-                                 find_or_create_access_token.token
-                               else
-                                 access_token.token
-                               end
+    @associated_access_token = begin
+      if access_token_id.nil?
+        find_or_create_access_token.token
+      else
+        access_token.token
+      end
+    end
   end
 
   class << self
     def unsubscribe_for(application_id, resource_owner)
-      access_token_ids = Doorkeeper::AccessToken.where(application_id: application_id, resource_owner_id: resource_owner.id, revoked_at: nil)
-                                                .pluck(:id)
-
+      access_token_ids = Doorkeeper::AccessToken.where(application_id: application_id, resource_owner_id: resource_owner.id, revoked_at: nil).pluck(:id)
       where(access_token_id: access_token_ids).delete_all
     end
   end
 
   private
 
-  def push_payload(message, ttl = 5.minutes.seconds)
-    Webpush.payload_send(
-      message: Oj.dump(message),
-      endpoint: endpoint,
-      p256dh: key_p256dh,
-      auth: key_auth,
-      ttl: ttl,
-      ssl_timeout: 10,
-      open_timeout: 10,
-      read_timeout: 10,
-      vapid: {
-        subject: "mailto:#{::Setting.site_contact_email}",
-        private_key: Rails.configuration.x.vapid_private_key,
-        public_key: Rails.configuration.x.vapid_public_key,
-      }
-    )
-  end
-
-  def payload_for_notification(notification)
-    ActiveModelSerializers::SerializableResource.new(
-      notification,
-      serializer: Web::NotificationSerializer,
-      scope: self,
-      scope_name: :current_push_subscription
-    ).as_json
-  end
-
   def find_or_create_access_token
     Doorkeeper::AccessToken.find_or_create_for(
       application: Doorkeeper::Application.find_by(superapp: true),
-      resource_owner: session_activation.user_id,
+      resource_owner: user_id || session_activation.user_id,
       scopes: Doorkeeper::OAuth::Scopes.from_string('read write follow push'),
       expires_in: Doorkeeper.configuration.access_token_expires_in,
       use_refresh_token: Doorkeeper.configuration.refresh_token_enabled?
     )
   end
+
+  def vapid_key
+    @vapid_key ||= Webpush::VapidKey.from_keys(Rails.configuration.x.vapid_public_key, Rails.configuration.x.vapid_private_key)
+  end
+
+  def contact_email
+    @contact_email ||= ::Setting.site_contact_email
+  end
+
+  def alert_enabled_for_notification_type?(notification)
+    truthy?(data&.dig('alerts', notification.type.to_s))
+  end
+
+  def policy_allows_notification?(notification)
+    case data&.dig('policy')
+    when nil, 'all'
+      true
+    when 'none'
+      false
+    when 'followed'
+      notification.account.following?(notification.from_account)
+    when 'follower'
+      notification.from_account.following?(notification.account)
+    end
+  end
+
+  def truthy?(val)
+    ActiveModel::Type::Boolean.new.cast(val)
+  end
 end