Fix refollowing (#4931)

* Make RefollowWorker ActivityPub-only to avoid potential identifier mismatches

* Don't call RefollowWorker on new accounts

app/services/activitypub/process_account_service.rb

@@ -15,11 +15,11 @@ class ActivityPub::ProcessAccountService < BaseService
     @account     = Account.find_by(uri: @uri)
     @collections = {}
 
+    old_public_key = @account&.public_key
     create_account  if @account.nil?
     upgrade_account if @account.ostatus?
-    old_public_key = @account.public_key
     update_account
-    RefollowWorker.perform_async(@account.id) if old_public_key != @account.public_key
+    RefollowWorker.perform_async(@account.id) if !old_public_key.nil? && old_public_key != @account.public_key
 
     @account
   rescue Oj::ParseError

app/services/resolve_remote_account_service.rb

@@ -85,10 +85,8 @@ class ResolveRemoteAccountService < BaseService
 
   def handle_ostatus
     create_account if @account.nil?
-    old_public_key = @account.public_key
     update_account
     update_account_profile if update_profile?
-    RefollowWorker.perform_async(@account.id) if old_public_key != @account.public_key
   end
 
   def update_profile?

app/workers/refollow_worker.rb

@@ -7,6 +7,7 @@ class RefollowWorker
 
   def perform(target_account_id)
     target_account = Account.find(target_account_id)
+    return unless target_account.protocol == :activitypub
 
     target_account.followers.where(domain: nil).find_each do |follower|
       # Locally unfollow remote account