Switch to static URIs, new URI format in both protocols for new statuses (#4815)

* Decouple Status#local? from uri being nil

* Replace on-the-fly URI generation with stored URIs

- Generate URI in after_save hook for local statuses
- Use static value in TagManager when available, fallback to tag format
- Make TagManager use ActivityPub::TagManager to understand new format
- Adjust tests

* Use other heuristic for locality of old statuses, do not perform long query

* Exclude tombstone stream entries from Atom feed

* Prevent nil statuses from landing in Pubsubhubbub::DistributionWorker

* Fix URI not being saved (#4818)

* Add more specs for Status

* Save generated uri immediately

and also fix method order to minimize diff.

* Fix alternate HTML URL in Atom

* Fix tests

* Remove not-null constraint from statuses migration to speed it up

app/controllers/accounts_controller.rb

@@ -22,7 +22,7 @@ class AccountsController < ApplicationController
 
       format.atom do
         @entries = @account.stream_entries.where(hidden: false).with_includes.paginate_by_max_id(20, params[:max_id], params[:since_id])
-        render xml: OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.feed(@account, @entries.to_a))
+        render xml: OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.feed(@account, @entries.reject { |entry| entry.status.nil? }))
       end
 
       format.json do

app/lib/ostatus/atom_serializer.rb

@@ -65,7 +65,7 @@ class OStatus::AtomSerializer
 
     add_namespaces(entry) if root
 
-    append_element(entry, 'id', TagManager.instance.unique_tag(stream_entry.created_at, stream_entry.activity_id, stream_entry.activity_type))
+    append_element(entry, 'id', TagManager.instance.uri_for(stream_entry.status))
     append_element(entry, 'published', stream_entry.created_at.iso8601)
     append_element(entry, 'updated', stream_entry.updated_at.iso8601)
     append_element(entry, 'title', stream_entry&.status&.title || "#{stream_entry.account.acct} deleted status")
@@ -86,7 +86,7 @@ class OStatus::AtomSerializer
       serialize_status_attributes(entry, stream_entry.status)
     end
 
-    append_element(entry, 'link', nil, rel: :alternate, type: 'text/html', href: account_stream_entry_url(stream_entry.account, stream_entry))
+    append_element(entry, 'link', nil, rel: :alternate, type: 'text/html', href: TagManager.instance.url_for(stream_entry.status))
     append_element(entry, 'link', nil, rel: :self, type: 'application/atom+xml', href: account_stream_entry_url(stream_entry.account, stream_entry, format: 'atom'))
     append_element(entry, 'thr:in-reply-to', nil, ref: TagManager.instance.uri_for(stream_entry.thread), href: TagManager.instance.url_for(stream_entry.thread)) if stream_entry.threaded?
     append_element(entry, 'ostatus:conversation', nil, ref: conversation_uri(stream_entry.status.conversation)) unless stream_entry&.status&.conversation_id.nil?

app/lib/tag_manager.rb

@@ -49,12 +49,17 @@ class TagManager
 
   def unique_tag_to_local_id(tag, expected_type)
     return nil unless local_id?(tag)
-    matches = Regexp.new("objectId=([\\d]+):objectType=#{expected_type}").match(tag)
-    return matches[1] unless matches.nil?
+
+    if ActivityPub::TagManager.instance.local_uri?(tag)
+      ActivityPub::TagManager.instance.uri_to_local_id(tag)
+    else
+      matches = Regexp.new("objectId=([\\d]+):objectType=#{expected_type}").match(tag)
+      return matches[1] unless matches.nil?
+    end
   end
 
   def local_id?(id)
-    id.start_with?("tag:#{Rails.configuration.x.local_domain}")
+    id.start_with?("tag:#{Rails.configuration.x.local_domain}") || ActivityPub::TagManager.instance.local_uri?(id)
   end
 
   def web_domain?(domain)
@@ -92,7 +97,7 @@ class TagManager
     when :person
       account_url(target)
     when :note, :comment, :activity
-      unique_tag(target.created_at, target.id, 'Status')
+      target.uri || unique_tag(target.created_at, target.id, 'Status')
     end
   end
 

app/models/status.rb

@@ -22,6 +22,7 @@
 #  reblogs_count          :integer          default(0), not null
 #  language               :string
 #  conversation_id        :integer
+#  local                  :boolean          default(FALSE)
 #
 
 class Status < ApplicationRecord
@@ -84,7 +85,7 @@ class Status < ApplicationRecord
   end
 
   def local?
-    uri.nil?
+    attributes['local'] || uri.nil?
   end
 
   def reblog?
@@ -131,11 +132,14 @@ class Status < ApplicationRecord
     !sensitive? && media_attachments.any?
   end
 
+  after_create :store_uri, if: :local?
+
   before_validation :prepare_contents, if: :local?
   before_validation :set_reblog
   before_validation :set_visibility
   before_validation :set_conversation
   before_validation :set_sensitivity
+  before_validation :set_local
 
   class << self
     def not_in_filtered_languages(account)
@@ -253,6 +257,10 @@ class Status < ApplicationRecord
 
   private
 
+  def store_uri
+    update_attribute(:uri, ActivityPub::TagManager.instance.uri_for(self)) if uri.nil?
+  end
+
   def prepare_contents
     text&.strip!
     spoiler_text&.strip!
@@ -292,4 +300,8 @@ class Status < ApplicationRecord
       thread.account_id
     end
   end
+
+  def set_local
+    self.local = account.local?
+  end
 end

app/workers/pubsubhubbub/distribution_worker.rb

@@ -6,7 +6,7 @@ class Pubsubhubbub::DistributionWorker
   sidekiq_options queue: 'push'
 
   def perform(stream_entry_ids)
-    stream_entries = StreamEntry.where(id: stream_entry_ids).includes(:status).reject { |e| e.status&.direct_visibility? }
+    stream_entries = StreamEntry.where(id: stream_entry_ids).includes(:status).reject { |e| e.status.nil? || e.status.direct_visibility? }
 
     return if stream_entries.empty?