Punycode URI normalization (#2370)

* Fix #2119 - Whenever about to send a HTTP request, normalize the URI

* Add test for IDN request in FetchLinkCardService

* Perform IDN normalization on domains before they are stored in the DB

app/services/fetch_link_card_service.rb

@@ -19,7 +19,7 @@ class FetchLinkCardService < BaseService
 
     card.title       = meta_property(page, 'og:title') || page.at_xpath('//title')&.content
     card.description = meta_property(page, 'og:description') || meta_property(page, 'description')
-    card.image       = URI.parse(meta_property(page, 'og:image')) if meta_property(page, 'og:image')
+    card.image       = URI.parse(Addressable::URI.parse(meta_property(page, 'og:image')).normalize.to_s) if meta_property(page, 'og:image')
 
     return if card.title.blank?
 

app/services/fetch_remote_account_service.rb

@@ -21,7 +21,7 @@ class FetchRemoteAccountService < BaseService
 
     email = xml.at_xpath('//xmlns:author/xmlns:email').try(:content)
     if email.nil?
-      url_parts = Addressable::URI.parse(url)
+      url_parts = Addressable::URI.parse(url).normalize
       username  = xml.at_xpath('//xmlns:author/xmlns:name').try(:content)
       domain    = url_parts.host
     else

app/services/fetch_remote_status_service.rb

@@ -31,7 +31,7 @@ class FetchRemoteStatusService < BaseService
   end
 
   def extract_author(url, xml)
-    url_parts = Addressable::URI.parse(url)
+    url_parts = Addressable::URI.parse(url).normalize
     username  = xml.at_xpath('//xmlns:author/xmlns:name').try(:content)
     domain    = url_parts.host
 

app/services/follow_remote_account_service.rb

@@ -73,7 +73,7 @@ class FollowRemoteAccountService < BaseService
   end
 
   def get_feed(url)
-    response = http_client.get(Addressable::URI.parse(url))
+    response = http_client.get(Addressable::URI.parse(url).normalize)
     [response.to_s, Nokogiri::XML(response)]
   end
 

app/services/process_feed_service.rb

@@ -174,7 +174,7 @@ class ProcessFeedService < BaseService
     end
 
     def account_from_href(href)
-      url = Addressable::URI.parse(href)
+      url = Addressable::URI.parse(href).normalize
 
       if TagManager.instance.web_domain?(url.host)
         Account.find_local(url.path.gsub('/users/', ''))
@@ -195,7 +195,7 @@ class ProcessFeedService < BaseService
         next unless link['href']
 
         media = MediaAttachment.where(status: parent, remote_url: link['href']).first_or_initialize(account: parent.account, status: parent, remote_url: link['href'])
-        parsed_url = URI.parse(link['href'])
+        parsed_url = Addressable::URI.parse(link['href']).normalize
 
         next if !%w[http https].include?(parsed_url.scheme) || parsed_url.host.empty?
 
@@ -271,7 +271,7 @@ class ProcessFeedService < BaseService
     def acct(xml = @xml)
       username = xml.at_xpath('./xmlns:author/xmlns:name', xmlns: TagManager::XMLNS).content
       url      = xml.at_xpath('./xmlns:author/xmlns:uri', xmlns: TagManager::XMLNS).content
-      domain   = Addressable::URI.parse(url).host
+      domain   = Addressable::URI.parse(url).normalize.host
 
       "#{username}@#{domain}"
     end

app/services/process_interaction_service.rb

@@ -14,7 +14,7 @@ class ProcessInteractionService < BaseService
 
     username = xml.at_xpath('/xmlns:entry/xmlns:author/xmlns:name', xmlns: TagManager::XMLNS).content
     url      = xml.at_xpath('/xmlns:entry/xmlns:author/xmlns:uri', xmlns: TagManager::XMLNS).content
-    domain   = Addressable::URI.parse(url).host
+    domain   = Addressable::URI.parse(url).normalize.host
     account  = Account.find_by(username: username, domain: domain)
 
     if account.nil?

app/services/pubsubhubbub/subscribe_service.rb

@@ -4,7 +4,7 @@ class Pubsubhubbub::SubscribeService < BaseService
   def call(account, callback, secret, lease_seconds)
     return ['Invalid topic URL',        422] if account.nil?
     return ['Invalid callback URL',     422] unless !callback.blank? && callback =~ /\A#{URI.regexp(%w(http https))}\z/
-    return ['Callback URL not allowed', 403] if DomainBlock.blocked?(Addressable::URI.parse(callback).host)
+    return ['Callback URL not allowed', 403] if DomainBlock.blocked?(Addressable::URI.parse(callback).normalize.host)
 
     subscription = Subscription.where(account: account, callback_url: callback).first_or_create!(account: account, callback_url: callback)
     Pubsubhubbub::ConfirmationWorker.perform_async(subscription.id, 'subscribe', secret, lease_seconds)