From 88a1d0cdb4df422594c3ffb335ad794df4e55ea7 Mon Sep 17 00:00:00 2001 From: Sam Schlinkert Date: Mon, 28 Jan 2019 16:57:42 -0500 Subject: [PATCH 01/42] Bumps copyright year in README.md to 2019 (#9939) This is so incredibly small, but assuming this is a needed change. Might want to check year in other files. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ad6894ca..03c8472b9 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ You can open issues for bugs you've found or features you think are missing. You ## License -Copyright (C) 2016-2018 Eugen Rochko & other Mastodon contributors (see [AUTHORS.md](AUTHORS.md)) +Copyright (C) 2016-2019 Eugen Rochko & other Mastodon contributors (see [AUTHORS.md](AUTHORS.md)) This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. From e31970b924a7eaa1279708919b2743a15fb099f0 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 1 Feb 2019 00:15:38 +0100 Subject: [PATCH 02/42] Fix link color in high-contrast theme, add underlines (#9949) Improve sorting of default themes in the dropdown --- app/javascript/styles/contrast/diff.scss | 51 ++++++++++++++++++++++++ config/locales/en.yml | 6 +-- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/app/javascript/styles/contrast/diff.scss b/app/javascript/styles/contrast/diff.scss index eee9ecc3e..7d8993a50 100644 --- a/app/javascript/styles/contrast/diff.scss +++ b/app/javascript/styles/contrast/diff.scss @@ -12,3 +12,54 @@ } } } + +.status__content a, +.reply-indicator__content a { + color: lighten($ui-highlight-color, 12%); + text-decoration: underline; + + &.mention { + text-decoration: none; + } + + &.mention span { + text-decoration: underline; + + &:hover, + &:focus, + &:active { + text-decoration: none; + } + } + + &:hover, + &:focus, + &:active { + text-decoration: none; + } + + &.status__content__spoiler-link { + color: $secondary-text-color; + text-decoration: none; + } +} + +.status__content__read-more-button { + text-decoration: underline; + + &:hover, + &:focus, + &:active { + text-decoration: none; + } +} + +.getting-started__footer a { + text-decoration: underline; + + &:hover, + &:focus, + &:active { + text-decoration: none; + } +} diff --git a/config/locales/en.yml b/config/locales/en.yml index 10749b21b..ba413f8fc 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -929,9 +929,9 @@ en:

Originally adapted from the Discourse privacy policy.

title: "%{instance} Terms of Service and Privacy Policy" themes: - contrast: High contrast - default: Mastodon - mastodon-light: Mastodon (light) + contrast: Mastodon (High contrast) + default: Mastodon (Dark) + mastodon-light: Mastodon (Light) time: formats: default: "%b %d, %Y, %H:%M" From 687a0cbcb036255d02aa1fab5132c51da2adc888 Mon Sep 17 00:00:00 2001 From: Clar Charr Date: Thu, 31 Jan 2019 07:45:15 -0500 Subject: [PATCH 03/42] Replace unlock-alt icon with unlock (#9952) --- app/helpers/stream_entries_helper.rb | 2 +- app/javascript/mastodon/components/account.js | 2 +- app/javascript/mastodon/components/domain.js | 2 +- app/javascript/mastodon/features/account/components/header.js | 2 +- .../mastodon/features/compose/components/privacy_dropdown.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/helpers/stream_entries_helper.rb b/app/helpers/stream_entries_helper.rb index 033d435c4..7a74c0b7d 100644 --- a/app/helpers/stream_entries_helper.rb +++ b/app/helpers/stream_entries_helper.rb @@ -170,7 +170,7 @@ module StreamEntriesHelper when 'public' fa_icon 'globe fw' when 'unlisted' - fa_icon 'unlock-alt fw' + fa_icon 'unlock fw' when 'private' fa_icon 'lock fw' when 'direct' diff --git a/app/javascript/mastodon/components/account.js b/app/javascript/mastodon/components/account.js index 206030c00..2705a6001 100644 --- a/app/javascript/mastodon/components/account.js +++ b/app/javascript/mastodon/components/account.js @@ -88,7 +88,7 @@ class Account extends ImmutablePureComponent { if (requested) { buttons = ; } else if (blocking) { - buttons = ; + buttons = ; } else if (muting) { let hidingNotificationsButton; if (account.getIn(['relationship', 'muting_notifications'])) { diff --git a/app/javascript/mastodon/components/domain.js b/app/javascript/mastodon/components/domain.js index 24f80e788..85729ca94 100644 --- a/app/javascript/mastodon/components/domain.js +++ b/app/javascript/mastodon/components/domain.js @@ -32,7 +32,7 @@ class Account extends ImmutablePureComponent {
- +
diff --git a/app/javascript/mastodon/features/account/components/header.js b/app/javascript/mastodon/features/account/components/header.js index 2ab25cde4..4f6d0712d 100644 --- a/app/javascript/mastodon/features/account/components/header.js +++ b/app/javascript/mastodon/features/account/components/header.js @@ -132,7 +132,7 @@ class Header extends ImmutablePureComponent { } else if (account.getIn(['relationship', 'blocking'])) { actionBtn = (
- +
); } diff --git a/app/javascript/mastodon/features/compose/components/privacy_dropdown.js b/app/javascript/mastodon/features/compose/components/privacy_dropdown.js index 5698765d9..e8f57a466 100644 --- a/app/javascript/mastodon/features/compose/components/privacy_dropdown.js +++ b/app/javascript/mastodon/features/compose/components/privacy_dropdown.js @@ -214,7 +214,7 @@ class PrivacyDropdown extends React.PureComponent { this.options = [ { icon: 'globe', value: 'public', text: formatMessage(messages.public_short), meta: formatMessage(messages.public_long) }, - { icon: 'unlock-alt', value: 'unlisted', text: formatMessage(messages.unlisted_short), meta: formatMessage(messages.unlisted_long) }, + { icon: 'unlock', value: 'unlisted', text: formatMessage(messages.unlisted_short), meta: formatMessage(messages.unlisted_long) }, { icon: 'lock', value: 'private', text: formatMessage(messages.private_short), meta: formatMessage(messages.private_long) }, { icon: 'envelope', value: 'direct', text: formatMessage(messages.direct_short), meta: formatMessage(messages.direct_long) }, ]; From fdf819b83e820576164074b6726cb6ffdb4a47f6 Mon Sep 17 00:00:00 2001 From: Jakub Mendyk Date: Sat, 2 Feb 2019 19:01:18 +0100 Subject: [PATCH 04/42] Allow most kinds of characters in URL query (fixes #8408) (#8447) * Allow unicode characters in URL query strings Fixes #8408 * Alternative approach to unicode support in urls Adds PoC/idea to approch this problem. --- app/lib/formatter.rb | 39 +++++++++++++++++++++++++++++++++++++- spec/lib/formatter_spec.rb | 32 ++++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 05fd9eeb1..2e3587169 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -99,7 +99,7 @@ class Formatter end def encode_and_link_urls(html, accounts = nil, options = {}) - entities = Extractor.extract_entities_with_indices(html, extract_url_without_protocol: false) + entities = utf8_friendly_extractor(html, extract_url_without_protocol: false) if accounts.is_a?(Hash) options = accounts @@ -199,6 +199,43 @@ class Formatter result.flatten.join end + def utf8_friendly_extractor(text, options = {}) + old_to_new_index = [0] + + escaped = text.chars.map do |c| + output = c.ord.to_s(16).length > 2 ? CGI.escape(c) : c + old_to_new_index << old_to_new_index.last + output.length + output + end.join + + # Note: I couldn't obtain list_slug with @user/list-name format + # for mention so this requires additional check + special = Extractor.extract_entities_with_indices(escaped, options).map do |extract| + # exactly one of :url, :hashtag, :screen_name, :cashtag keys is present + key = (extract.keys & [:url, :hashtag, :screen_name, :cashtag]).first + + new_indices = [ + old_to_new_index.find_index(extract[:indices].first), + old_to_new_index.find_index(extract[:indices].last), + ] + + has_prefix_char = [:hashtag, :screen_name, :cashtag].include?(key) + value_indices = [ + new_indices.first + (has_prefix_char ? 1 : 0), # account for #, @ or $ + new_indices.last - 1, + ] + + next extract.merge( + :indices => new_indices, + key => text[value_indices.first..value_indices.last] + ) + end + + standard = Extractor.extract_entities_with_indices(text, options) + + Extractor.remove_overlapping_entities(special + standard) + end + def link_to_url(entity, options = {}) url = Addressable::URI.parse(entity[:url]) html_attrs = { target: '_blank', rel: 'nofollow noopener' } diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb index 0c1efe7c3..9872d3756 100644 --- a/spec/lib/formatter_spec.rb +++ b/spec/lib/formatter_spec.rb @@ -74,10 +74,36 @@ RSpec.describe Formatter do end context 'given a URL with a query string' do - let(:text) { 'https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink' } + context 'with escaped unicode character' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink' } - it 'matches the full URL' do - is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink"' + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&q=autolink"' + end + end + + context 'with unicode character' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=✓&q=autolink' } + + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=✓&q=autolink"' + end + end + + context 'with unicode character at the end' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=✓' } + + it 'matches the full URL' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=✓"' + end + end + + context 'with escaped and not escaped unicode characters' do + let(:text) { 'https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&utf81=✓&q=autolink' } + + it 'preserves escaped unicode characters' do + is_expected.to include 'href="https://www.ruby-toolbox.com/search?utf8=%E2%9C%93&utf81=✓&q=autolink"' + end end end From a742a09530b6bdca78713e14dbab51bc3a56d222 Mon Sep 17 00:00:00 2001 From: rinsuki <428rinsuki+git@gmail.com> Date: Tue, 5 Feb 2019 06:25:42 +0900 Subject: [PATCH 05/42] Fix authorized applications list page design (#9969) --- app/controllers/oauth/authorized_applications_controller.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/app/controllers/oauth/authorized_applications_controller.rb b/app/controllers/oauth/authorized_applications_controller.rb index 0c28d194b..f3d235366 100644 --- a/app/controllers/oauth/authorized_applications_controller.rb +++ b/app/controllers/oauth/authorized_applications_controller.rb @@ -5,6 +5,7 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio before_action :store_current_location before_action :authenticate_resource_owner! + before_action :set_body_classes include Localized @@ -15,6 +16,10 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio private + def set_body_classes + @body_classes = 'admin' + end + def store_current_location store_location_for(:user, request.url) end From 5e7c75cfd328af81a557985e1b5b2c70e6c68645 Mon Sep 17 00:00:00 2001 From: rinsuki <428rinsuki+git@gmail.com> Date: Tue, 5 Feb 2019 07:14:57 +0900 Subject: [PATCH 06/42] Fix not showing custom emojis in share page emoji picker (#9970) --- app/javascript/mastodon/containers/compose_container.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/javascript/mastodon/containers/compose_container.js b/app/javascript/mastodon/containers/compose_container.js index 5ee1d2f14..7bc7bbaa4 100644 --- a/app/javascript/mastodon/containers/compose_container.js +++ b/app/javascript/mastodon/containers/compose_container.js @@ -7,6 +7,7 @@ import { IntlProvider, addLocaleData } from 'react-intl'; import { getLocale } from '../locales'; import Compose from '../features/standalone/compose'; import initialState from '../initial_state'; +import { fetchCustomEmojis } from '../actions/custom_emojis'; const { localeData, messages } = getLocale(); addLocaleData(localeData); @@ -17,6 +18,8 @@ if (initialState) { store.dispatch(hydrateStore(initialState)); } +store.dispatch(fetchCustomEmojis()); + export default class TimelineContainer extends React.PureComponent { static propTypes = { From cd36ff43fd214b6b1447e88457b5a3f56461c1dd Mon Sep 17 00:00:00 2001 From: trwnh Date: Mon, 4 Feb 2019 21:46:18 -0600 Subject: [PATCH 07/42] [UI] Fix whitespace being applied to div instead of p (#9968) * fix large line breaks * fix ascii art posts --- app/javascript/styles/mastodon/components.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 10e094648..32fd77385 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -638,7 +638,6 @@ font-weight: 400; overflow: hidden; text-overflow: ellipsis; - white-space: pre-wrap; padding-top: 2px; color: $primary-text-color; @@ -662,6 +661,7 @@ p { margin-bottom: 20px; + white-space: pre-wrap; &:last-child { margin-bottom: 0; From edde07f5ab235d6ceca95db66bb53161d372f830 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 5 Feb 2019 15:11:35 +0100 Subject: [PATCH 08/42] =?UTF-8?q?Hide=20misleading=20=E2=80=9CYou=20will?= =?UTF-8?q?=20be=20sent=20a=20confirmation=20e-mail=E2=80=9D=20hint=20from?= =?UTF-8?q?=20admin=20view=20(#9973)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thanks @wryk for noticing this issue. --- app/views/admin/change_emails/show.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/admin/change_emails/show.html.haml b/app/views/admin/change_emails/show.html.haml index 6febef9b1..6ff0d785e 100644 --- a/app/views/admin/change_emails/show.html.haml +++ b/app/views/admin/change_emails/show.html.haml @@ -3,7 +3,7 @@ = simple_form_for @user, url: admin_account_change_email_path(@account.id) do |f| .fields-group - = f.input :email, wrapper: :with_label, disabled: true, label: t('admin.accounts.change_email.current_email') + = f.input :email, wrapper: :with_label, hint: false, disabled: true, label: t('admin.accounts.change_email.current_email') .fields-group = f.input :unconfirmed_email, wrapper: :with_label, label: t('admin.accounts.change_email.new_email') From e2afe5fdfba352b35731acf028bad54f32223282 Mon Sep 17 00:00:00 2001 From: abcang Date: Wed, 6 Feb 2019 10:50:52 +0900 Subject: [PATCH 09/42] Fix Tombstone.delete_all ArgumentError (#9978) --- app/services/activitypub/process_account_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index 487456f3a..5e3308428 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -212,7 +212,7 @@ class ActivityPub::ProcessAccountService < BaseService end def clear_tombstones! - Tombstone.delete_all(account_id: @account.id) + Tombstone.where(account_id: @account.id).delete_all end def protocol_changed? From 2a7c091eae68b06ae4ad7c566878a04f9926ac92 Mon Sep 17 00:00:00 2001 From: Hinaloe Date: Sat, 9 Feb 2019 11:39:38 +0900 Subject: [PATCH 10/42] Only URLs extract with pre-escaped text (#9991) * [test] add japanese hashtag testcase * Only URLs extract with pre-escaped text ( https://github.com/tootsuite/mastodon/issues/9989 ) --- app/lib/formatter.rb | 2 +- spec/lib/formatter_spec.rb | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 2e3587169..6603b8df1 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -210,7 +210,7 @@ class Formatter # Note: I couldn't obtain list_slug with @user/list-name format # for mention so this requires additional check - special = Extractor.extract_entities_with_indices(escaped, options).map do |extract| + special = Extractor.extract_urls_with_indices(escaped, options).map do |extract| # exactly one of :url, :hashtag, :screen_name, :cashtag keys is present key = (extract.keys & [:url, :hashtag, :screen_name, :cashtag]).first diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb index 9872d3756..8fb6695a9 100644 --- a/spec/lib/formatter_spec.rb +++ b/spec/lib/formatter_spec.rb @@ -194,6 +194,14 @@ RSpec.describe Formatter do is_expected.to include '/tags/hashtag" class="mention hashtag" rel="tag">#hashtag' end end + + context 'given text containing a hashtag with Unicode chars' do + let(:text) { '#hashtagタグ' } + + it 'creates a hashtag link' do + is_expected.to include '/tags/hashtag%E3%82%BF%E3%82%B0" class="mention hashtag" rel="tag">#hashtagタグ' + end + end end describe '#format_spoiler' do From 6ea4cd5b86b0f96bfa9cfc904ee2c1adb48079a4 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 9 Feb 2019 20:13:11 +0100 Subject: [PATCH 11/42] Fix URL linkifier grabbing full-width spaces and quotations (#9997) Fix #9993 Fix #5654 --- app/lib/formatter.rb | 12 +++++++- config/initializers/twitter_regex.rb | 4 +-- spec/lib/formatter_spec.rb | 44 ++++++++++++++++++++++++++-- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 6603b8df1..0653214f5 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -199,12 +199,22 @@ class Formatter result.flatten.join end + UNICODE_ESCAPE_BLACKLIST_RE = /\p{Z}|\p{P}/ + def utf8_friendly_extractor(text, options = {}) old_to_new_index = [0] escaped = text.chars.map do |c| - output = c.ord.to_s(16).length > 2 ? CGI.escape(c) : c + output = begin + if c.ord.to_s(16).length > 2 && UNICODE_ESCAPE_BLACKLIST_RE.match(c).nil? + CGI.escape(c) + else + c + end + end + old_to_new_index << old_to_new_index.last + output.length + output end.join diff --git a/config/initializers/twitter_regex.rb b/config/initializers/twitter_regex.rb index 0e8f5bfeb..0ddbbee98 100644 --- a/config/initializers/twitter_regex.rb +++ b/config/initializers/twitter_regex.rb @@ -1,7 +1,7 @@ module Twitter class Regex - REGEXEN[:valid_general_url_path_chars] = /[^\p{White_Space}\(\)\?]/iou - REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}\(\)\?!\*';:=\,\.\$%\[\]~&\|@]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou + REGEXEN[:valid_general_url_path_chars] = /[^\p{White_Space}<>\(\)\?]/iou + REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}\(\)\?!\*"'「」<>;:=\,\.\$%\[\]~&\|@]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou REGEXEN[:valid_url_balanced_parens] = / \( (?: diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb index 8fb6695a9..96d2fc7e0 100644 --- a/spec/lib/formatter_spec.rb +++ b/spec/lib/formatter_spec.rb @@ -115,6 +115,22 @@ RSpec.describe Formatter do end end + context 'given a URL in quotation marks' do + let(:text) { '"https://example.com/"' } + + it 'does not match the quotation marks' do + is_expected.to include 'href="https://example.com/"' + end + end + + context 'given a URL in angle brackets' do + let(:text) { '' } + + it 'does not match the angle brackets' do + is_expected.to include 'href="https://example.com/"' + end + end + context 'given a URL with Japanese path string' do let(:text) { 'https://ja.wikipedia.org/wiki/日本' } @@ -131,6 +147,22 @@ RSpec.describe Formatter do end end + context 'given a URL with a full-width space' do + let(:text) { 'https://example.com/ abc123' } + + it 'does not match the full-width space' do + is_expected.to include 'href="https://example.com/"' + end + end + + context 'given a URL in Japanese quotation marks' do + let(:text) { '「[https://example.org/」' } + + it 'does not match the quotation marks' do + is_expected.to include 'href="https://example.org/"' + end + end + context 'given a URL with Simplified Chinese path string' do let(:text) { 'https://baike.baidu.com/item/中华人民共和国' } @@ -150,7 +182,11 @@ RSpec.describe Formatter do context 'given a URL containing unsafe code (XSS attack, visible part)' do let(:text) { %q{http://example.com/bb} } - it 'escapes the HTML in the URL' do + it 'does not include the HTML in the URL' do + is_expected.to include '"http://example.com/b"' + end + + it 'escapes the HTML' do is_expected.to include '<del>b</del>' end end @@ -158,7 +194,11 @@ RSpec.describe Formatter do context 'given a URL containing unsafe code (XSS attack, invisible part)' do let(:text) { %q{http://example.com/blahblahblahblah/a} } - it 'escapes the HTML in the URL' do + it 'does not include the HTML in the URL' do + is_expected.to include '"http://example.com/blahblahblahblah/a"' + end + + it 'escapes the HTML' do is_expected.to include '<script>alert("Hello")</script>' end end From d9f0c7fb841bebfa942ddc5b7aae2857eb1381e3 Mon Sep 17 00:00:00 2001 From: ThibG Date: Wed, 6 Feb 2019 23:36:43 +0100 Subject: [PATCH 12/42] Fix IntersectionObserverArticle not hiding some out-of-view items (#9982) IntersectionObserverArticle is made to save on RAM by avoiding fully rendering items that are far out of view. However, it did not work for items spawned outside the intersection observer. --- .../mastodon/components/intersection_observer_article.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/javascript/mastodon/components/intersection_observer_article.js b/app/javascript/mastodon/components/intersection_observer_article.js index de2203a4b..e453730ba 100644 --- a/app/javascript/mastodon/components/intersection_observer_article.js +++ b/app/javascript/mastodon/components/intersection_observer_article.js @@ -65,7 +65,7 @@ export default class IntersectionObserverArticle extends React.Component { } updateStateAfterIntersection = (prevState) => { - if (prevState.isIntersecting && !this.entry.isIntersecting) { + if (prevState.isIntersecting !== false && !this.entry.isIntersecting) { scheduleIdleTask(this.hideIfNotIntersecting); } return { From e1dbdf7377f20ace894ee92cf681542cc4b5eddb Mon Sep 17 00:00:00 2001 From: ThibG Date: Mon, 11 Feb 2019 13:19:59 +0100 Subject: [PATCH 13/42] Fix timeline jumps (#10001) * Avoid two-step rendering of statuses as much as possible Cache width shared by Video player, MediaGallery and Cards at the ScrollableList level, pass it down through StatusList and Notifications. * Adjust scroll when new preview cards appear * Adjust scroll when statuses above the current scroll position are deleted --- .../mastodon/components/media_gallery.js | 10 ++- .../mastodon/components/scrollable_list.js | 28 +++++++- app/javascript/mastodon/components/status.js | 67 +++++++++++++++++-- .../notifications/components/notification.js | 32 ++++++++- .../features/status/components/card.js | 5 +- .../mastodon/features/video/index.js | 4 +- 6 files changed, 134 insertions(+), 12 deletions(-) diff --git a/app/javascript/mastodon/components/media_gallery.js b/app/javascript/mastodon/components/media_gallery.js index c507920d0..a2bc95255 100644 --- a/app/javascript/mastodon/components/media_gallery.js +++ b/app/javascript/mastodon/components/media_gallery.js @@ -194,6 +194,8 @@ class MediaGallery extends React.PureComponent { height: PropTypes.number.isRequired, onOpenMedia: PropTypes.func.isRequired, intl: PropTypes.object.isRequired, + defaultWidth: PropTypes.number, + cacheWidth: PropTypes.func, }; static defaultProps = { @@ -202,6 +204,7 @@ class MediaGallery extends React.PureComponent { state = { visible: displayMedia !== 'hide_all' && !this.props.sensitive || displayMedia === 'show_all', + width: this.props.defaultWidth, }; componentWillReceiveProps (nextProps) { @@ -221,6 +224,7 @@ class MediaGallery extends React.PureComponent { handleRef = (node) => { if (node /*&& this.isStandaloneEligible()*/) { // offsetWidth triggers a layout, so only calculate when we need to + if (this.props.cacheWidth) this.props.cacheWidth(node.offsetWidth); this.setState({ width: node.offsetWidth, }); @@ -233,8 +237,10 @@ class MediaGallery extends React.PureComponent { } render () { - const { media, intl, sensitive, height } = this.props; - const { width, visible } = this.state; + const { media, intl, sensitive, height, defaultWidth } = this.props; + const { visible } = this.state; + + const width = this.state.width || defaultWidth; let children; diff --git a/app/javascript/mastodon/components/scrollable_list.js b/app/javascript/mastodon/components/scrollable_list.js index fec06e263..0376cf85a 100644 --- a/app/javascript/mastodon/components/scrollable_list.js +++ b/app/javascript/mastodon/components/scrollable_list.js @@ -40,6 +40,7 @@ export default class ScrollableList extends PureComponent { state = { fullscreen: null, + cachedMediaWidth: 250, // Default media/card width using default Mastodon theme }; intersectionObserverWrapper = new IntersectionObserverWrapper(); @@ -130,6 +131,20 @@ export default class ScrollableList extends PureComponent { this.handleScroll(); } + getScrollPosition = () => { + if (this.node && (this.node.scrollTop > 0 || this.mouseMovedRecently)) { + return { height: this.node.scrollHeight, top: this.node.scrollTop }; + } else { + return null; + } + } + + updateScrollBottom = (snapshot) => { + const newScrollTop = this.node.scrollHeight - snapshot; + + this.setScrollTop(newScrollTop); + } + getSnapshotBeforeUpdate (prevProps) { const someItemInserted = React.Children.count(prevProps.children) > 0 && React.Children.count(prevProps.children) < React.Children.count(this.props.children) && @@ -150,6 +165,12 @@ export default class ScrollableList extends PureComponent { } } + cacheMediaWidth = (width) => { + if (width && this.state.cachedMediaWidth !== width) { + this.setState({ cachedMediaWidth: width }); + } + } + componentWillUnmount () { this.clearMouseIdleTimer(); this.detachScrollListener(); @@ -239,7 +260,12 @@ export default class ScrollableList extends PureComponent { intersectionObserverWrapper={this.intersectionObserverWrapper} saveHeightKey={trackScroll ? `${this.context.router.route.location.key}:${scrollKey}` : null} > - {child} + {React.cloneElement(child, { + getScrollPosition: this.getScrollPosition, + updateScrollBottom: this.updateScrollBottom, + cachedMediaWidth: this.state.cachedMediaWidth, + cacheMediaWidth: this.cacheMediaWidth, + })} ))} diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js index 20d838500..fde8e60a4 100644 --- a/app/javascript/mastodon/components/status.js +++ b/app/javascript/mastodon/components/status.js @@ -68,6 +68,10 @@ class Status extends ImmutablePureComponent { onMoveUp: PropTypes.func, onMoveDown: PropTypes.func, showThread: PropTypes.bool, + getScrollPosition: PropTypes.func, + updateScrollBottom: PropTypes.func, + cacheMediaWidth: PropTypes.func, + cachedMediaWidth: PropTypes.number, }; // Avoid checking props that are functions (and whose equality will always @@ -79,6 +83,43 @@ class Status extends ImmutablePureComponent { 'hidden', ]; + // Track height changes we know about to compensate scrolling + componentDidMount () { + this.didShowCard = !this.props.muted && !this.props.hidden && this.props.status.get('card'); + } + + getSnapshotBeforeUpdate () { + if (this.props.getScrollPosition) { + return this.props.getScrollPosition(); + } else { + return null; + } + } + + // Compensate height changes + componentDidUpdate (prevProps, prevState, snapshot) { + const doShowCard = !this.props.muted && !this.props.hidden && this.props.status.get('card'); + if (doShowCard && !this.didShowCard) { + this.didShowCard = true; + if (snapshot !== null && this.props.updateScrollBottom) { + if (this.node && this.node.offsetTop < snapshot.top) { + this.props.updateScrollBottom(snapshot.height - snapshot.top); + } + } + } + } + + componentWillUnmount() { + if (this.node && this.props.getScrollPosition) { + const position = this.props.getScrollPosition(); + if (position !== null && this.node.offsetTop < position.top) { + requestAnimationFrame(() => { + this.props.updateScrollBottom(position.height - position.top); + }); + } + } + } + handleClick = () => { if (this.props.onClick) { this.props.onClick(); @@ -165,6 +206,10 @@ class Status extends ImmutablePureComponent { } } + handleRef = c => { + this.node = c; + } + render () { let media = null; let statusAvatar, prepend, rebloggedByText; @@ -179,7 +224,7 @@ class Status extends ImmutablePureComponent { if (hidden) { return ( -
+
{status.getIn(['account', 'display_name']) || status.getIn(['account', 'username'])} {status.get('content')}
@@ -194,7 +239,7 @@ class Status extends ImmutablePureComponent { return ( -
+
@@ -242,11 +287,12 @@ class Status extends ImmutablePureComponent { preview={video.get('preview_url')} src={video.get('url')} alt={video.get('description')} - width={239} + width={this.props.cachedMediaWidth} height={110} inline sensitive={status.get('sensitive')} onOpenVideo={this.handleOpenVideo} + cacheWidth={this.props.cacheMediaWidth} /> )} @@ -254,7 +300,16 @@ class Status extends ImmutablePureComponent { } else { media = ( - {Component => } + {Component => ( + + )} ); } @@ -264,6 +319,8 @@ class Status extends ImmutablePureComponent { onOpenMedia={this.props.onOpenMedia} card={status.get('card')} compact + cacheWidth={this.props.cacheMediaWidth} + defaultWidth={this.props.cachedMediaWidth} /> ); } @@ -290,7 +347,7 @@ class Status extends ImmutablePureComponent { return ( -
+
{prepend}
diff --git a/app/javascript/mastodon/features/notifications/components/notification.js b/app/javascript/mastodon/features/notifications/components/notification.js index 97efff69c..24843a2b6 100644 --- a/app/javascript/mastodon/features/notifications/components/notification.js +++ b/app/javascript/mastodon/features/notifications/components/notification.js @@ -34,6 +34,10 @@ class Notification extends ImmutablePureComponent { onToggleHidden: PropTypes.func.isRequired, status: PropTypes.option, intl: PropTypes.object.isRequired, + getScrollPosition: PropTypes.func, + updateScrollBottom: PropTypes.func, + cacheMediaWidth: PropTypes.func, + cachedMediaWidth: PropTypes.number, }; handleMoveUp = () => { @@ -128,6 +132,10 @@ class Notification extends ImmutablePureComponent { onMoveDown={this.handleMoveDown} onMoveUp={this.handleMoveUp} contextType='notifications' + getScrollPosition={this.props.getScrollPosition} + updateScrollBottom={this.props.updateScrollBottom} + cachedMediaWidth={this.props.cachedMediaWidth} + cacheMediaWidth={this.props.cacheMediaWidth} /> ); } @@ -148,7 +156,17 @@ class Notification extends ImmutablePureComponent {
-
); @@ -170,7 +188,17 @@ class Notification extends ImmutablePureComponent {
-
); diff --git a/app/javascript/mastodon/features/status/components/card.js b/app/javascript/mastodon/features/status/components/card.js index 8491299ef..30ed3fc21 100644 --- a/app/javascript/mastodon/features/status/components/card.js +++ b/app/javascript/mastodon/features/status/components/card.js @@ -60,6 +60,8 @@ export default class Card extends React.PureComponent { maxDescription: PropTypes.number, onOpenMedia: PropTypes.func.isRequired, compact: PropTypes.bool, + defaultWidth: PropTypes.number, + cacheWidth: PropTypes.func, }; static defaultProps = { @@ -68,7 +70,7 @@ export default class Card extends React.PureComponent { }; state = { - width: 280, + width: this.props.defaultWidth || 280, embedded: false, }; @@ -111,6 +113,7 @@ export default class Card extends React.PureComponent { setRef = c => { if (c) { + if (this.props.cacheWidth) this.props.cacheWidth(c.offsetWidth); this.setState({ width: c.offsetWidth }); } } diff --git a/app/javascript/mastodon/features/video/index.js b/app/javascript/mastodon/features/video/index.js index 0d0c24d71..823bc25ab 100644 --- a/app/javascript/mastodon/features/video/index.js +++ b/app/javascript/mastodon/features/video/index.js @@ -99,6 +99,7 @@ class Video extends React.PureComponent { onCloseVideo: PropTypes.func, detailed: PropTypes.bool, inline: PropTypes.bool, + cacheWidth: PropTypes.func, intl: PropTypes.object.isRequired, }; @@ -108,7 +109,7 @@ class Video extends React.PureComponent { volume: 0.5, paused: true, dragging: false, - containerWidth: false, + containerWidth: this.props.width, fullscreen: false, hovered: false, muted: false, @@ -128,6 +129,7 @@ class Video extends React.PureComponent { this.player = c; if (c) { + if (this.props.cacheWidth) this.props.cacheWidth(this.player.offsetWidth); this.setState({ containerWidth: c.offsetWidth, }); From 41ecf80645d465d67dfee54d3cbb14a825ed8953 Mon Sep 17 00:00:00 2001 From: Hinaloe Date: Tue, 12 Feb 2019 13:10:31 +0900 Subject: [PATCH 14/42] Don't focus spiler input when disabled spoiler (#10017) --- .../mastodon/features/compose/components/compose_form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/javascript/mastodon/features/compose/components/compose_form.js b/app/javascript/mastodon/features/compose/components/compose_form.js index ac458fd25..294dcb5c1 100644 --- a/app/javascript/mastodon/features/compose/components/compose_form.js +++ b/app/javascript/mastodon/features/compose/components/compose_form.js @@ -179,7 +179,7 @@ class ComposeForm extends ImmutablePureComponent {
From d66267508ac316ffc2756a6677fcbae5a44605c2 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 12 Feb 2019 05:10:43 +0100 Subject: [PATCH 15/42] =?UTF-8?q?Move=20sending=20account=20Delete=20to=20?= =?UTF-8?q?anyone=20but=20the=20account's=20followers=20to=20the=20pull?= =?UTF-8?q?=CC=80=20queue=20(#10016)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/services/suspend_account_service.rb | 10 +++++++++- .../activitypub/low_priority_delivery_worker.rb | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 app/workers/activitypub/low_priority_delivery_worker.rb diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 1bc2314de..fc3bc03a5 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -102,6 +102,10 @@ class SuspendAccountService < BaseService ActivityPub::DeliveryWorker.push_bulk(delivery_inboxes) do |inbox_url| [delete_actor_json, @account.id, inbox_url] end + + ActivityPub::LowPriorityDeliveryWorker.push_bulk(low_priority_delivery_inboxes) do |inbox_url| + [delete_actor_json, @account.id, inbox_url] + end end def delete_actor_json @@ -117,7 +121,11 @@ class SuspendAccountService < BaseService end def delivery_inboxes - Account.inboxes + Relay.enabled.pluck(:inbox_url) + @delivery_inboxes ||= @account.followers.inboxes + Relay.enabled.pluck(:inbox_url) + end + + def low_priority_delivery_inboxes + Account.inboxes - delivery_inboxes end def associations_for_destruction diff --git a/app/workers/activitypub/low_priority_delivery_worker.rb b/app/workers/activitypub/low_priority_delivery_worker.rb new file mode 100644 index 000000000..a141b8f78 --- /dev/null +++ b/app/workers/activitypub/low_priority_delivery_worker.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class ActivityPub::LowPriorityDeliveryWorker < ActivityPub::DeliveryWorker + sidekiq_options queue: 'pull', retry: 8, dead: false +end From 27310a84a43623da8f08832e30e078ce936f6a1d Mon Sep 17 00:00:00 2001 From: Franck Zoccolo Date: Tue, 12 Feb 2019 14:48:04 +0100 Subject: [PATCH 16/42] Add support for IPv6 only MXes in Email validation (#10009) * Add support for IPv6 only MXes * Fixed email validator tests --- app/validators/email_mx_validator.rb | 1 + spec/validators/email_mx_validator_spec.rb | 38 ++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/app/validators/email_mx_validator.rb b/app/validators/email_mx_validator.rb index 5b4c684b2..96fbedcfc 100644 --- a/app/validators/email_mx_validator.rb +++ b/app/validators/email_mx_validator.rb @@ -24,6 +24,7 @@ class EmailMxValidator < ActiveModel::Validator ([domain] + hostnames).uniq.each do |hostname| ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::A).to_a.map { |e| e.address.to_s }) + ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::AAAA).to_a.map { |e| e.address.to_s }) end end diff --git a/spec/validators/email_mx_validator_spec.rb b/spec/validators/email_mx_validator_spec.rb index bc68f63cf..48e17a4f1 100644 --- a/spec/validators/email_mx_validator_spec.rb +++ b/spec/validators/email_mx_validator_spec.rb @@ -11,6 +11,7 @@ describe EmailMxValidator do allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) allow(resolver).to receive(:timeouts=).and_return(nil) allow(Resolv::DNS).to receive(:open).and_yield(resolver) @@ -23,7 +24,9 @@ describe EmailMxValidator do allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) allow(resolver).to receive(:timeouts=).and_return(nil) allow(Resolv::DNS).to receive(:open).and_yield(resolver) @@ -37,6 +40,21 @@ describe EmailMxValidator do allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([double(address: '1.2.3.4')]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) + allow(resolver).to receive(:timeouts=).and_return(nil) + allow(Resolv::DNS).to receive(:open).and_yield(resolver) + + subject.validate(user) + expect(user.errors).to have_received(:add) + end + + it 'adds an error if the AAAA record is blacklisted' do + EmailDomainBlock.create!(domain: 'fd00::1') + resolver = double + + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([double(address: 'fd00::1')]) allow(resolver).to receive(:timeouts=).and_return(nil) allow(Resolv::DNS).to receive(:open).and_yield(resolver) @@ -50,7 +68,25 @@ describe EmailMxValidator do allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::A).and_return([double(address: '2.3.4.5')]) + allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) + allow(resolver).to receive(:timeouts=).and_return(nil) + allow(Resolv::DNS).to receive(:open).and_yield(resolver) + + subject.validate(user) + expect(user.errors).to have_received(:add) + end + + it 'adds an error if the MX IPv6 record is blacklisted' do + EmailDomainBlock.create!(domain: 'fd00::2') + resolver = double + + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) + allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::AAAA).and_return([double(address: 'fd00::2')]) allow(resolver).to receive(:timeouts=).and_return(nil) allow(Resolv::DNS).to receive(:open).and_yield(resolver) @@ -64,7 +100,9 @@ describe EmailMxValidator do allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) + allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::A).and_return([double(address: '2.3.4.5')]) + allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::AAAA).and_return([double(address: 'fd00::2')]) allow(resolver).to receive(:timeouts=).and_return(nil) allow(Resolv::DNS).to receive(:open).and_yield(resolver) From 1186b9abebc0b92acecebcd020d2f9c17de5d0b3 Mon Sep 17 00:00:00 2001 From: ThibG Date: Tue, 12 Feb 2019 22:24:14 +0100 Subject: [PATCH 17/42] Save IP address used for sign-up, not only sign-in (#10026) Fixes #9995 --- app/controllers/auth/registrations_controller.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/auth/registrations_controller.rb b/app/controllers/auth/registrations_controller.rb index f2a832542..ad7b1859f 100644 --- a/app/controllers/auth/registrations_controller.rb +++ b/app/controllers/auth/registrations_controller.rb @@ -28,6 +28,7 @@ class Auth::RegistrationsController < Devise::RegistrationsController resource.invite_code = params[:invite_code] if resource.invite_code.blank? resource.agreement = true + resource.current_sign_in_ip = request.remote_ip if resource.current_sign_in_ip.nil? resource.build_account if resource.account.nil? end From f0f657e77c65923e6d77b5f62f7ee8544b4d9e00 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 13 Feb 2019 05:30:49 +0100 Subject: [PATCH 18/42] Fix color of static page links in high contrast theme (#10028) --- app/javascript/styles/contrast/diff.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/javascript/styles/contrast/diff.scss b/app/javascript/styles/contrast/diff.scss index 7d8993a50..8429103b8 100644 --- a/app/javascript/styles/contrast/diff.scss +++ b/app/javascript/styles/contrast/diff.scss @@ -13,6 +13,10 @@ } } +.rich-formatting a, +.rich-formatting p a, +.rich-formatting li a, +.landing-page__short-description p a, .status__content a, .reply-indicator__content a { color: lighten($ui-highlight-color, 12%); From a46487e895fbba23922888820c6b88ad07ebe56b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 13 Feb 2019 18:34:58 +0100 Subject: [PATCH 19/42] Fix hashtags select styling in default and high contrast themes (#10029) --- .../components/column_settings.js | 55 ++++++----- app/javascript/styles/mastodon/_mixins.scss | 31 ++++++ .../styles/mastodon/components.scss | 95 +++++++++++++------ 3 files changed, 128 insertions(+), 53 deletions(-) diff --git a/app/javascript/mastodon/features/hashtag_timeline/components/column_settings.js b/app/javascript/mastodon/features/hashtag_timeline/components/column_settings.js index 9c9f62d82..cdc138c8b 100644 --- a/app/javascript/mastodon/features/hashtag_timeline/components/column_settings.js +++ b/app/javascript/mastodon/features/hashtag_timeline/components/column_settings.js @@ -1,10 +1,15 @@ import React from 'react'; import PropTypes from 'prop-types'; import ImmutablePropTypes from 'react-immutable-proptypes'; -import { injectIntl, FormattedMessage } from 'react-intl'; +import { defineMessages, injectIntl, FormattedMessage } from 'react-intl'; import Toggle from 'react-toggle'; import AsyncSelect from 'react-select/lib/Async'; +const messages = defineMessages({ + placeholder: { id: 'hashtag.column_settings.select.placeholder', defaultMessage: 'Enter hashtags…' }, + noOptions: { id: 'hashtag.column_settings.select.no_options_message', defaultMessage: 'No suggestions found' }, +}); + export default @injectIntl class ColumnSettings extends React.PureComponent { @@ -25,6 +30,7 @@ class ColumnSettings extends React.PureComponent { tags (mode) { let tags = this.props.settings.getIn(['tags', mode]) || []; + if (tags.toJSON) { return tags.toJSON(); } else { @@ -32,33 +38,36 @@ class ColumnSettings extends React.PureComponent { } }; - onSelect = (mode) => { - return (value) => { - this.props.onChange(['tags', mode], value); - }; - }; + onSelect = mode => value => this.props.onChange(['tags', mode], value); onToggle = () => { if (this.state.open && this.hasTags()) { this.props.onChange('tags', {}); } + this.setState({ open: !this.state.open }); }; + noOptionsMessage = () => this.props.intl.formatMessage(messages.noOptions); + modeSelect (mode) { return ( -
- {this.modeLabel(mode)} +
+ + {this.modeLabel(mode)} + +
); @@ -66,11 +75,15 @@ class ColumnSettings extends React.PureComponent { modeLabel (mode) { switch(mode) { - case 'any': return ; - case 'all': return ; - case 'none': return ; + case 'any': + return ; + case 'all': + return ; + case 'none': + return ; + default: + return ''; } - return ''; }; render () { @@ -78,23 +91,21 @@ class ColumnSettings extends React.PureComponent {
- + +
- {this.state.open && + + {this.state.open && (
{this.modeSelect('any')} {this.modeSelect('all')} {this.modeSelect('none')}
- } + )}
); } diff --git a/app/javascript/styles/mastodon/_mixins.scss b/app/javascript/styles/mastodon/_mixins.scss index d5bafe6b6..08806599e 100644 --- a/app/javascript/styles/mastodon/_mixins.scss +++ b/app/javascript/styles/mastodon/_mixins.scss @@ -41,3 +41,34 @@ font-size: 16px; } } + +@mixin search-popout() { + background: $simple-background-color; + border-radius: 4px; + padding: 10px 14px; + padding-bottom: 14px; + margin-top: 10px; + color: $light-text-color; + box-shadow: 2px 4px 15px rgba($base-shadow-color, 0.4); + + h4 { + text-transform: uppercase; + color: $light-text-color; + font-size: 13px; + font-weight: 500; + margin-bottom: 10px; + } + + li { + padding: 4px 0; + } + + ul { + margin-bottom: 10px; + } + + em { + font-weight: 500; + color: $inverted-text-color; + } +} diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index 32fd77385..02bbd345a 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -3056,14 +3056,41 @@ a.status-card.compact:hover { display: block; font-weight: 500; margin-bottom: 10px; +} - .column-settings__hashtag-select { +.column-settings__hashtags { + .column-settings__row { + margin-bottom: 15px; + } + + .column-select { &__control { @include search-input(); } + &__placeholder { + color: $dark-text-color; + padding-left: 2px; + font-size: 12px; + } + + &__value-container { + padding-left: 6px; + } + &__multi-value { background: lighten($ui-base-color, 8%); + + &__remove { + cursor: pointer; + + &:hover, + &:active, + &:focus { + background: lighten($ui-base-color, 12%); + color: lighten($darker-text-color, 4%); + } + } } &__multi-value__label, @@ -3071,9 +3098,42 @@ a.status-card.compact:hover { color: $darker-text-color; } - &__indicator-separator, + &__clear-indicator, &__dropdown-indicator { - display: none; + cursor: pointer; + transition: none; + color: $dark-text-color; + + &:hover, + &:active, + &:focus { + color: lighten($dark-text-color, 4%); + } + } + + &__indicator-separator { + background-color: lighten($ui-base-color, 8%); + } + + &__menu { + @include search-popout(); + padding: 0; + background: $ui-secondary-color; + } + + &__menu-list { + padding: 6px; + } + + &__option { + color: $inverted-text-color; + border-radius: 4px; + font-size: 14px; + + &--is-focused, + &--is-selected { + background: darken($ui-secondary-color, 10%); + } } } } @@ -4867,34 +4927,7 @@ a.status-card.compact:hover { } .search-popout { - background: $simple-background-color; - border-radius: 4px; - padding: 10px 14px; - padding-bottom: 14px; - margin-top: 10px; - color: $light-text-color; - box-shadow: 2px 4px 15px rgba($base-shadow-color, 0.4); - - h4 { - text-transform: uppercase; - color: $light-text-color; - font-size: 13px; - font-weight: 500; - margin-bottom: 10px; - } - - li { - padding: 4px 0; - } - - ul { - margin-bottom: 10px; - } - - em { - font-weight: 500; - color: $inverted-text-color; - } + @include search-popout(); } noscript { From 2a1adab7d7824df9fa148a9431e942d6677c1d71 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 13 Feb 2019 18:36:40 +0100 Subject: [PATCH 20/42] Fix style regressions on landing page (#10030) --- app/javascript/styles/mastodon/about.scss | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/app/javascript/styles/mastodon/about.scss b/app/javascript/styles/mastodon/about.scss index b6c92a09e..b078d4d24 100644 --- a/app/javascript/styles/mastodon/about.scss +++ b/app/javascript/styles/mastodon/about.scss @@ -49,15 +49,9 @@ $small-breakpoint: 960px; } } + strong, em { - display: inline; - margin: 0; - padding: 0; font-weight: 700; - background: transparent; - font-family: inherit; - font-size: inherit; - line-height: inherit; color: lighten($darker-text-color, 10%); } @@ -796,7 +790,7 @@ $small-breakpoint: 960px; width: 100%; display: flex; flex-direction: row-reverse; - flex-wrap: wrap; + flex-wrap: nowrap; justify-content: space-between; align-items: center; } @@ -846,14 +840,7 @@ $small-breakpoint: 960px; } strong { - display: inline; - margin: 0; - padding: 0; - font-weight: 700; - background: transparent; - font-family: inherit; - font-size: inherit; - line-height: inherit; + font-weight: 500; color: lighten($darker-text-color, 10%); } From 5a04861c7f2e98c80f315a19d7eadade044d8aae Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 14 Feb 2019 06:27:54 +0100 Subject: [PATCH 21/42] Add tight rate-limit for API deletions (#10042) Deletions take a lot of resources to execute and cause a lot of federation traffic, so it makes sense to decrease the number someone can queue up through the API. 30 per 30 minutes --- config/initializers/rack_attack.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb index 35302e37b..28201cc64 100644 --- a/config/initializers/rack_attack.rb +++ b/config/initializers/rack_attack.rb @@ -46,14 +46,14 @@ class Rack::Attack end throttle('throttle_authenticated_api', limit: 300, period: 5.minutes) do |req| - req.api_request? && req.authenticated_user_id + req.authenticated_user_id if req.api_request? end throttle('throttle_unauthenticated_api', limit: 7_500, period: 5.minutes) do |req| req.ip if req.api_request? end - throttle('throttle_media', limit: 30, period: 30.minutes) do |req| + throttle('throttle_api_media', limit: 30, period: 30.minutes) do |req| req.authenticated_user_id if req.post? && req.path.start_with?('/api/v1/media') end @@ -61,6 +61,13 @@ class Rack::Attack req.ip if req.post? && req.path == '/api/v1/accounts' end + API_DELETE_REBLOG_REGEX = /\A\/api\/v1\/statuses\/[\d]+\/unreblog/.freeze + API_DELETE_STATUS_REGEX = /\A\/api\/v1\/statuses\/[\d]+/.freeze + + throttle('throttle_api_delete', limit: 30, period: 30.minutes) do |req| + req.authenticated_user_id if (req.post? && req.path =~ API_DELETE_REBLOG_REGEX) || (req.delete? && req.path =~ API_DELETE_STATUS_REGEX) + end + throttle('protected_paths', limit: 25, period: 5.minutes) do |req| req.ip if req.post? && req.path =~ PROTECTED_PATHS_REGEX end From 17a41e1f779874a270f531086f5576affcdb6cb1 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 14 Feb 2019 15:46:42 +0100 Subject: [PATCH 22/42] Fix hashtag column not subscribing to stream on mount (#10040) Fix #9895 --- .../mastodon/features/hashtag_timeline/index.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/app/javascript/mastodon/features/hashtag_timeline/index.js b/app/javascript/mastodon/features/hashtag_timeline/index.js index c2e026d13..0d3c97a64 100644 --- a/app/javascript/mastodon/features/hashtag_timeline/index.js +++ b/app/javascript/mastodon/features/hashtag_timeline/index.js @@ -41,15 +41,19 @@ class HashtagTimeline extends React.PureComponent { title = () => { let title = [this.props.params.id]; + if (this.additionalFor('any')) { - title.push(' ', ); + title.push(' ', ); } + if (this.additionalFor('all')) { - title.push(' ', ); + title.push(' ', ); } + if (this.additionalFor('none')) { - title.push(' ', ); + title.push(' ', ); } + return title; } @@ -77,9 +81,10 @@ class HashtagTimeline extends React.PureComponent { let all = (tags.all || []).map(tag => tag.value); let none = (tags.none || []).map(tag => tag.value); - [id, ...any].map((tag) => { - this.disconnects.push(dispatch(connectHashtagStream(id, tag, (status) => { + [id, ...any].map(tag => { + this.disconnects.push(dispatch(connectHashtagStream(id, tag, status => { let tags = status.tags.map(tag => tag.name); + return all.filter(tag => tags.includes(tag)).length === all.length && none.filter(tag => tags.includes(tag)).length === 0; }))); @@ -95,12 +100,14 @@ class HashtagTimeline extends React.PureComponent { const { dispatch } = this.props; const { id, tags } = this.props.params; + this._subscribe(dispatch, id, tags); dispatch(expandHashtagTimeline(id, { tags })); } componentWillReceiveProps (nextProps) { const { dispatch, params } = this.props; const { id, tags } = nextProps.params; + if (id !== params.id || !isEqual(tags, params.tags)) { this._unsubscribe(); this._subscribe(dispatch, id, tags); From 737ac4b59df77e43bed71abc6de80bc89f893de1 Mon Sep 17 00:00:00 2001 From: ysksn Date: Sun, 3 Feb 2019 03:11:38 +0900 Subject: [PATCH 23/42] Create Redisable#redis (#9633) * Create Redisable * Use #redis instead of Redis.current --- app/lib/activity_tracker.rb | 6 ++---- app/lib/activitypub/activity.rb | 5 +---- app/lib/feed_manager.rb | 9 +++------ app/lib/ostatus/activity/base.rb | 6 ++---- app/lib/potential_friendship_tracker.rb | 8 ++------ app/models/concerns/redisable.rb | 11 +++++++++++ app/models/feed.rb | 6 ++---- app/models/trending_tags.rb | 6 ++---- app/services/batched_remove_status_service.rb | 5 +---- app/services/follow_service.rb | 6 ++---- app/services/post_status_service.rb | 6 ++---- app/services/remove_status_service.rb | 19 ++++++++----------- .../scheduler/feed_cleanup_scheduler.rb | 5 +---- 13 files changed, 39 insertions(+), 59 deletions(-) create mode 100644 app/models/concerns/redisable.rb diff --git a/app/lib/activity_tracker.rb b/app/lib/activity_tracker.rb index 5b4972674..ae3c11b6a 100644 --- a/app/lib/activity_tracker.rb +++ b/app/lib/activity_tracker.rb @@ -4,6 +4,8 @@ class ActivityTracker EXPIRE_AFTER = 90.days.seconds class << self + include Redisable + def increment(prefix) key = [prefix, current_week].join(':') @@ -20,10 +22,6 @@ class ActivityTracker private - def redis - Redis.current - end - def current_week Time.zone.today.cweek end diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 87318fb1c..919678618 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -2,6 +2,7 @@ class ActivityPub::Activity include JsonLdHelper + include Redisable def initialize(json, account, **options) @json = json @@ -70,10 +71,6 @@ class ActivityPub::Activity @object_uri ||= value_or_id(@object) end - def redis - Redis.current - end - def distribute(status) crawl_links(status) diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index f99df33e5..d77cdb3a3 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -4,6 +4,7 @@ require 'singleton' class FeedManager include Singleton + include Redisable MAX_ITEMS = 400 @@ -35,7 +36,7 @@ class FeedManager def unpush_from_home(account, status) return false unless remove_from_feed(:home, account.id, status) - Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) + redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) true end @@ -53,7 +54,7 @@ class FeedManager def unpush_from_list(list, status) return false unless remove_from_feed(:list, list.id, status) - Redis.current.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) + redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) true end @@ -142,10 +143,6 @@ class FeedManager private - def redis - Redis.current - end - def push_update_required?(timeline_id) redis.exists("subscribed:#{timeline_id}") end diff --git a/app/lib/ostatus/activity/base.rb b/app/lib/ostatus/activity/base.rb index c5933f3ad..db70f1998 100644 --- a/app/lib/ostatus/activity/base.rb +++ b/app/lib/ostatus/activity/base.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class OStatus::Activity::Base + include Redisable + def initialize(xml, account = nil, **options) @xml = xml @account = account @@ -66,8 +68,4 @@ class OStatus::Activity::Base Status.find_by(uri: uri) end end - - def redis - Redis.current - end end diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb index 017a9748d..188aa4a27 100644 --- a/app/lib/potential_friendship_tracker.rb +++ b/app/lib/potential_friendship_tracker.rb @@ -11,6 +11,8 @@ class PotentialFriendshipTracker }.freeze class << self + include Redisable + def record(account_id, target_account_id, action) return if account_id == target_account_id @@ -31,11 +33,5 @@ class PotentialFriendshipTracker return [] if account_ids.empty? Account.searchable.where(id: account_ids) end - - private - - def redis - Redis.current - end end end diff --git a/app/models/concerns/redisable.rb b/app/models/concerns/redisable.rb new file mode 100644 index 000000000..c6cf97359 --- /dev/null +++ b/app/models/concerns/redisable.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Redisable + extend ActiveSupport::Concern + + private + + def redis + Redis.current + end +end diff --git a/app/models/feed.rb b/app/models/feed.rb index 5bce88f25..0e8943ff8 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Feed + include Redisable + def initialize(type, id) @type = type @id = id @@ -27,8 +29,4 @@ class Feed def key FeedManager.instance.key(@type, @id) end - - def redis - Redis.current - end end diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb index 3a8be2164..148535c21 100644 --- a/app/models/trending_tags.rb +++ b/app/models/trending_tags.rb @@ -7,6 +7,8 @@ class TrendingTags THRESHOLD = 5 class << self + include Redisable + def record_use!(tag, account, at_time = Time.now.utc) return if disallowed_hashtags.include?(tag.name) || account.silenced? || account.bot? @@ -59,9 +61,5 @@ class TrendingTags @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String @disallowed_hashtags = @disallowed_hashtags.map(&:downcase) end - - def redis - Redis.current - end end end diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index 2e61904fc..cd3b14e08 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -2,6 +2,7 @@ class BatchedRemoveStatusService < BaseService include StreamEntryRenderer + include Redisable # Delete given statuses and reblogs of them # Dispatch PuSH updates of the deleted statuses, but only local ones @@ -109,10 +110,6 @@ class BatchedRemoveStatusService < BaseService end end - def redis - Redis.current - end - def build_xml(stream_entry) return @activity_xml[stream_entry.id] if @activity_xml.key?(stream_entry.id) diff --git a/app/services/follow_service.rb b/app/services/follow_service.rb index 9d36a1449..92d8c864a 100644 --- a/app/services/follow_service.rb +++ b/app/services/follow_service.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class FollowService < BaseService + include Redisable + # Follow a remote user, notify remote user about the follow # @param [Account] source_account From which to follow # @param [String, Account] uri User URI to follow in the form of username@domain (or account record) @@ -67,10 +69,6 @@ class FollowService < BaseService follow end - def redis - Redis.current - end - def build_follow_request_xml(follow_request) OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.follow_request_salmon(follow_request)) end diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 9959bb1fb..686b10c58 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class PostStatusService < BaseService + include Redisable + MIN_SCHEDULE_OFFSET = 5.minutes.freeze # Post a text status update, fetch and notify remote users mentioned @@ -110,10 +112,6 @@ class PostStatusService < BaseService ProcessHashtagsService.new end - def redis - Redis.current - end - def scheduled? @scheduled_at.present? end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 11d28e783..28c5224b0 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -2,6 +2,7 @@ class RemoveStatusService < BaseService include StreamEntryRenderer + include Redisable def call(status, **options) @payload = Oj.dump(event: :delete, payload: status.id.to_s) @@ -55,7 +56,7 @@ class RemoveStatusService < BaseService def remove_from_affected @mentions.map(&:account).select(&:local?).each do |account| - Redis.current.publish("timeline:#{account.id}", @payload) + redis.publish("timeline:#{account.id}", @payload) end end @@ -133,26 +134,22 @@ class RemoveStatusService < BaseService return unless @status.public_visibility? @tags.each do |hashtag| - Redis.current.publish("timeline:hashtag:#{hashtag}", @payload) - Redis.current.publish("timeline:hashtag:#{hashtag}:local", @payload) if @status.local? + redis.publish("timeline:hashtag:#{hashtag}", @payload) + redis.publish("timeline:hashtag:#{hashtag}:local", @payload) if @status.local? end end def remove_from_public return unless @status.public_visibility? - Redis.current.publish('timeline:public', @payload) - Redis.current.publish('timeline:public:local', @payload) if @status.local? + redis.publish('timeline:public', @payload) + redis.publish('timeline:public:local', @payload) if @status.local? end def remove_from_media return unless @status.public_visibility? - Redis.current.publish('timeline:public:media', @payload) - Redis.current.publish('timeline:public:local:media', @payload) if @status.local? - end - - def redis - Redis.current + redis.publish('timeline:public:media', @payload) + redis.publish('timeline:public:local:media', @payload) if @status.local? end end diff --git a/app/workers/scheduler/feed_cleanup_scheduler.rb b/app/workers/scheduler/feed_cleanup_scheduler.rb index cd2273418..bf5e20757 100644 --- a/app/workers/scheduler/feed_cleanup_scheduler.rb +++ b/app/workers/scheduler/feed_cleanup_scheduler.rb @@ -2,6 +2,7 @@ class Scheduler::FeedCleanupScheduler include Sidekiq::Worker + include Redisable sidekiq_options unique: :until_executed, retry: 0 @@ -57,8 +58,4 @@ class Scheduler::FeedCleanupScheduler def feed_manager FeedManager.instance end - - def redis - Redis.current - end end From 6c11f0f8cf91aa4d0b83a0a9c5a92cfa99dcbfdd Mon Sep 17 00:00:00 2001 From: ThibG Date: Wed, 13 Feb 2019 18:36:23 +0100 Subject: [PATCH 24/42] Alternative handling of private self-boosts (#9998) * When self-boosting, embed original toot into Announce serialization * Process unknown self-boosts from Announce object if it is more than an URI * Add some self-boost specs * Only serialize private toots in self-Announces --- app/lib/activitypub/activity.rb | 32 +++++++++++ app/lib/activitypub/activity/announce.rb | 4 +- app/lib/activitypub/activity/create.rb | 15 ------ .../activitypub/activity_serializer.rb | 8 ++- .../lib/activitypub/activity/announce_spec.rb | 53 ++++++++++++++++--- 5 files changed, 86 insertions(+), 26 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 919678618..7e4e19531 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -4,6 +4,9 @@ class ActivityPub::Activity include JsonLdHelper include Redisable + SUPPORTED_TYPES = %w(Note).freeze + CONVERTED_TYPES = %w(Image Video Article Page).freeze + def initialize(json, account, **options) @json = json @account = account @@ -71,6 +74,18 @@ class ActivityPub::Activity @object_uri ||= value_or_id(@object) end + def unsupported_object_type? + @object.is_a?(String) || !(supported_object_type? || converted_object_type?) + end + + def supported_object_type? + equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) + end + + def converted_object_type? + equals_or_includes_any?(@object['type'], CONVERTED_TYPES) + end + def distribute(status) crawl_links(status) @@ -120,6 +135,23 @@ class ActivityPub::Activity redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, uri) end + def status_from_object + # If the status is already known, return it + status = status_from_uri(object_uri) + return status unless status.nil? + + # If the boosted toot is embedded and it is a self-boost, handle it like a Create + unless unsupported_object_type? + actor_id = value_or_id(first_of_value(@object['attributedTo'])) || @account.uri + if actor_id == @account.uri + return ActivityPub::Activity.factory({ 'type' => 'Create', 'actor' => actor_id, 'object' => @object }, @account).perform + end + end + + # If the status is not from the actor, try to fetch it + return fetch_remote_original_status if value_or_id(first_of_value(@json['attributedTo'])) == @account.uri + end + def fetch_remote_original_status if object_uri.start_with?('http') return if ActivityPub::TagManager.instance.local_uri?(object_uri) diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 34d1b7cbd..04afeea20 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -2,9 +2,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def perform - original_status = status_from_uri(object_uri) - original_status ||= fetch_remote_original_status - + original_status = status_from_object return if original_status.nil? || delete_arrived_first?(@json['id']) || !announceable?(original_status) status = Status.find_by(account: @account, reblog: original_status) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index b49657d4b..9a3db51dd 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -1,9 +1,6 @@ # frozen_string_literal: true class ActivityPub::Activity::Create < ActivityPub::Activity - SUPPORTED_TYPES = %w(Note).freeze - CONVERTED_TYPES = %w(Image Video Article Page).freeze - def perform return if unsupported_object_type? || invalid_origin?(@object['id']) return if Tombstone.exists?(uri: @object['id']) @@ -318,22 +315,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity @object['nameMap'].is_a?(Hash) && !@object['nameMap'].empty? end - def unsupported_object_type? - @object.is_a?(String) || !(supported_object_type? || converted_object_type?) - end - def unsupported_media_type?(mime_type) mime_type.present? && !(MediaAttachment::IMAGE_MIME_TYPES + MediaAttachment::VIDEO_MIME_TYPES).include?(mime_type) end - def supported_object_type? - equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) - end - - def converted_object_type? - equals_or_includes_any?(@object['type'], CONVERTED_TYPES) - end - def skip_download? return @skip_download if defined?(@skip_download) @skip_download ||= DomainBlock.find_by(domain: @account.domain)&.reject_media? diff --git a/app/serializers/activitypub/activity_serializer.rb b/app/serializers/activitypub/activity_serializer.rb index 50c4f6a04..b51e8c544 100644 --- a/app/serializers/activitypub/activity_serializer.rb +++ b/app/serializers/activitypub/activity_serializer.rb @@ -3,8 +3,8 @@ class ActivityPub::ActivitySerializer < ActiveModel::Serializer attributes :id, :type, :actor, :published, :to, :cc - has_one :proper, key: :object, serializer: ActivityPub::NoteSerializer, unless: :announce? - attribute :proper_uri, key: :object, if: :announce? + has_one :proper, key: :object, serializer: ActivityPub::NoteSerializer, unless: :owned_announce? + attribute :proper_uri, key: :object, if: :owned_announce? attribute :atom_uri, if: :announce? def id @@ -42,4 +42,8 @@ class ActivityPub::ActivitySerializer < ActiveModel::Serializer def announce? object.reblog? end + + def owned_announce? + announce? && object.account == object.proper.account && object.proper.private_visibility? + end end diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index 54dd52a60..1725c2843 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -1,7 +1,7 @@ require 'rails_helper' RSpec.describe ActivityPub::Activity::Announce do - let(:sender) { Fabricate(:account) } + let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers') } let(:recipient) { Fabricate(:account) } let(:status) { Fabricate(:status, account: recipient) } @@ -11,19 +11,60 @@ RSpec.describe ActivityPub::Activity::Announce do id: 'foo', type: 'Announce', actor: ActivityPub::TagManager.instance.uri_for(sender), - object: ActivityPub::TagManager.instance.uri_for(status), + object: object_json, }.with_indifferent_access end - describe '#perform' do - subject { described_class.new(json, sender) } + subject { described_class.new(json, sender) } + before do + sender.update(uri: ActivityPub::TagManager.instance.uri_for(sender)) + end + + describe '#perform' do before do subject.perform end - it 'creates a reblog by sender of status' do - expect(sender.reblogged?(status)).to be true + context 'a known status' do + let(:object_json) do + ActivityPub::TagManager.instance.uri_for(status) + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(status)).to be true + end + end + + context 'self-boost of a previously unknown status with missing attributedTo' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'http://example.com/followers', + } + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(sender.statuses.first)).to be true + end + end + + context 'self-boost of a previously unknown status with correct attributedTo' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + attributedTo: ActivityPub::TagManager.instance.uri_for(sender), + to: 'http://example.com/followers', + } + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(sender.statuses.first)).to be true + end end end end From ef45411c537d37f06b23914135460544d348bfd6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 13 Feb 2019 18:42:47 +0100 Subject: [PATCH 25/42] Filter incoming Create activities by relation to local activity (#10005) Reject those from accounts with no local followers, from relays that are not enabled, which do not address local accounts and are not replies to accounts that do have local followers --- app/lib/activitypub/activity/create.rb | 34 +++++++++++++++++-- .../activitypub/process_collection_service.rb | 1 + app/workers/activitypub/processing_worker.rb | 2 +- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 9a3db51dd..1b31768d9 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -2,8 +2,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def perform - return if unsupported_object_type? || invalid_origin?(@object['id']) - return if Tombstone.exists?(uri: @object['id']) + return if unsupported_object_type? || invalid_origin?(@object['id']) || Tombstone.exists?(uri: @object['id']) || !related_to_local_activity? RedisLock.acquire(lock_options) do |lock| if lock.acquired? @@ -337,6 +336,37 @@ class ActivityPub::Activity::Create < ActivityPub::Activity !replied_to_status.nil? && replied_to_status.account.local? end + def related_to_local_activity? + fetch? || followed_by_local_accounts? || requested_through_relay? || + responds_to_followed_account? || addresses_local_accounts? + end + + def fetch? + !@options[:delivery] + end + + def followed_by_local_accounts? + @account.passive_relationships.exists? + end + + def requested_through_relay? + @options[:relayed_through_account] && Relay.find_by(inbox_url: @options[:relayed_through_account].inbox_url)&.enabled? + end + + def responds_to_followed_account? + !replied_to_status.nil? && (replied_to_status.account.local? || replied_to_status.account.passive_relationships.exists?) + end + + def addresses_local_accounts? + return true if @options[:delivered_to_account_id] + + local_usernames = (as_array(@object['to']) + as_array(@object['cc'])).uniq.select { |uri| ActivityPub::TagManager.instance.local_uri?(uri) }.map { |uri| ActivityPub::TagManager.instance.uri_to_local_id(uri, :username) } + + return false if local_usernames.empty? + + Account.local.where(username: local_usernames).exists? + end + def forward_for_reply return unless @json['signature'].present? && reply_to_local? ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url]) diff --git a/app/services/activitypub/process_collection_service.rb b/app/services/activitypub/process_collection_service.rb index 5c54aad89..881df478b 100644 --- a/app/services/activitypub/process_collection_service.rb +++ b/app/services/activitypub/process_collection_service.rb @@ -44,6 +44,7 @@ class ActivityPub::ProcessCollectionService < BaseService end def verify_account! + @options[:relayed_through_account] = @account @account = ActivityPub::LinkedDataSignature.new(@json).verify_account! rescue JSON::LD::JsonLdError => e Rails.logger.debug "Could not verify LD-Signature for #{value_or_id(@json['actor'])}: #{e.message}" diff --git a/app/workers/activitypub/processing_worker.rb b/app/workers/activitypub/processing_worker.rb index a8a3ebf0f..a3abe72cf 100644 --- a/app/workers/activitypub/processing_worker.rb +++ b/app/workers/activitypub/processing_worker.rb @@ -6,6 +6,6 @@ class ActivityPub::ProcessingWorker sidekiq_options backtrace: true def perform(account_id, body, delivered_to_account_id = nil) - ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true, delivered_to_account_id: delivered_to_account_id) + ActivityPub::ProcessCollectionService.new.call(body, Account.find(account_id), override_timestamps: true, delivered_to_account_id: delivered_to_account_id, delivery: true) end end From e84c7618192b1ba1538e3c0af79acdf604aea5fc Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 15 Feb 2019 18:19:45 +0100 Subject: [PATCH 26/42] Filter incoming Announce activities by relation to local activity (#10041) * Filter incoming Announce activities by relation to local activity Reject if announcer is not followed by local accounts, and is not from an enabled relay, and the object is not a local status Follow-up to #10005 * Fix tests --- app/lib/activitypub/activity.rb | 14 ++++++++++++++ app/lib/activitypub/activity/announce.rb | 11 ++++++++++- app/lib/activitypub/activity/create.rb | 12 ------------ spec/lib/activitypub/activity/announce_spec.rb | 1 + 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 7e4e19531..3cf38764a 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -138,11 +138,13 @@ class ActivityPub::Activity def status_from_object # If the status is already known, return it status = status_from_uri(object_uri) + return status unless status.nil? # If the boosted toot is embedded and it is a self-boost, handle it like a Create unless unsupported_object_type? actor_id = value_or_id(first_of_value(@object['attributedTo'])) || @account.uri + if actor_id == @account.uri return ActivityPub::Activity.factory({ 'type' => 'Create', 'actor' => actor_id, 'object' => @object }, @account).perform end @@ -166,4 +168,16 @@ class ActivityPub::Activity ensure redis.del(key) end + + def fetch? + !@options[:delivery] + end + + def followed_by_local_accounts? + @account.passive_relationships.exists? + end + + def requested_through_relay? + @options[:relayed_through_account] && Relay.find_by(inbox_url: @options[:relayed_through_account].inbox_url)&.enabled? + end end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 04afeea20..28a1cda02 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -3,7 +3,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def perform original_status = status_from_object - return if original_status.nil? || delete_arrived_first?(@json['id']) || !announceable?(original_status) + + return if original_status.nil? || delete_arrived_first?(@json['id']) || !announceable?(original_status) || !related_to_local_activity? status = Status.find_by(account: @account, reblog: original_status) @@ -39,4 +40,12 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def announceable?(status) status.account_id == @account.id || status.public_visibility? || status.unlisted_visibility? end + + def related_to_local_activity? + followed_by_local_accounts? || requested_through_relay? || reblog_of_local_status? + end + + def reblog_of_local_status? + status_from_uri(object_uri)&.account&.local? + end end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 1b31768d9..4fc37fb4b 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -341,18 +341,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity responds_to_followed_account? || addresses_local_accounts? end - def fetch? - !@options[:delivery] - end - - def followed_by_local_accounts? - @account.passive_relationships.exists? - end - - def requested_through_relay? - @options[:relayed_through_account] && Relay.find_by(inbox_url: @options[:relayed_through_account].inbox_url)&.enabled? - end - def responds_to_followed_account? !replied_to_status.nil? && (replied_to_status.account.local? || replied_to_status.account.passive_relationships.exists?) end diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index 1725c2843..5e6f008ec 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -18,6 +18,7 @@ RSpec.describe ActivityPub::Activity::Announce do subject { described_class.new(json, sender) } before do + Fabricate(:account).follow!(sender) sender.update(uri: ActivityPub::TagManager.instance.uri_for(sender)) end From 71b831601db2fdc41aaf4ddbe9fd60db109a3153 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 17 Feb 2019 03:38:25 +0100 Subject: [PATCH 27/42] Add logging for rejected ActivityPub payloads and add tests (#10062) --- app/lib/activitypub/activity.rb | 5 + app/lib/activitypub/activity/announce.rb | 4 +- app/lib/activitypub/activity/create.rb | 2 +- .../lib/activitypub/activity/announce_spec.rb | 117 ++- spec/lib/activitypub/activity/create_spec.rb | 738 ++++++++++-------- 5 files changed, 539 insertions(+), 327 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 3cf38764a..8265810a0 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -180,4 +180,9 @@ class ActivityPub::Activity def requested_through_relay? @options[:relayed_through_account] && Relay.find_by(inbox_url: @options[:relayed_through_account].inbox_url)&.enabled? end + + def reject_payload! + Rails.logger.info("Rejected #{@json['type']} activity #{@json['id']} from #{@account.uri}#{@options[:relayed_through_account] && "via #{@options[:relayed_through_account].uri}"}") + nil + end end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 28a1cda02..9f8ffd9fb 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -2,9 +2,11 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def perform + return reject_payload! if delete_arrived_first?(@json['id']) || !related_to_local_activity? + original_status = status_from_object - return if original_status.nil? || delete_arrived_first?(@json['id']) || !announceable?(original_status) || !related_to_local_activity? + return reject_payload! if original_status.nil? || !announceable?(original_status) status = Status.find_by(account: @account, reblog: original_status) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 4fc37fb4b..d7bd65c80 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -2,7 +2,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def perform - return if unsupported_object_type? || invalid_origin?(@object['id']) || Tombstone.exists?(uri: @object['id']) || !related_to_local_activity? + return reject_payload! if unsupported_object_type? || invalid_origin?(@object['id']) || Tombstone.exists?(uri: @object['id']) || !related_to_local_activity? RedisLock.acquire(lock_options) do |lock| if lock.acquired? diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index 5e6f008ec..94b9d348d 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -18,16 +18,63 @@ RSpec.describe ActivityPub::Activity::Announce do subject { described_class.new(json, sender) } before do - Fabricate(:account).follow!(sender) sender.update(uri: ActivityPub::TagManager.instance.uri_for(sender)) end describe '#perform' do - before do - subject.perform + context 'when sender is followed by a local account' do + before do + Fabricate(:account).follow!(sender) + subject.perform + end + + context 'a known status' do + let(:object_json) do + ActivityPub::TagManager.instance.uri_for(status) + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(status)).to be true + end + end + + context 'self-boost of a previously unknown status with missing attributedTo' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'http://example.com/followers', + } + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(sender.statuses.first)).to be true + end + end + + context 'self-boost of a previously unknown status with correct attributedTo' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + attributedTo: ActivityPub::TagManager.instance.uri_for(sender), + to: 'http://example.com/followers', + } + end + + it 'creates a reblog by sender of status' do + expect(sender.reblogged?(sender.statuses.first)).to be true + end + end end - context 'a known status' do + context 'when the status belongs to a local user' do + before do + subject.perform + end + let(:object_json) do ActivityPub::TagManager.instance.uri_for(status) end @@ -37,34 +84,68 @@ RSpec.describe ActivityPub::Activity::Announce do end end - context 'self-boost of a previously unknown status with missing attributedTo' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - to: 'http://example.com/followers', - } + context 'when the sender is relayed' do + let!(:relay_account) { Fabricate(:account, inbox_url: 'https://relay.example.com/inbox') } + let!(:relay) { Fabricate(:relay, inbox_url: 'https://relay.example.com/inbox') } + + subject { described_class.new(json, sender, relayed_through_account: relay_account) } + + context 'and the relay is enabled' do + before do + relay.update(state: :accepted) + subject.perform + end + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'http://example.com/followers', + } + end + + it 'creates a reblog by sender of status' do + expect(sender.statuses.count).to eq 2 + end end - it 'creates a reblog by sender of status' do - expect(sender.reblogged?(sender.statuses.first)).to be true + context 'and the relay is disabled' do + before do + subject.perform + end + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'http://example.com/followers', + } + end + + it 'does not create anything' do + expect(sender.statuses.count).to eq 0 + end end end - context 'self-boost of a previously unknown status with correct attributedTo' do + context 'when the sender has no relevance to local activity' do + before do + subject.perform + end + let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - attributedTo: ActivityPub::TagManager.instance.uri_for(sender), to: 'http://example.com/followers', } end - it 'creates a reblog by sender of status' do - expect(sender.reblogged?(sender.statuses.first)).to be true + it 'does not create anything' do + expect(sender.statuses.count).to eq 0 end end end diff --git a/spec/lib/activitypub/activity/create_spec.rb b/spec/lib/activitypub/activity/create_spec.rb index cd20b7c7c..26cb84871 100644 --- a/spec/lib/activitypub/activity/create_spec.rb +++ b/spec/lib/activitypub/activity/create_spec.rb @@ -13,8 +13,6 @@ RSpec.describe ActivityPub::Activity::Create do }.with_indifferent_access end - subject { described_class.new(json, sender) } - before do sender.update(uri: ActivityPub::TagManager.instance.uri_for(sender)) @@ -23,11 +21,402 @@ RSpec.describe ActivityPub::Activity::Create do end describe '#perform' do - before do - subject.perform + context 'when fetching' do + subject { described_class.new(json, sender) } + + before do + subject.perform + end + + context 'standalone' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.text).to eq 'Lorem ipsum' + end + + it 'missing to/cc defaults to direct privacy' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'direct' + end + end + + context 'public' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'https://www.w3.org/ns/activitystreams#Public', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'public' + end + end + + context 'unlisted' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + cc: 'https://www.w3.org/ns/activitystreams#Public', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'unlisted' + end + end + + context 'private' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: 'http://example.com/followers', + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'private' + end + end + + context 'limited' do + let(:recipient) { Fabricate(:account) } + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: ActivityPub::TagManager.instance.uri_for(recipient), + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'limited' + end + + it 'creates silent mention' do + status = sender.statuses.first + expect(status.mentions.first).to be_silent + end + end + + context 'direct' do + let(:recipient) { Fabricate(:account) } + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + to: ActivityPub::TagManager.instance.uri_for(recipient), + tag: { + type: 'Mention', + href: ActivityPub::TagManager.instance.uri_for(recipient), + }, + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.visibility).to eq 'direct' + end + end + + context 'as a reply' do + let(:original_status) { Fabricate(:status) } + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + inReplyTo: ActivityPub::TagManager.instance.uri_for(original_status), + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.thread).to eq original_status + expect(status.reply?).to be true + expect(status.in_reply_to_account).to eq original_status.account + expect(status.conversation).to eq original_status.conversation + end + end + + context 'with mentions' do + let(:recipient) { Fabricate(:account) } + + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + tag: [ + { + type: 'Mention', + href: ActivityPub::TagManager.instance.uri_for(recipient), + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.mentions.map(&:account)).to include(recipient) + end + end + + context 'with mentions missing href' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + tag: [ + { + type: 'Mention', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + expect(status).to_not be_nil + end + end + + context 'with media attachments' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + attachment: [ + { + type: 'Document', + mediaType: 'image/png', + url: 'http://example.com/attachment.png', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.media_attachments.map(&:remote_url)).to include('http://example.com/attachment.png') + end + end + + context 'with media attachments with focal points' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + attachment: [ + { + type: 'Document', + mediaType: 'image/png', + url: 'http://example.com/attachment.png', + focalPoint: [0.5, -0.7], + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.media_attachments.map(&:focus)).to include('0.5,-0.7') + end + end + + context 'with media attachments missing url' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + attachment: [ + { + type: 'Document', + mediaType: 'image/png', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + expect(status).to_not be_nil + end + end + + context 'with hashtags' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + tag: [ + { + type: 'Hashtag', + href: 'http://example.com/blah', + name: '#test', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.tags.map(&:name)).to include('test') + end + end + + context 'with hashtags missing name' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum', + tag: [ + { + type: 'Hashtag', + href: 'http://example.com/blah', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + expect(status).to_not be_nil + end + end + + context 'with emojis' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum :tinking:', + tag: [ + { + type: 'Emoji', + icon: { + url: 'http://example.com/emoji.png', + }, + name: 'tinking', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + + expect(status).to_not be_nil + expect(status.emojis.map(&:shortcode)).to include('tinking') + end + end + + context 'with emojis missing name' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum :tinking:', + tag: [ + { + type: 'Emoji', + icon: { + url: 'http://example.com/emoji.png', + }, + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + expect(status).to_not be_nil + end + end + + context 'with emojis missing icon' do + let(:object_json) do + { + id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + type: 'Note', + content: 'Lorem ipsum :tinking:', + tag: [ + { + type: 'Emoji', + name: 'tinking', + }, + ], + } + end + + it 'creates status' do + status = sender.statuses.first + expect(status).to_not be_nil + end + end end - context 'standalone' do + context 'when sender is followed by local users' do + subject { described_class.new(json, sender, delivery: true) } + + before do + Fabricate(:account).follow!(sender) + subject.perform + end + let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, @@ -42,78 +431,23 @@ RSpec.describe ActivityPub::Activity::Create do expect(status).to_not be_nil expect(status.text).to eq 'Lorem ipsum' end - - it 'missing to/cc defaults to direct privacy' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.visibility).to eq 'direct' - end end - context 'public' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - to: 'https://www.w3.org/ns/activitystreams#Public', - } + context 'when sender replies to local status' do + let!(:local_status) { Fabricate(:status) } + + subject { described_class.new(json, sender, delivery: true) } + + before do + subject.perform end - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.visibility).to eq 'public' - end - end - - context 'unlisted' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - cc: 'https://www.w3.org/ns/activitystreams#Public', - } - end - - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.visibility).to eq 'unlisted' - end - end - - context 'private' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - to: 'http://example.com/followers', - } - end - - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.visibility).to eq 'private' - end - end - - context 'limited' do - let(:recipient) { Fabricate(:account) } - let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - to: ActivityPub::TagManager.instance.uri_for(recipient), + inReplyTo: ActivityPub::TagManager.instance.uri_for(local_status), } end @@ -121,28 +455,25 @@ RSpec.describe ActivityPub::Activity::Create do status = sender.statuses.first expect(status).to_not be_nil - expect(status.visibility).to eq 'limited' - end - - it 'creates silent mention' do - status = sender.statuses.first - expect(status.mentions.first).to be_silent + expect(status.text).to eq 'Lorem ipsum' end end - context 'direct' do - let(:recipient) { Fabricate(:account) } + context 'when sender targets a local user' do + let!(:local_account) { Fabricate(:account) } + + subject { described_class.new(json, sender, delivery: true) } + + before do + subject.perform + end let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - to: ActivityPub::TagManager.instance.uri_for(recipient), - tag: { - type: 'Mention', - href: ActivityPub::TagManager.instance.uri_for(recipient), - }, + to: ActivityPub::TagManager.instance.uri_for(local_account), } end @@ -150,19 +481,25 @@ RSpec.describe ActivityPub::Activity::Create do status = sender.statuses.first expect(status).to_not be_nil - expect(status.visibility).to eq 'direct' + expect(status.text).to eq 'Lorem ipsum' end end - context 'as a reply' do - let(:original_status) { Fabricate(:status) } + context 'when sender cc\'s a local user' do + let!(:local_account) { Fabricate(:account) } + + subject { described_class.new(json, sender, delivery: true) } + + before do + subject.perform + end let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - inReplyTo: ActivityPub::TagManager.instance.uri_for(original_status), + cc: ActivityPub::TagManager.instance.uri_for(local_account), } end @@ -170,240 +507,27 @@ RSpec.describe ActivityPub::Activity::Create do status = sender.statuses.first expect(status).to_not be_nil - expect(status.thread).to eq original_status - expect(status.reply?).to be true - expect(status.in_reply_to_account).to eq original_status.account - expect(status.conversation).to eq original_status.conversation + expect(status.text).to eq 'Lorem ipsum' end end - context 'with mentions' do - let(:recipient) { Fabricate(:account) } + context 'when the sender has no relevance to local activity' do + subject { described_class.new(json, sender, delivery: true) } + + before do + subject.perform + end let(:object_json) do { id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, type: 'Note', content: 'Lorem ipsum', - tag: [ - { - type: 'Mention', - href: ActivityPub::TagManager.instance.uri_for(recipient), - }, - ], } end - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.mentions.map(&:account)).to include(recipient) - end - end - - context 'with mentions missing href' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - tag: [ - { - type: 'Mention', - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - expect(status).to_not be_nil - end - end - - context 'with media attachments' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - attachment: [ - { - type: 'Document', - mediaType: 'image/png', - url: 'http://example.com/attachment.png', - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.media_attachments.map(&:remote_url)).to include('http://example.com/attachment.png') - end - end - - context 'with media attachments with focal points' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - attachment: [ - { - type: 'Document', - mediaType: 'image/png', - url: 'http://example.com/attachment.png', - focalPoint: [0.5, -0.7], - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.media_attachments.map(&:focus)).to include('0.5,-0.7') - end - end - - context 'with media attachments missing url' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - attachment: [ - { - type: 'Document', - mediaType: 'image/png', - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - expect(status).to_not be_nil - end - end - - context 'with hashtags' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - tag: [ - { - type: 'Hashtag', - href: 'http://example.com/blah', - name: '#test', - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.tags.map(&:name)).to include('test') - end - end - - context 'with hashtags missing name' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum', - tag: [ - { - type: 'Hashtag', - href: 'http://example.com/blah', - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - expect(status).to_not be_nil - end - end - - context 'with emojis' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum :tinking:', - tag: [ - { - type: 'Emoji', - icon: { - url: 'http://example.com/emoji.png', - }, - name: 'tinking', - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - - expect(status).to_not be_nil - expect(status.emojis.map(&:shortcode)).to include('tinking') - end - end - - context 'with emojis missing name' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum :tinking:', - tag: [ - { - type: 'Emoji', - icon: { - url: 'http://example.com/emoji.png', - }, - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - expect(status).to_not be_nil - end - end - - context 'with emojis missing icon' do - let(:object_json) do - { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, - type: 'Note', - content: 'Lorem ipsum :tinking:', - tag: [ - { - type: 'Emoji', - name: 'tinking', - }, - ], - } - end - - it 'creates status' do - status = sender.statuses.first - expect(status).to_not be_nil + it 'does not create anything' do + expect(sender.statuses.count).to eq 0 end end end From b163368c3e3e3a22c4ef98c0d0cd6c07a2ad13e6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 17 Feb 2019 15:16:36 +0100 Subject: [PATCH 28/42] Fix Announce activities of unknown statuses not fetching those statuses (#10065) Regression from #9998 --- app/lib/activitypub/activity.rb | 3 +- .../lib/activitypub/activity/announce_spec.rb | 43 +++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 8265810a0..11fa3363a 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -150,8 +150,7 @@ class ActivityPub::Activity end end - # If the status is not from the actor, try to fetch it - return fetch_remote_original_status if value_or_id(first_of_value(@json['attributedTo'])) == @account.uri + fetch_remote_original_status end def fetch_remote_original_status diff --git a/spec/lib/activitypub/activity/announce_spec.rb b/spec/lib/activitypub/activity/announce_spec.rb index 94b9d348d..aa58d9e23 100644 --- a/spec/lib/activitypub/activity/announce_spec.rb +++ b/spec/lib/activitypub/activity/announce_spec.rb @@ -1,7 +1,7 @@ require 'rails_helper' RSpec.describe ActivityPub::Activity::Announce do - let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers') } + let(:sender) { Fabricate(:account, followers_url: 'http://example.com/followers', uri: 'https://example.com/actor') } let(:recipient) { Fabricate(:account) } let(:status) { Fabricate(:status, account: recipient) } @@ -10,21 +10,29 @@ RSpec.describe ActivityPub::Activity::Announce do '@context': 'https://www.w3.org/ns/activitystreams', id: 'foo', type: 'Announce', - actor: ActivityPub::TagManager.instance.uri_for(sender), + actor: 'https://example.com/actor', object: object_json, }.with_indifferent_access end - subject { described_class.new(json, sender) } - - before do - sender.update(uri: ActivityPub::TagManager.instance.uri_for(sender)) + let(:unknown_object_json) do + { + '@context': 'https://www.w3.org/ns/activitystreams', + id: 'https://example.com/actor/hello-world', + type: 'Note', + attributedTo: 'https://example.com/actor', + content: 'Hello world', + to: 'http://example.com/followers', + } end + subject { described_class.new(json, sender) } + describe '#perform' do context 'when sender is followed by a local account' do before do Fabricate(:account).follow!(sender) + stub_request(:get, 'https://example.com/actor/hello-world').to_return(body: Oj.dump(unknown_object_json)) subject.perform end @@ -38,10 +46,21 @@ RSpec.describe ActivityPub::Activity::Announce do end end + context 'an unknown status' do + let(:object_json) { 'https://example.com/actor/hello-world' } + + it 'creates a reblog by sender of status' do + reblog = sender.statuses.first + + expect(reblog).to_not be_nil + expect(reblog.reblog.text).to eq 'Hello world' + end + end + context 'self-boost of a previously unknown status with missing attributedTo' do let(:object_json) do { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + id: 'https://example.com/actor#bar', type: 'Note', content: 'Lorem ipsum', to: 'http://example.com/followers', @@ -56,10 +75,10 @@ RSpec.describe ActivityPub::Activity::Announce do context 'self-boost of a previously unknown status with correct attributedTo' do let(:object_json) do { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + id: 'https://example.com/actor#bar', type: 'Note', content: 'Lorem ipsum', - attributedTo: ActivityPub::TagManager.instance.uri_for(sender), + attributedTo: 'https://example.com/actor', to: 'http://example.com/followers', } end @@ -98,7 +117,7 @@ RSpec.describe ActivityPub::Activity::Announce do let(:object_json) do { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + id: 'https://example.com/actor#bar', type: 'Note', content: 'Lorem ipsum', to: 'http://example.com/followers', @@ -117,7 +136,7 @@ RSpec.describe ActivityPub::Activity::Announce do let(:object_json) do { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + id: 'https://example.com/actor#bar', type: 'Note', content: 'Lorem ipsum', to: 'http://example.com/followers', @@ -137,7 +156,7 @@ RSpec.describe ActivityPub::Activity::Announce do let(:object_json) do { - id: [ActivityPub::TagManager.instance.uri_for(sender), '#bar'].join, + id: 'https://example.com/actor#bar', type: 'Note', content: 'Lorem ipsum', to: 'http://example.com/followers', From 8ad75eea62117f8635de8897c74297a5a8b3bd5c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 15 Feb 2019 16:08:59 +0100 Subject: [PATCH 29/42] Fix relay enabling/disabling not resetting inbox availability status (#10048) Fix #10033 --- app/models/relay.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/models/relay.rb b/app/models/relay.rb index 7478c110d..6934a5c62 100644 --- a/app/models/relay.rb +++ b/app/models/relay.rb @@ -29,6 +29,7 @@ class Relay < ApplicationRecord payload = Oj.dump(follow_activity(activity_id)) update!(state: :pending, follow_activity_id: activity_id) + DeliveryFailureTracker.new(inbox_url).track_success! ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url) end @@ -37,6 +38,7 @@ class Relay < ApplicationRecord payload = Oj.dump(unfollow_activity(activity_id)) update!(state: :idle, follow_activity_id: nil) + DeliveryFailureTracker.new(inbox_url).track_success! ActivityPub::DeliveryWorker.perform_async(payload, some_local_account.id, inbox_url) end From 637f0007b9535ed988cbe94c9b9b40a8f8e8a24f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 13 Feb 2019 21:28:18 +0100 Subject: [PATCH 30/42] Change robots.txt to exclude some URLs (#10037) - Exclude static assets - Exclude uploaded files - Exclude alternate versions of the profile page - Exclude media proxy URLs --- public/robots.txt | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/public/robots.txt b/public/robots.txt index 3c9c7c01f..36afc85ef 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,5 +1,13 @@ -# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: -# User-agent: * -# Disallow: / +User-Agent: * +Disallow: /users/*/followers +Disallow: /users/*/following +Disallow: /@*/media +Disallow: /@*/with_replies +Disallow: /@*/tagged/* +Disallow: /media_proxy/* +Disallow: /emoji/* +Disallow: /packs/* +Disallow: /sounds/* +Disallow: /system/* +Disallow: /avatars/* +Disallow: /headers/* From 45b2bb464b5ff5b5b5805004a5dc856b495dfc54 Mon Sep 17 00:00:00 2001 From: nightpool Date: Wed, 13 Feb 2019 21:11:47 -0500 Subject: [PATCH 31/42] Change robots.txt to exclude only media proxy URLs (#10038) * Revert "Change robots.txt to exclude some URLs (#10037)" This reverts commit 80161f43510ad9316c60c9b50dd5c09c2dae4d54. * Let's block media_proxy /media_proxy/ is a dynamic route used for requesting uncached media, so it's probably bad to let crawlers use it * misleading comment --- public/robots.txt | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/public/robots.txt b/public/robots.txt index 36afc85ef..d93648bee 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,13 +1,4 @@ -User-Agent: * -Disallow: /users/*/followers -Disallow: /users/*/following -Disallow: /@*/media -Disallow: /@*/with_replies -Disallow: /@*/tagged/* -Disallow: /media_proxy/* -Disallow: /emoji/* -Disallow: /packs/* -Disallow: /sounds/* -Disallow: /system/* -Disallow: /avatars/* -Disallow: /headers/* +# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file + +User-agent: * +Disallow: /media_proxy/ From 1ad0d232b3ed1c2005d64f1746ecd2d476379852 Mon Sep 17 00:00:00 2001 From: Ben Lubar Date: Wed, 13 Feb 2019 18:04:43 -0600 Subject: [PATCH 32/42] Improve image description user experience (#10036) * Add image descriptions to searchable post content. * Allow multi-line image descriptions. * Request image descriptions in the same query as posts when creating the search index. (see https://github.com/tootsuite/mastodon/pull/10036#discussion_r256551624) --- app/chewy/statuses_index.rb | 4 ++-- app/javascript/mastodon/features/compose/components/upload.js | 3 +-- app/javascript/styles/mastodon/components.scss | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index d3104172c..eafc1818b 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -31,7 +31,7 @@ class StatusesIndex < Chewy::Index }, } - define_type ::Status.unscoped.without_reblogs do + define_type ::Status.unscoped.without_reblogs.includes(:media_attachments) do crutch :mentions do |collection| data = ::Mention.where(status_id: collection.map(&:id)).pluck(:status_id, :account_id) data.each.with_object({}) { |(id, name), result| (result[id] ||= []).push(name) } @@ -50,7 +50,7 @@ class StatusesIndex < Chewy::Index root date_detection: false do field :account_id, type: 'long' - field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].join("\n\n") } do + field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].concat(status.media_attachments.map(&:description)).join("\n\n") } do field :stemmed, type: 'text', analyzer: 'content' end diff --git a/app/javascript/mastodon/features/compose/components/upload.js b/app/javascript/mastodon/features/compose/components/upload.js index a1e99dcbb..947c3acc9 100644 --- a/app/javascript/mastodon/features/compose/components/upload.js +++ b/app/javascript/mastodon/features/compose/components/upload.js @@ -107,9 +107,8 @@ class Upload extends ImmutablePureComponent {