From 5d2b69c256aaa58768b608a0800b733d0b386d68 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Fri, 4 Jun 2021 14:39:56 +0200 Subject: [PATCH] moving stuff around --- internal/cliactions/server/server.go | 2 +- internal/db/pg/pg.go | 4 +- internal/gtsmodel/README.md | 5 - internal/processing/federation.go | 2 +- internal/processing/processor.go | 28 +-- .../{ => synchronous}/status/boost.go | 2 +- .../{ => synchronous}/status/boostedby.go | 0 .../{ => synchronous}/status/context.go | 0 .../{ => synchronous}/status/create.go | 2 +- .../{ => synchronous}/status/delete.go | 3 +- .../{ => synchronous}/status/fave.go | 2 +- .../{ => synchronous}/status/favedby.go | 0 .../{ => synchronous}/status/get.go | 0 .../{ => synchronous}/status/status.go | 2 +- .../{ => synchronous}/status/unfave.go | 2 +- .../{ => synchronous}/status/util.go | 0 internal/processing/util.go | 222 ------------------ internal/{processing => }/timeline/manager.go | 0 .../{processing => }/timeline/postindex.go | 0 .../timeline/preparedposts.go | 0 .../{processing => }/timeline/timeline.go | 0 testrig/timelinemanager.go | 2 +- 22 files changed, 14 insertions(+), 264 deletions(-) delete mode 100644 internal/gtsmodel/README.md rename internal/processing/{ => synchronous}/status/boost.go (98%) rename internal/processing/{ => synchronous}/status/boostedby.go (100%) rename internal/processing/{ => synchronous}/status/context.go (100%) rename internal/processing/{ => synchronous}/status/create.go (98%) rename internal/processing/{ => synchronous}/status/delete.go (97%) rename internal/processing/{ => synchronous}/status/fave.go (98%) rename internal/processing/{ => synchronous}/status/favedby.go (100%) rename internal/processing/{ => synchronous}/status/get.go (100%) rename internal/processing/{ => synchronous}/status/status.go (99%) rename internal/processing/{ => synchronous}/status/unfave.go (98%) rename internal/processing/{ => synchronous}/status/util.go (100%) rename internal/{processing => }/timeline/manager.go (100%) rename internal/{processing => }/timeline/postindex.go (100%) rename internal/{processing => }/timeline/preparedposts.go (100%) rename internal/{processing => }/timeline/timeline.go (100%) diff --git a/internal/cliactions/server/server.go b/internal/cliactions/server/server.go index 3f8b48f..1f0ceb3 100644 --- a/internal/cliactions/server/server.go +++ b/internal/cliactions/server/server.go @@ -39,8 +39,8 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" - timelineprocessing "github.com/superseriousbusiness/gotosocial/internal/processing/timeline" "github.com/superseriousbusiness/gotosocial/internal/router" + timelineprocessing "github.com/superseriousbusiness/gotosocial/internal/timeline" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) diff --git a/internal/db/pg/pg.go b/internal/db/pg/pg.go index d4f8d10..09eb79b 100644 --- a/internal/db/pg/pg.go +++ b/internal/db/pg/pg.go @@ -467,8 +467,8 @@ func (ps *postgresService) GetFollowersByAccountID(accountID string, followers * // for local accounts let's get where domain is null OR where domain is an empty string, just to be safe whereGroup := func(q *pg.Query) (*pg.Query, error) { q = q. - WhereOr("? IS NULL", pg.Ident("a.domain")). - WhereOr("a.domain = ?", "") + WhereOr("? IS NULL", pg.Ident("a.domain")). + WhereOr("a.domain = ?", "") return q, nil } diff --git a/internal/gtsmodel/README.md b/internal/gtsmodel/README.md deleted file mode 100644 index 12a05dd..0000000 --- a/internal/gtsmodel/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# gtsmodel - -This package contains types used *internally* by GoToSocial and added/removed/selected from the database. As such, they contain sensitive fields which should **never** be serialized or reach the API level. Use the [mastotypes](../../pkg/mastotypes) package for that. - -The annotation used on these structs is for handling them via the go-pg ORM. See [here](https://pg.uptrace.dev/models/). diff --git a/internal/processing/federation.go b/internal/processing/federation.go index dacaa25..eaa27d1 100644 --- a/internal/processing/federation.go +++ b/internal/processing/federation.go @@ -80,7 +80,7 @@ func (p *processor) authenticateAndDereferenceFediRequest(username string, r *ht } // put it in our channel to queue it for async processing - p.FromFederator() <- gtsmodel.FromFederator{ + p.fromFederator <- gtsmodel.FromFederator{ APObjectType: gtsmodel.ActivityStreamsProfile, APActivityType: gtsmodel.ActivityStreamsCreate, GTSModel: requestingAccount, diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 65ecfa5..8d263e3 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -32,8 +32,8 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/oauth" - "github.com/superseriousbusiness/gotosocial/internal/processing/status" - "github.com/superseriousbusiness/gotosocial/internal/processing/timeline" + "github.com/superseriousbusiness/gotosocial/internal/processing/synchronous/status" + "github.com/superseriousbusiness/gotosocial/internal/timeline" "github.com/superseriousbusiness/gotosocial/internal/typeutils" ) @@ -44,14 +44,6 @@ import ( // fire messages into the processor and not wait for a reply before proceeding with other work. This allows // for clean distribution of messages without slowing down the client API and harming the user experience. type Processor interface { - // ToClientAPI returns a channel for putting in messages that need to go to the gts client API. - // ToClientAPI() chan gtsmodel.ToClientAPI - // FromClientAPI returns a channel for putting messages in that come from the client api going to the processor - FromClientAPI() chan gtsmodel.FromClientAPI - // ToFederator returns a channel for putting in messages that need to go to the federator (activitypub). - // ToFederator() chan gtsmodel.ToFederator - // FromFederator returns a channel for putting messages in that come from the federator (activitypub) going into the processor - FromFederator() chan gtsmodel.FromFederator // Start starts the Processor, reading from its channels and passing messages back and forth. Start() error // Stop stops the processor cleanly, finishing handling any remaining messages before closing down. @@ -227,22 +219,6 @@ func NewProcessor(config *config.Config, tc typeutils.TypeConverter, federator f } } -// func (p *processor) ToClientAPI() chan gtsmodel.ToClientAPI { -// return p.toClientAPI -// } - -func (p *processor) FromClientAPI() chan gtsmodel.FromClientAPI { - return p.fromClientAPI -} - -// func (p *processor) ToFederator() chan gtsmodel.ToFederator { -// return p.toFederator -// } - -func (p *processor) FromFederator() chan gtsmodel.FromFederator { - return p.fromFederator -} - // Start starts the Processor, reading from its channels and passing messages back and forth. func (p *processor) Start() error { go func() { diff --git a/internal/processing/status/boost.go b/internal/processing/synchronous/status/boost.go similarity index 98% rename from internal/processing/status/boost.go rename to internal/processing/synchronous/status/boost.go index 1abdbc2..84cb02c 100644 --- a/internal/processing/status/boost.go +++ b/internal/processing/synchronous/status/boost.go @@ -60,7 +60,7 @@ func (p *processor) Boost(account *gtsmodel.Account, application *gtsmodel.Appli return nil, gtserror.NewErrorInternalError(err) } - // send it to the processor for async processing + // send it back to the processor for async processing p.fromClientAPI <- gtsmodel.FromClientAPI{ APObjectType: gtsmodel.ActivityStreamsAnnounce, APActivityType: gtsmodel.ActivityStreamsCreate, diff --git a/internal/processing/status/boostedby.go b/internal/processing/synchronous/status/boostedby.go similarity index 100% rename from internal/processing/status/boostedby.go rename to internal/processing/synchronous/status/boostedby.go diff --git a/internal/processing/status/context.go b/internal/processing/synchronous/status/context.go similarity index 100% rename from internal/processing/status/context.go rename to internal/processing/synchronous/status/context.go diff --git a/internal/processing/status/create.go b/internal/processing/synchronous/status/create.go similarity index 98% rename from internal/processing/status/create.go rename to internal/processing/synchronous/status/create.go index e57abfe..e90855a 100644 --- a/internal/processing/status/create.go +++ b/internal/processing/synchronous/status/create.go @@ -80,7 +80,7 @@ func (p *processor) Create(account *gtsmodel.Account, application *gtsmodel.Appl } } - // put the new status in the appropriate channel for async processing + // send it back to the processor for async processing p.fromClientAPI <- gtsmodel.FromClientAPI{ APObjectType: gtsmodel.ActivityStreamsNote, APActivityType: gtsmodel.ActivityStreamsCreate, diff --git a/internal/processing/status/delete.go b/internal/processing/synchronous/status/delete.go similarity index 97% rename from internal/processing/status/delete.go rename to internal/processing/synchronous/status/delete.go index 569edba..5d98722 100644 --- a/internal/processing/status/delete.go +++ b/internal/processing/synchronous/status/delete.go @@ -43,7 +43,8 @@ func (p *processor) Delete(account *gtsmodel.Account, targetStatusID string) (*a if err := p.db.DeleteByID(targetStatus.ID, targetStatus); err != nil { return nil, gtserror.NewErrorInternalError(fmt.Errorf("error deleting status from the database: %s", err)) } - + + // send it back to the processor for async processing p.fromClientAPI <- gtsmodel.FromClientAPI{ APObjectType: gtsmodel.ActivityStreamsNote, APActivityType: gtsmodel.ActivityStreamsDelete, diff --git a/internal/processing/status/fave.go b/internal/processing/synchronous/status/fave.go similarity index 98% rename from internal/processing/status/fave.go rename to internal/processing/synchronous/status/fave.go index fc10279..e7bf44a 100644 --- a/internal/processing/status/fave.go +++ b/internal/processing/synchronous/status/fave.go @@ -84,7 +84,7 @@ func (p *processor) Fave(account *gtsmodel.Account, targetStatusID string) (*api return nil, gtserror.NewErrorInternalError(fmt.Errorf("error putting fave in database: %s", err)) } - // send the new fave through the processor channel for federation etc + // send it back to the processor for async processing p.fromClientAPI <- gtsmodel.FromClientAPI{ APObjectType: gtsmodel.ActivityStreamsLike, APActivityType: gtsmodel.ActivityStreamsCreate, diff --git a/internal/processing/status/favedby.go b/internal/processing/synchronous/status/favedby.go similarity index 100% rename from internal/processing/status/favedby.go rename to internal/processing/synchronous/status/favedby.go diff --git a/internal/processing/status/get.go b/internal/processing/synchronous/status/get.go similarity index 100% rename from internal/processing/status/get.go rename to internal/processing/synchronous/status/get.go diff --git a/internal/processing/status/status.go b/internal/processing/synchronous/status/status.go similarity index 99% rename from internal/processing/status/status.go rename to internal/processing/synchronous/status/status.go index e03ccf1..5dd26a2 100644 --- a/internal/processing/status/status.go +++ b/internal/processing/synchronous/status/status.go @@ -47,6 +47,6 @@ func New(db db.DB, tc typeutils.TypeConverter, config *config.Config, fromClient config: config, db: db, fromClientAPI: fromClientAPI, - log: log, + log: log, } } diff --git a/internal/processing/status/unfave.go b/internal/processing/synchronous/status/unfave.go similarity index 98% rename from internal/processing/status/unfave.go rename to internal/processing/synchronous/status/unfave.go index a304d80..237eedc 100644 --- a/internal/processing/status/unfave.go +++ b/internal/processing/synchronous/status/unfave.go @@ -65,7 +65,7 @@ func (p *processor) Unfave(account *gtsmodel.Account, targetStatusID string) (*a return nil, gtserror.NewErrorInternalError(fmt.Errorf("error unfaveing status: %s", err)) } - // send the unfave through the processor channel for federation etc + // send it back to the processor for async processing p.fromClientAPI <- gtsmodel.FromClientAPI{ APObjectType: gtsmodel.ActivityStreamsLike, APActivityType: gtsmodel.ActivityStreamsUndo, diff --git a/internal/processing/status/util.go b/internal/processing/synchronous/status/util.go similarity index 100% rename from internal/processing/status/util.go rename to internal/processing/synchronous/status/util.go diff --git a/internal/processing/util.go b/internal/processing/util.go index af62afb..6474100 100644 --- a/internal/processing/util.go +++ b/internal/processing/util.go @@ -25,233 +25,11 @@ import ( "io" "mime/multipart" - apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" - "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/transport" - "github.com/superseriousbusiness/gotosocial/internal/util" ) -func (p *processor) processVisibility(form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error { - // by default all flags are set to true - gtsAdvancedVis := >smodel.VisibilityAdvanced{ - Federated: true, - Boostable: true, - Replyable: true, - Likeable: true, - } - - var gtsBasicVis gtsmodel.Visibility - // Advanced takes priority if it's set. - // If it's not set, take whatever masto visibility is set. - // If *that's* not set either, then just take the account default. - // If that's also not set, take the default for the whole instance. - if form.VisibilityAdvanced != nil { - gtsBasicVis = gtsmodel.Visibility(*form.VisibilityAdvanced) - } else if form.Visibility != "" { - gtsBasicVis = p.tc.MastoVisToVis(form.Visibility) - } else if accountDefaultVis != "" { - gtsBasicVis = accountDefaultVis - } else { - gtsBasicVis = gtsmodel.VisibilityDefault - } - - switch gtsBasicVis { - case gtsmodel.VisibilityPublic: - // for public, there's no need to change any of the advanced flags from true regardless of what the user filled out - break - case gtsmodel.VisibilityUnlocked: - // for unlocked the user can set any combination of flags they like so look at them all to see if they're set and then apply them - if form.Federated != nil { - gtsAdvancedVis.Federated = *form.Federated - } - - if form.Boostable != nil { - gtsAdvancedVis.Boostable = *form.Boostable - } - - if form.Replyable != nil { - gtsAdvancedVis.Replyable = *form.Replyable - } - - if form.Likeable != nil { - gtsAdvancedVis.Likeable = *form.Likeable - } - - case gtsmodel.VisibilityFollowersOnly, gtsmodel.VisibilityMutualsOnly: - // for followers or mutuals only, boostable will *always* be false, but the other fields can be set so check and apply them - gtsAdvancedVis.Boostable = false - - if form.Federated != nil { - gtsAdvancedVis.Federated = *form.Federated - } - - if form.Replyable != nil { - gtsAdvancedVis.Replyable = *form.Replyable - } - - if form.Likeable != nil { - gtsAdvancedVis.Likeable = *form.Likeable - } - - case gtsmodel.VisibilityDirect: - // direct is pretty easy: there's only one possible setting so return it - gtsAdvancedVis.Federated = true - gtsAdvancedVis.Boostable = false - gtsAdvancedVis.Federated = true - gtsAdvancedVis.Likeable = true - } - - status.Visibility = gtsBasicVis - status.VisibilityAdvanced = gtsAdvancedVis - return nil -} - -func (p *processor) processReplyToID(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error { - if form.InReplyToID == "" { - return nil - } - - // If this status is a reply to another status, we need to do a bit of work to establish whether or not this status can be posted: - // - // 1. Does the replied status exist in the database? - // 2. Is the replied status marked as replyable? - // 3. Does a block exist between either the current account or the account that posted the status it's replying to? - // - // If this is all OK, then we fetch the repliedStatus and the repliedAccount for later processing. - repliedStatus := >smodel.Status{} - repliedAccount := >smodel.Account{} - // check replied status exists + is replyable - if err := p.db.GetByID(form.InReplyToID, repliedStatus); err != nil { - if _, ok := err.(db.ErrNoEntries); ok { - return fmt.Errorf("status with id %s not replyable because it doesn't exist", form.InReplyToID) - } - return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err) - } - - if repliedStatus.VisibilityAdvanced != nil { - if !repliedStatus.VisibilityAdvanced.Replyable { - return fmt.Errorf("status with id %s is marked as not replyable", form.InReplyToID) - } - } - - // check replied account is known to us - if err := p.db.GetByID(repliedStatus.AccountID, repliedAccount); err != nil { - if _, ok := err.(db.ErrNoEntries); ok { - return fmt.Errorf("status with id %s not replyable because account id %s is not known", form.InReplyToID, repliedStatus.AccountID) - } - return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err) - } - // check if a block exists - if blocked, err := p.db.Blocked(thisAccountID, repliedAccount.ID); err != nil { - if _, ok := err.(db.ErrNoEntries); !ok { - return fmt.Errorf("status with id %s not replyable: %s", form.InReplyToID, err) - } - } else if blocked { - return fmt.Errorf("status with id %s not replyable", form.InReplyToID) - } - status.InReplyToID = repliedStatus.ID - status.InReplyToAccountID = repliedAccount.ID - - return nil -} - -func (p *processor) processMediaIDs(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error { - if form.MediaIDs == nil { - return nil - } - - gtsMediaAttachments := []*gtsmodel.MediaAttachment{} - attachments := []string{} - for _, mediaID := range form.MediaIDs { - // check these attachments exist - a := >smodel.MediaAttachment{} - if err := p.db.GetByID(mediaID, a); err != nil { - return fmt.Errorf("invalid media type or media not found for media id %s", mediaID) - } - // check they belong to the requesting account id - if a.AccountID != thisAccountID { - return fmt.Errorf("media with id %s does not belong to account %s", mediaID, thisAccountID) - } - // check they're not already used in a status - if a.StatusID != "" || a.ScheduledStatusID != "" { - return fmt.Errorf("media with id %s is already attached to a status", mediaID) - } - gtsMediaAttachments = append(gtsMediaAttachments, a) - attachments = append(attachments, a.ID) - } - status.GTSMediaAttachments = gtsMediaAttachments - status.Attachments = attachments - return nil -} - -func (p *processor) processLanguage(form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error { - if form.Language != "" { - status.Language = form.Language - } else { - status.Language = accountDefaultLanguage - } - if status.Language == "" { - return errors.New("no language given either in status create form or account default") - } - return nil -} - -func (p *processor) processMentions(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error { - menchies := []string{} - gtsMenchies, err := p.db.MentionStringsToMentions(util.DeriveMentionsFromStatus(form.Status), accountID, status.ID) - if err != nil { - return fmt.Errorf("error generating mentions from status: %s", err) - } - for _, menchie := range gtsMenchies { - if err := p.db.Put(menchie); err != nil { - return fmt.Errorf("error putting mentions in db: %s", err) - } - menchies = append(menchies, menchie.ID) - } - // add full populated gts menchies to the status for passing them around conveniently - status.GTSMentions = gtsMenchies - // add just the ids of the mentioned accounts to the status for putting in the db - status.Mentions = menchies - return nil -} - -func (p *processor) processTags(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error { - tags := []string{} - gtsTags, err := p.db.TagStringsToTags(util.DeriveHashtagsFromStatus(form.Status), accountID, status.ID) - if err != nil { - return fmt.Errorf("error generating hashtags from status: %s", err) - } - for _, tag := range gtsTags { - if err := p.db.Upsert(tag, "name"); err != nil { - return fmt.Errorf("error putting tags in db: %s", err) - } - tags = append(tags, tag.ID) - } - // add full populated gts tags to the status for passing them around conveniently - status.GTSTags = gtsTags - // add just the ids of the used tags to the status for putting in the db - status.Tags = tags - return nil -} - -func (p *processor) processEmojis(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error { - emojis := []string{} - gtsEmojis, err := p.db.EmojiStringsToEmojis(util.DeriveEmojisFromStatus(form.Status), accountID, status.ID) - if err != nil { - return fmt.Errorf("error generating emojis from status: %s", err) - } - for _, e := range gtsEmojis { - emojis = append(emojis, e.ID) - } - // add full populated gts emojis to the status for passing them around conveniently - status.GTSEmojis = gtsEmojis - // add just the ids of the used emojis to the status for putting in the db - status.Emojis = emojis - return nil -} - /* HELPER FUNCTIONS */ diff --git a/internal/processing/timeline/manager.go b/internal/timeline/manager.go similarity index 100% rename from internal/processing/timeline/manager.go rename to internal/timeline/manager.go diff --git a/internal/processing/timeline/postindex.go b/internal/timeline/postindex.go similarity index 100% rename from internal/processing/timeline/postindex.go rename to internal/timeline/postindex.go diff --git a/internal/processing/timeline/preparedposts.go b/internal/timeline/preparedposts.go similarity index 100% rename from internal/processing/timeline/preparedposts.go rename to internal/timeline/preparedposts.go diff --git a/internal/processing/timeline/timeline.go b/internal/timeline/timeline.go similarity index 100% rename from internal/processing/timeline/timeline.go rename to internal/timeline/timeline.go diff --git a/testrig/timelinemanager.go b/testrig/timelinemanager.go index 5dae16c..31c817e 100644 --- a/testrig/timelinemanager.go +++ b/testrig/timelinemanager.go @@ -2,7 +2,7 @@ package testrig import ( "github.com/superseriousbusiness/gotosocial/internal/db" - "github.com/superseriousbusiness/gotosocial/internal/processing/timeline" + "github.com/superseriousbusiness/gotosocial/internal/timeline" ) // NewTestTimelineManager retuts a new timeline.Manager, suitable for testing, using the given db.