From 6994859d03d827ba040f8b199e889e7602aabdb3 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Fri, 11 Jun 2021 19:40:14 +0200 Subject: [PATCH] fiddley --- internal/federation/federatingdb/create.go | 2 +- internal/processing/fromfederator.go | 14 ++ internal/processing/timeline.go | 142 ++++++++++----------- internal/util/uri.go | 19 ++- 4 files changed, 93 insertions(+), 84 deletions(-) diff --git a/internal/federation/federatingdb/create.go b/internal/federation/federatingdb/create.go index 04b0a7f..5e679b2 100644 --- a/internal/federation/federatingdb/create.go +++ b/internal/federation/federatingdb/create.go @@ -137,7 +137,7 @@ func (f *federatingDB) Create(ctx context.Context, asType vocab.Type) error { return fmt.Errorf("could not convert Follow to follow request: %s", err) } - newID, err := id.NewULIDFromTime(followRequest.CreatedAt) + newID, err := id.NewULID() if err != nil { return err } diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go index 2a44ac1..f010a7a 100644 --- a/internal/processing/fromfederator.go +++ b/internal/processing/fromfederator.go @@ -277,6 +277,14 @@ func (p *processor) dereferenceStatusFields(status *gtsmodel.Status, requestingU // We should dereference any accounts mentioned here which we don't have in our db yet, by their URI. mentions := []string{} for _, m := range status.GTSMentions { + if m.ID == "" { + mID, err := id.NewRandomULID() + if err != nil { + return err + } + m.ID = mID + } + uri, err := url.Parse(m.MentionedAccountURI) if err != nil { l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err) @@ -308,6 +316,12 @@ func (p *processor) dereferenceStatusFields(status *gtsmodel.Status, requestingU continue } + targetAccountID, err := id.NewRandomULID() + if err != nil { + return err + } + targetAccount.ID = targetAccountID + if err := p.db.Put(targetAccount); err != nil { return fmt.Errorf("db error inserting account with uri %s", uri.String()) } diff --git a/internal/processing/timeline.go b/internal/processing/timeline.go index ec49ecd..7e78d43 100644 --- a/internal/processing/timeline.go +++ b/internal/processing/timeline.go @@ -54,87 +54,83 @@ func (p *processor) HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID st minIDMarker := minID l.Debugf("\n entering grabloop \n") -grabloop: - for len(apiStatuses) < limit { - l.Debugf("\n querying the db \n") - gtsStatuses, err := p.db.GetStatusesWhereFollowing(authed.Account.ID, maxIDMarker, sinceIDMarker, minIDMarker, limit, local) + + l.Debugf("\n querying the db \n") + gtsStatuses, err := p.db.GetStatusesWhereFollowing(authed.Account.ID, maxIDMarker, sinceIDMarker, minIDMarker, limit, local) + if err != nil { + if _, ok := err.(db.ErrNoEntries); !ok { + return nil, gtserror.NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error getting statuses from db: %s", err)) + } + } + + for _, gtsStatus := range gtsStatuses { + // haveAlready := false + // for _, apiStatus := range apiStatuses { + // if apiStatus.ID == gtsStatus.ID { + // haveAlready = true + // break + // } + // } + // if haveAlready { + // l.Debugf("\n we have status with id %d already so continuing past this iteration of the loop \n", gtsStatus.ID) + // continue + // } + + // pull relevant accounts from the status -- we need this both for checking visibility and for serializing + relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(gtsStatus) if err != nil { - if _, ok := err.(db.ErrNoEntries); !ok { - return nil, gtserror.NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error getting statuses from db: %s", err)) - } - l.Debug("\n breaking from grabloop because no statuses were returned \n") - break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail + continue + } + visible, err := p.db.StatusVisible(gtsStatus, authed.Account, relevantAccounts) + if err != nil { + continue } - for _, gtsStatus := range gtsStatuses { - // haveAlready := false - // for _, apiStatus := range apiStatuses { - // if apiStatus.ID == gtsStatus.ID { - // haveAlready = true - // break - // } - // } - // if haveAlready { - // l.Debugf("\n we have status with id %d already so continuing past this iteration of the loop \n", gtsStatus.ID) - // continue - // } - - // pull relevant accounts from the status -- we need this both for checking visibility and for serializing - relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(gtsStatus) - if err != nil { - continue - } - visible, err := p.db.StatusVisible(gtsStatus, authed.Account, relevantAccounts) - if err != nil { - continue - } - - if visible { - // check if this is a boost... - var reblogOfStatus *gtsmodel.Status - if gtsStatus.BoostOfID != "" { - s := >smodel.Status{} - if err := p.db.GetByID(s.BoostOfID, s); err != nil { - continue - } - reblogOfStatus = s - } - - // serialize the status (or, at least, convert it to a form that's ready to be serialized) - apiStatus, err := p.tc.StatusToMasto(gtsStatus, relevantAccounts.StatusAuthor, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus) - if err != nil { + if visible { + // check if this is a boost... + var reblogOfStatus *gtsmodel.Status + if gtsStatus.BoostOfID != "" { + s := >smodel.Status{} + if err := p.db.GetByID(s.BoostOfID, s); err != nil { continue } - - l.Debug("\n appending to the statuses slice \n") - apiStatuses = append(apiStatuses, apiStatus) - sort.Slice(apiStatuses, func(i int, j int) bool { - is, err := time.Parse(time.RFC3339, apiStatuses[i].CreatedAt) - if err != nil { - panic(err) - } - - js, err := time.Parse(time.RFC3339, apiStatuses[j].CreatedAt) - if err != nil { - panic(err) - } - - return is.After(js) - }) - - if len(apiStatuses) == limit { - l.Debugf("\n we have enough statuses, returning \n") - // we have enough - break grabloop - } + reblogOfStatus = s } - if len(apiStatuses) != 0 { - if maxIDMarker != "" { - maxIDMarker = apiStatuses[len(apiStatuses)-1].ID + + // serialize the status (or, at least, convert it to a form that's ready to be serialized) + apiStatus, err := p.tc.StatusToMasto(gtsStatus, relevantAccounts.StatusAuthor, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus) + if err != nil { + continue + } + + l.Debug("\n appending to the statuses slice \n") + apiStatuses = append(apiStatuses, apiStatus) + sort.Slice(apiStatuses, func(i int, j int) bool { + is, err := time.Parse(time.RFC3339, apiStatuses[i].CreatedAt) + if err != nil { + panic(err) } - if minIDMarker != "" { - minIDMarker = apiStatuses[0].ID + + js, err := time.Parse(time.RFC3339, apiStatuses[j].CreatedAt) + if err != nil { + panic(err) } + + return is.After(js) + }) + + if len(apiStatuses) == limit { + l.Debugf("\n we have enough statuses, returning \n") + // we have enough + break + } + } + if len(apiStatuses) != 0 { + if maxIDMarker != "" { + maxIDMarker = apiStatuses[len(apiStatuses)-1].ID + } + if minIDMarker != "" { + minIDMarker = apiStatuses[0].ID } } } diff --git a/internal/util/uri.go b/internal/util/uri.go index f86f224..b046338 100644 --- a/internal/util/uri.go +++ b/internal/util/uri.go @@ -21,7 +21,6 @@ package util import ( "fmt" "net/url" - "strings" ) const ( @@ -162,47 +161,47 @@ func GenerateURIsForAccount(username string, protocol string, host string) *User // IsUserPath returns true if the given URL path corresponds to eg /users/example_username func IsUserPath(id *url.URL) bool { - return userPathRegex.MatchString(strings.ToLower(id.Path)) + return userPathRegex.MatchString(id.Path) } // IsInboxPath returns true if the given URL path corresponds to eg /users/example_username/inbox func IsInboxPath(id *url.URL) bool { - return inboxPathRegex.MatchString(strings.ToLower(id.Path)) + return inboxPathRegex.MatchString(id.Path) } // IsOutboxPath returns true if the given URL path corresponds to eg /users/example_username/outbox func IsOutboxPath(id *url.URL) bool { - return outboxPathRegex.MatchString(strings.ToLower(id.Path)) + return outboxPathRegex.MatchString(id.Path) } // IsInstanceActorPath returns true if the given URL path corresponds to eg /actors/example_username func IsInstanceActorPath(id *url.URL) bool { - return actorPathRegex.MatchString(strings.ToLower(id.Path)) + return actorPathRegex.MatchString(id.Path) } // IsFollowersPath returns true if the given URL path corresponds to eg /users/example_username/followers func IsFollowersPath(id *url.URL) bool { - return followersPathRegex.MatchString(strings.ToLower(id.Path)) + return followersPathRegex.MatchString(id.Path) } // IsFollowingPath returns true if the given URL path corresponds to eg /users/example_username/following func IsFollowingPath(id *url.URL) bool { - return followingPathRegex.MatchString(strings.ToLower(id.Path)) + return followingPathRegex.MatchString(id.Path) } // IsLikedPath returns true if the given URL path corresponds to eg /users/example_username/liked func IsLikedPath(id *url.URL) bool { - return likedPathRegex.MatchString(strings.ToLower(id.Path)) + return likedPathRegex.MatchString(id.Path) } // IsLikePath returns true if the given URL path corresponds to eg /users/example_username/liked/SOME_ULID_OF_A_STATUS func IsLikePath(id *url.URL) bool { - return likePathRegex.MatchString(strings.ToLower(id.Path)) + return likePathRegex.MatchString(id.Path) } // IsStatusesPath returns true if the given URL path corresponds to eg /users/example_username/statuses/SOME_ULID_OF_A_STATUS func IsStatusesPath(id *url.URL) bool { - return statusesPathRegex.MatchString(strings.ToLower(id.Path)) + return statusesPathRegex.MatchString(id.Path) } // ParseStatusesPath returns the username and ulid from a path such as /users/example_username/statuses/SOME_ULID_OF_A_STATUS