From 625e6d654c44f5e708c19d3efff33efec20c0043 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Mon, 7 Jun 2021 20:20:36 +0200 Subject: [PATCH] more timeline stuff --- internal/api/client/timeline/home.go | 7 +- internal/api/model/timeline.go | 6 ++ internal/db/db.go | 2 +- internal/db/pg/pg.go | 46 ++++++----- internal/processing/processor.go | 2 +- internal/processing/timeline.go | 110 +++++++++++++++++++++++---- internal/router/router.go | 12 +-- 7 files changed, 139 insertions(+), 46 deletions(-) create mode 100644 internal/api/model/timeline.go diff --git a/internal/api/client/timeline/home.go b/internal/api/client/timeline/home.go index 977a464..86606a0 100644 --- a/internal/api/client/timeline/home.go +++ b/internal/api/client/timeline/home.go @@ -87,12 +87,13 @@ func (m *Module) HomeTimelineGETHandler(c *gin.Context) { local = i } - statuses, errWithCode := m.processor.HomeTimelineGet(authed, maxID, sinceID, minID, limit, local) + resp, errWithCode := m.processor.HomeTimelineGet(authed, maxID, sinceID, minID, limit, local) if errWithCode != nil { - l.Debugf("error from processor account statuses get: %s", errWithCode) + l.Debugf("error from processor HomeTimelineGet: %s", errWithCode) c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()}) return } - c.JSON(http.StatusOK, statuses) + c.Header("Link", resp.LinkHeader) + c.JSON(http.StatusOK, resp.Statuses) } diff --git a/internal/api/model/timeline.go b/internal/api/model/timeline.go new file mode 100644 index 0000000..f4f7880 --- /dev/null +++ b/internal/api/model/timeline.go @@ -0,0 +1,6 @@ +package model + +type StatusTimelineResponse struct { + Statuses []*Status + LinkHeader string +} diff --git a/internal/db/db.go b/internal/db/db.go index dd51331..51685f0 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -256,7 +256,7 @@ type DB interface { WhoBoostedStatus(status *gtsmodel.Status) ([]*gtsmodel.Account, error) // GetStatusesWhereFollowing returns a slice of statuses from accounts that are followed by the given account id. - GetStatusesWhereFollowing(accountID string, limit int, maxID string, minID string, sinceID string) ([]*gtsmodel.Status, error) + GetStatusesWhereFollowing(accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*gtsmodel.Status, error) // GetPublicTimelineForAccount fetches the account's PUBLIC timline -- ie., posts and replies that are public. // It will use the given filters and try to return as many statuses as possible up to the limit. diff --git a/internal/db/pg/pg.go b/internal/db/pg/pg.go index e2002db..44657aa 100644 --- a/internal/db/pg/pg.go +++ b/internal/db/pg/pg.go @@ -1133,7 +1133,7 @@ func (ps *postgresService) WhoBoostedStatus(status *gtsmodel.Status) ([]*gtsmode return accounts, nil } -func (ps *postgresService) GetStatusesWhereFollowing(accountID string, limit int, offsetStatusID string, includeOffsetStatus bool, ascending bool) ([]*gtsmodel.Status, error) { +func (ps *postgresService) GetStatusesWhereFollowing(accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*gtsmodel.Status, error) { statuses := []*gtsmodel.Status{} q := ps.conn.Model(&statuses) @@ -1142,23 +1142,35 @@ func (ps *postgresService) GetStatusesWhereFollowing(accountID string, limit int Join("JOIN follows AS f ON f.target_account_id = status.account_id"). Where("f.account_id = ?", accountID) - if ascending { + if maxID != "" { + s := >smodel.Status{} + if err := ps.conn.Model(s).Where("id = ?", maxID).Select(); err == nil { + q = q.Where("status.created_at < ?", s.CreatedAt) + } + } + + if sinceID != "" { + s := >smodel.Status{} + if err := ps.conn.Model(s).Where("id = ?", sinceID).Select(); err == nil { + q = q.Where("status.created_at > ?", s.CreatedAt) + } + } + + if minID != "" { + s := >smodel.Status{} + if err := ps.conn.Model(s).Where("id = ?", minID).Select(); err == nil { + q = q.Where("status.created_at > ?", s.CreatedAt) + } + } + + if minID != "" { q = q.Order("status.created_at") } else { q = q.Order("status.created_at DESC") } - s := >smodel.Status{} - if offsetStatusID != "" { - if err := ps.conn.Model(s).Where("id = ?", offsetStatusID).Select(); err != nil { - return nil, err - } - - if ascending { - q = q.Where("status.created_at > ?", s.CreatedAt) - } else { - q = q.Where("status.created_at < ?", s.CreatedAt) - } + if local { + q = q.Where("status.local = ?", local) } if limit > 0 { @@ -1172,14 +1184,6 @@ func (ps *postgresService) GetStatusesWhereFollowing(accountID string, limit int } } - if includeOffsetStatus { - if ascending { - statuses = append([]*gtsmodel.Status{s}, statuses...) - } else { - statuses = append(statuses, s) - } - } - return statuses, nil } diff --git a/internal/processing/processor.go b/internal/processing/processor.go index 8d263e3..1ccf71e 100644 --- a/internal/processing/processor.go +++ b/internal/processing/processor.go @@ -127,7 +127,7 @@ type Processor interface { StatusGetContext(authed *oauth.Auth, targetStatusID string) (*apimodel.Context, gtserror.WithCode) // HomeTimelineGet returns statuses from the home timeline, with the given filters/parameters. - HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, gtserror.WithCode) + HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.StatusTimelineResponse, gtserror.WithCode) // PublicTimelineGet returns statuses from the public/local timeline, with the given filters/parameters. PublicTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, gtserror.WithCode) diff --git a/internal/processing/timeline.go b/internal/processing/timeline.go index 0d984ed..204d7d8 100644 --- a/internal/processing/timeline.go +++ b/internal/processing/timeline.go @@ -20,7 +20,10 @@ package processing import ( "fmt" + "net/url" + "sort" "sync" + "time" "github.com/sirupsen/logrus" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" @@ -30,27 +33,58 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/oauth" ) -func (p *processor) HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, gtserror.WithCode) { +func (p *processor) HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.StatusTimelineResponse, gtserror.WithCode) { + l := p.log.WithFields(logrus.Fields{ + "func": "HomeTimelineGet", + "maxID": maxID, + "sinceID": sinceID, + "minID": minID, + "limit": limit, + "local": local, + }) - statuses := []*apimodel.Status{} + resp := &apimodel.StatusTimelineResponse{ + Statuses: []*apimodel.Status{}, + } + apiStatuses := []*apimodel.Status{} + + maxIDMarker := maxID + sinceIDMarker := sinceID + minIDMarker := minID + +l.Debugf("\n entering grabloop \n") grabloop: - for len(statuses) < limit { - gtsStatuses, err := p.db.GetStatusesWhereFollowing(authed.Account.ID, limit, maxID, false, false) - + for len(apiStatuses) < limit { + l.Debugf("\n querying the db \n") + gtsStatuses, err := p.db.GetStatusesWhereFollowing(authed.Account.ID, maxIDMarker, sinceIDMarker, minIDMarker, limit, local) if err != nil { if _, ok := err.(db.ErrNoEntries); !ok { return nil, gtserror.NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error getting statuses from db: %s", err)) } + l.Debug("\n breaking from grabloop because no statuses were returned \n") break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail } - for _, s := range gtsStatuses { - relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(s) + for _, gtsStatus := range gtsStatuses { + // haveAlready := false + // for _, apiStatus := range apiStatuses { + // if apiStatus.ID == gtsStatus.ID { + // haveAlready = true + // break + // } + // } + // if haveAlready { + // l.Debugf("\n we have status with id %d already so continuing past this iteration of the loop \n", gtsStatus.ID) + // continue + // } + + // pull relevant accounts from the status -- we need this both for checking visibility and for serializing + relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(gtsStatus) if err != nil { continue } - visible, err := p.db.StatusVisible(s, authed.Account, relevantAccounts) + visible, err := p.db.StatusVisible(gtsStatus, authed.Account, relevantAccounts) if err != nil { continue } @@ -58,7 +92,7 @@ grabloop: if visible { // check if this is a boost... var reblogOfStatus *gtsmodel.Status - if s.BoostOfID != "" { + if gtsStatus.BoostOfID != "" { s := >smodel.Status{} if err := p.db.GetByID(s.BoostOfID, s); err != nil { continue @@ -67,21 +101,67 @@ grabloop: } // serialize the status (or, at least, convert it to a form that's ready to be serialized) - apiModelStatus, err := p.tc.StatusToMasto(s, relevantAccounts.StatusAuthor, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus) + apiStatus, err := p.tc.StatusToMasto(gtsStatus, relevantAccounts.StatusAuthor, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus) if err != nil { continue } - statuses = append(statuses, apiModelStatus) - if len(statuses) == limit { + l.Debug("\n appending to the statuses slice \n") + apiStatuses = append(apiStatuses, apiStatus) + sort.Slice(apiStatuses, func(i int, j int) bool { + is, err := time.Parse(time.RFC3339, apiStatuses[i].CreatedAt) + if err != nil { + panic(err) + } + + js, err := time.Parse(time.RFC3339, apiStatuses[j].CreatedAt) + if err != nil { + panic(err) + } + + return is.After(js) + }) + + if len(apiStatuses) == limit { + l.Debugf("\n we have enough statuses, returning \n") // we have enough break grabloop } } + if len(apiStatuses) != 0 { + if maxIDMarker != "" { + maxIDMarker = apiStatuses[len(apiStatuses)-1].ID + } + if minIDMarker != "" { + minIDMarker = apiStatuses[0].ID + } + } } } - return statuses, nil + resp.Statuses = apiStatuses + + if len(resp.Statuses) != 0 { + nextLink := &url.URL{ + Scheme: p.config.Protocol, + Host: p.config.Host, + Path: "/api/v1/timelines/home", + RawPath: url.PathEscape("api/v1/timelines/home"), + RawQuery: url.QueryEscape(fmt.Sprintf("limit=%d&max_id=%s", limit, apiStatuses[len(apiStatuses)-1].ID)), + } + next := fmt.Sprintf("<%s>; rel=\"next\"", nextLink.String()) + + prevLink := &url.URL{ + Scheme: p.config.Protocol, + Host: p.config.Host, + Path: "/api/v1/timelines/home", + RawQuery: fmt.Sprintf("limit=%d&min_id=%s", limit, apiStatuses[0].ID), + } + prev := fmt.Sprintf("<%s>; rel=\"prev\"", prevLink.String()) + resp.LinkHeader = fmt.Sprintf("%s, %s", next, prev) + } + + return resp, nil } func (p *processor) PublicTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, gtserror.WithCode) { @@ -207,7 +287,7 @@ func (p *processor) initTimelineFor(account *gtsmodel.Account, wg *sync.WaitGrou desiredIndexLength := p.timelineManager.GetDesiredIndexLength() - statuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength, "", false, false) + statuses, err := p.db.GetStatusesWhereFollowing(account.ID, "", "", "", desiredIndexLength, false) if err != nil { l.Error(fmt.Errorf("initTimelineFor: error getting statuses: %s", err)) return @@ -224,7 +304,7 @@ func (p *processor) initTimelineFor(account *gtsmodel.Account, wg *sync.WaitGrou } if rearmostStatusID != "" { - moreStatuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength/2, rearmostStatusID, false, false) + moreStatuses, err := p.db.GetStatusesWhereFollowing(account.ID, rearmostStatusID, "", "", desiredIndexLength/2, false) if err != nil { l.Error(fmt.Errorf("initTimelineFor: error getting more statuses: %s", err)) return diff --git a/internal/router/router.go b/internal/router/router.go index 3e0435e..adfd2ce 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -125,11 +125,13 @@ func New(config *config.Config, logger *logrus.Logger) (Router, error) { // create the actual engine here -- this is the core request routing handler for gts engine := gin.Default() engine.Use(cors.New(cors.Config{ - AllowAllOrigins: true, - AllowMethods: []string{"GET", "POST", "PUT", "PATCH", "DELETE"}, - AllowHeaders: []string{"Origin", "Content-Length", "Content-Type", "Authorization"}, - AllowCredentials: false, - MaxAge: 12 * time.Hour, + AllowAllOrigins: true, + AllowBrowserExtensions: true, + AllowMethods: []string{"POST", "PUT", "DELETE", "GET", "PATCH", "OPTIONS"}, + AllowHeaders: []string{"Origin", "Content-Length", "Content-Type", "Authorization"}, + AllowWebSockets: true, + ExposeHeaders: []string{"Link", "X-RateLimit-Reset", "X-RateLimit-Limit", " X-RateLimit-Remaining", "X-Request-Id"}, + MaxAge: 2 * time.Minute, })) engine.MaxMultipartMemory = 8 << 20 // 8 MiB