diff --git a/internal/db/db.go b/internal/db/db.go index b5c3266..dd51331 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -255,7 +255,8 @@ type DB interface { // This slice will be unfiltered, not taking account of blocks and whatnot, so filter it before serving it back to a user. WhoBoostedStatus(status *gtsmodel.Status) ([]*gtsmodel.Account, error) - GetStatusesWhereFollowing(accountID string, limit int, offsetStatusID string) ([]*gtsmodel.Status, error) + // GetStatusesWhereFollowing returns a slice of statuses from accounts that are followed by the given account id. + GetStatusesWhereFollowing(accountID string, limit int, maxID string, minID string, sinceID string) ([]*gtsmodel.Status, error) // GetPublicTimelineForAccount fetches the account's PUBLIC timline -- ie., posts and replies that are public. // It will use the given filters and try to return as many statuses as possible up to the limit. diff --git a/internal/db/pg/pg.go b/internal/db/pg/pg.go index 788a330..e2002db 100644 --- a/internal/db/pg/pg.go +++ b/internal/db/pg/pg.go @@ -788,7 +788,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti // if target account is suspended then don't show the status if !targetAccount.SuspendedAt.IsZero() { - l.Debug("target account suspended at is not zero") + l.Trace("target account suspended at is not zero") return false, nil } @@ -807,7 +807,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti // if target user is disabled, not yet approved, or not confirmed then don't show the status // (although in the latter two cases it's unlikely they posted a status yet anyway, but you never know!) if targetUser.Disabled || !targetUser.Approved || targetUser.ConfirmedAt.IsZero() { - l.Debug("target user is disabled, not approved, or not confirmed") + l.Trace("target user is disabled, not approved, or not confirmed") return false, nil } } @@ -818,14 +818,14 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti if targetStatus.Visibility == gtsmodel.VisibilityPublic { return true, nil } - l.Debug("requesting account is nil but the target status isn't public") + l.Trace("requesting account is nil but the target status isn't public") return false, nil } // if requesting account is suspended then don't show the status -- although they probably shouldn't have gotten // this far (ie., been authed) in the first place: this is just for safety. if !requestingAccount.SuspendedAt.IsZero() { - l.Debug("requesting account is suspended") + l.Trace("requesting account is suspended") return false, nil } @@ -843,7 +843,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti } // okay, user exists, so make sure it has full privileges/is confirmed/approved if requestingUser.Disabled || !requestingUser.Approved || requestingUser.ConfirmedAt.IsZero() { - l.Debug("requesting account is local but corresponding user is either disabled, not approved, or not confirmed") + l.Trace("requesting account is local but corresponding user is either disabled, not approved, or not confirmed") return false, nil } } @@ -860,7 +860,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti return false, err } else if blocked { // don't allow the status to be viewed if a block exists in *either* direction between these two accounts, no creepy stalking please - l.Debug("a block exists between requesting account and target account") + l.Trace("a block exists between requesting account and target account") return false, nil } @@ -871,7 +871,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti if blocked, err := ps.Blocked(relevantAccounts.ReplyToAccount.ID, requestingAccount.ID); err != nil { return false, err } else if blocked { - l.Debug("a block exists between requesting account and reply to account") + l.Trace("a block exists between requesting account and reply to account") return false, nil } @@ -882,7 +882,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti return false, err } if !followsRepliedAccount { - l.Debug("target status is a followers-only reply to an account that is not followed by the requesting account") + l.Trace("target status is a followers-only reply to an account that is not followed by the requesting account") return false, nil } } @@ -893,7 +893,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti if blocked, err := ps.Blocked(relevantAccounts.BoostedAccount.ID, requestingAccount.ID); err != nil { return false, err } else if blocked { - l.Debug("a block exists between requesting account and boosted account") + l.Trace("a block exists between requesting account and boosted account") return false, nil } } @@ -903,7 +903,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti if blocked, err := ps.Blocked(relevantAccounts.BoostedReplyToAccount.ID, requestingAccount.ID); err != nil { return false, err } else if blocked { - l.Debug("a block exists between requesting account and boosted reply to account") + l.Trace("a block exists between requesting account and boosted reply to account") return false, nil } } @@ -913,7 +913,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti if blocked, err := ps.Blocked(a.ID, requestingAccount.ID); err != nil { return false, err } else if blocked { - l.Debug("a block exists between requesting account and a mentioned account") + l.Trace("a block exists between requesting account and a mentioned account") return false, nil } } @@ -939,7 +939,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti return false, err } if !follows { - l.Debug("requested status is followers only but requesting account is not a follower") + l.Trace("requested status is followers only but requesting account is not a follower") return false, nil } return true, nil @@ -950,12 +950,12 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti return false, err } if !mutuals { - l.Debug("requested status is mutuals only but accounts aren't mufos") + l.Trace("requested status is mutuals only but accounts aren't mufos") return false, nil } return true, nil case gtsmodel.VisibilityDirect: - l.Debug("requesting account requests a status it's not mentioned in") + l.Trace("requesting account requests a status it's not mentioned in") return false, nil // it's not mentioned -_- } @@ -1133,22 +1133,32 @@ func (ps *postgresService) WhoBoostedStatus(status *gtsmodel.Status) ([]*gtsmode return accounts, nil } -func (ps *postgresService) GetStatusesWhereFollowing(accountID string, limit int, offsetStatusID string) ([]*gtsmodel.Status, error) { +func (ps *postgresService) GetStatusesWhereFollowing(accountID string, limit int, offsetStatusID string, includeOffsetStatus bool, ascending bool) ([]*gtsmodel.Status, error) { statuses := []*gtsmodel.Status{} q := ps.conn.Model(&statuses) q = q.ColumnExpr("status.*"). Join("JOIN follows AS f ON f.target_account_id = status.account_id"). - Where("f.account_id = ?", accountID). - Order("status.created_at DESC") + Where("f.account_id = ?", accountID) + if ascending { + q = q.Order("status.created_at") + } else { + q = q.Order("status.created_at DESC") + } + + s := >smodel.Status{} if offsetStatusID != "" { - s := >smodel.Status{} if err := ps.conn.Model(s).Where("id = ?", offsetStatusID).Select(); err != nil { return nil, err } - q = q.Where("status.created_at < ?", s.CreatedAt) + + if ascending { + q = q.Where("status.created_at > ?", s.CreatedAt) + } else { + q = q.Where("status.created_at < ?", s.CreatedAt) + } } if limit > 0 { @@ -1162,6 +1172,14 @@ func (ps *postgresService) GetStatusesWhereFollowing(accountID string, limit int } } + if includeOffsetStatus { + if ascending { + statuses = append([]*gtsmodel.Status{s}, statuses...) + } else { + statuses = append(statuses, s) + } + } + return statuses, nil } diff --git a/internal/processing/timeline.go b/internal/processing/timeline.go index 387972c..0d984ed 100644 --- a/internal/processing/timeline.go +++ b/internal/processing/timeline.go @@ -32,9 +32,53 @@ import ( func (p *processor) HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, gtserror.WithCode) { - statuses, err := p.timelineManager.HomeTimeline(authed.Account.ID, maxID, sinceID, minID, limit, local) - if err != nil { - return nil, gtserror.NewErrorInternalError(err) + statuses := []*apimodel.Status{} + +grabloop: + for len(statuses) < limit { + gtsStatuses, err := p.db.GetStatusesWhereFollowing(authed.Account.ID, limit, maxID, false, false) + + if err != nil { + if _, ok := err.(db.ErrNoEntries); !ok { + return nil, gtserror.NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error getting statuses from db: %s", err)) + } + break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail + } + + for _, s := range gtsStatuses { + relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(s) + if err != nil { + continue + } + visible, err := p.db.StatusVisible(s, authed.Account, relevantAccounts) + if err != nil { + continue + } + + if visible { + // check if this is a boost... + var reblogOfStatus *gtsmodel.Status + if s.BoostOfID != "" { + s := >smodel.Status{} + if err := p.db.GetByID(s.BoostOfID, s); err != nil { + continue + } + reblogOfStatus = s + } + + // serialize the status (or, at least, convert it to a form that's ready to be serialized) + apiModelStatus, err := p.tc.StatusToMasto(s, relevantAccounts.StatusAuthor, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus) + if err != nil { + continue + } + + statuses = append(statuses, apiModelStatus) + if len(statuses) == limit { + // we have enough + break grabloop + } + } + } } return statuses, nil @@ -163,7 +207,7 @@ func (p *processor) initTimelineFor(account *gtsmodel.Account, wg *sync.WaitGrou desiredIndexLength := p.timelineManager.GetDesiredIndexLength() - statuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength, "") + statuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength, "", false, false) if err != nil { l.Error(fmt.Errorf("initTimelineFor: error getting statuses: %s", err)) return @@ -180,7 +224,7 @@ func (p *processor) initTimelineFor(account *gtsmodel.Account, wg *sync.WaitGrou } if rearmostStatusID != "" { - moreStatuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength/2, rearmostStatusID) + moreStatuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength/2, rearmostStatusID, false, false) if err != nil { l.Error(fmt.Errorf("initTimelineFor: error getting more statuses: %s", err)) return diff --git a/internal/timeline/get.go b/internal/timeline/get.go new file mode 100644 index 0000000..8ad70c9 --- /dev/null +++ b/internal/timeline/get.go @@ -0,0 +1,250 @@ +package timeline + +import ( + "container/list" + "errors" + "fmt" + + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" +) + +func (t *timeline) GetXFromTop(amount int) ([]*apimodel.Status, error) { + // make a slice of statuses with the length we need to return + statuses := make([]*apimodel.Status, 0, amount) + + // if there are no prepared posts, just return the empty slice + if t.preparedPosts.data == nil { + t.preparedPosts.data = &list.List{} + } + + // make sure we have enough posts prepared to return + if t.preparedPosts.data.Len() < amount { + if err := t.PrepareFromTop(amount); err != nil { + return nil, err + } + } + + // work through the prepared posts from the top and return + var served int + for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() { + entry, ok := e.Value.(*preparedPostsEntry) + if !ok { + return nil, errors.New("GetXFromTop: could not parse e as a preparedPostsEntry") + } + statuses = append(statuses, entry.prepared) + served = served + 1 + if served >= amount { + break + } + } + + return statuses, nil +} + +func (t *timeline) GetXBehindID(amount int, behindID string) ([]*apimodel.Status, error) { + // make a slice of statuses with the length we need to return + statuses := make([]*apimodel.Status, 0, amount) + + // if there are no prepared posts, just return the empty slice + if t.preparedPosts.data == nil { + t.preparedPosts.data = &list.List{} + } + + // iterate through the modified list until we hit the mark we're looking for + var position int + var behindIDMark *list.Element +findMarkLoop: + for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() { + position = position + 1 + entry, ok := e.Value.(*preparedPostsEntry) + if !ok { + return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry") + } + + if entry.statusID == behindID { + fmt.Printf("\n\n\n GETXBEHINDID: FOUND BEHINDID %s WITH POSITION %d AND CREATEDAT %s \n\n\n", behindID, position, entry.createdAt.String()) + behindIDMark = e + break findMarkLoop + } + } + + // we didn't find it, so we need to make sure it's indexed and prepared and then try again + if behindIDMark == nil { + if err := t.IndexBehind(behindID, true, amount); err != nil { + return nil, fmt.Errorf("GetXBehindID: error indexing behind and including ID %s", behindID) + } + if err := t.PrepareBehind(behindID, true, amount); err != nil { + return nil, fmt.Errorf("GetXBehindID: error preparing behind and including ID %s", behindID) + } + return t.GetXBehindID(amount, behindID) + } + + // make sure we have enough posts prepared behind it to return what we're being asked for + if t.preparedPosts.data.Len() < amount+position { + fmt.Printf("\n\n\n GETXBEHINDID: PREPARED POSTS LENGTH %d WAS LESS THAN AMOUNT %d PLUS POSITION %d", t.preparedPosts.data.Len(), amount, position) + if err := t.PrepareBehind(behindID, false, amount); err != nil { + return nil, err + } + fmt.Printf("\n\n\n GETXBEHINDID: PREPARED POSTS LENGTH IS NOW %d", t.preparedPosts.data.Len()) + } + + // start serving from the entry right after the mark + var served int +serveloop: + for e := behindIDMark.Next(); e != nil; e = e.Next() { + entry, ok := e.Value.(*preparedPostsEntry) + if !ok { + return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry") + } + + fmt.Printf("\n\n\n GETXBEHINDID: SERVING STATUS ID %s WITH CREATEDAT %s \n\n\n", entry.statusID, entry.createdAt.String()) + // serve up to the amount requested + statuses = append(statuses, entry.prepared) + served = served + 1 + if served >= amount { + break serveloop + } + } + + return statuses, nil +} + +func (t *timeline) GetXBeforeID(amount int, beforeID string, startFromTop bool) ([]*apimodel.Status, error) { + // make a slice of statuses with the length we need to return + statuses := make([]*apimodel.Status, 0, amount) + + if t.preparedPosts.data == nil { + t.preparedPosts.data = &list.List{} + } + + // iterate through the modified list until we hit the mark we're looking for + var beforeIDMark *list.Element +findMarkLoop: + for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() { + entry, ok := e.Value.(*preparedPostsEntry) + if !ok { + return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry") + } + + if entry.statusID == beforeID { + beforeIDMark = e + break findMarkLoop + } + } + + // we didn't find it, so we need to make sure it's indexed and prepared and then try again + if beforeIDMark == nil { + if err := t.IndexBefore(beforeID, true, amount); err != nil { + return nil, fmt.Errorf("GetXBeforeID: error indexing before and including ID %s", beforeID) + } + if err := t.PrepareBefore(beforeID, true, amount); err != nil { + return nil, fmt.Errorf("GetXBeforeID: error preparing before and including ID %s", beforeID) + } + return t.GetXBeforeID(amount, beforeID, startFromTop) + } + + var served int + + if startFromTop { + // start serving from the front/top and keep going until we hit mark or get x amount statuses + serveloopFromTop: + for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() { + entry, ok := e.Value.(*preparedPostsEntry) + if !ok { + return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry") + } + + if entry.statusID == beforeID { + break serveloopFromTop + } + + // serve up to the amount requested + statuses = append(statuses, entry.prepared) + served = served + 1 + if served >= amount { + break serveloopFromTop + } + } + + } else if startFromTop { + // start serving from the entry right before the mark + serveloopFromBottom: + for e := beforeIDMark.Prev(); e != nil; e = e.Prev() { + entry, ok := e.Value.(*preparedPostsEntry) + if !ok { + return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry") + } + + // serve up to the amount requested + statuses = append(statuses, entry.prepared) + served = served + 1 + if served >= amount { + break serveloopFromBottom + } + } + } + + return statuses, nil +} + +func (t *timeline) GetXBetweenID(amount int, behindID string, beforeID string) ([]*apimodel.Status, error) { + // make a slice of statuses with the length we need to return + statuses := make([]*apimodel.Status, 0, amount) + + // if there are no prepared posts, just return the empty slice + if t.preparedPosts.data == nil { + t.preparedPosts.data = &list.List{} + } + + // iterate through the modified list until we hit the mark we're looking for + var position int + var behindIDMark *list.Element +findMarkLoop: + for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() { + position = position + 1 + entry, ok := e.Value.(*preparedPostsEntry) + if !ok { + return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry") + } + + if entry.statusID == behindID { + behindIDMark = e + break findMarkLoop + } + } + + // we didn't find it + if behindIDMark == nil { + return nil, fmt.Errorf("GetXBetweenID: couldn't find status with ID %s", behindID) + } + + // make sure we have enough posts prepared behind it to return what we're being asked for + if t.preparedPosts.data.Len() < amount+position { + if err := t.PrepareBehind(behindID, false, amount); err != nil { + return nil, err + } + } + + // start serving from the entry right after the mark + var served int +serveloop: + for e := behindIDMark.Next(); e != nil; e = e.Next() { + entry, ok := e.Value.(*preparedPostsEntry) + if !ok { + return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry") + } + + if entry.statusID == beforeID { + break serveloop + } + + // serve up to the amount requested + statuses = append(statuses, entry.prepared) + served = served + 1 + if served >= amount { + break serveloop + } + } + + return statuses, nil +} diff --git a/internal/timeline/index.go b/internal/timeline/index.go new file mode 100644 index 0000000..f8bd49d --- /dev/null +++ b/internal/timeline/index.go @@ -0,0 +1,120 @@ +package timeline + +import ( + "fmt" + "time" +) + +func (t *timeline) IndexBefore(statusID string, include bool, amount int) error { + // filtered := []*gtsmodel.Status{} + // offsetStatus := statusID + + // grabloop: + // for len(filtered) < amount { + // statuses, err := t.db.GetStatusesWhereFollowing(t.accountID, amount, offsetStatus, include, true) + // if err != nil { + // if _, ok := err.(db.ErrNoEntries); !ok { + // return fmt.Errorf("IndexBeforeAndIncluding: error getting statuses from db: %s", err) + // } + // break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail + // } + + // for _, s := range statuses { + // relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(s) + // if err != nil { + // continue + // } + // visible, err := t.db.StatusVisible(s, t.account, relevantAccounts) + // if err != nil { + // continue + // } + // if visible { + // filtered = append(filtered, s) + // } + // offsetStatus = s.ID + // } + // } + + // for _, s := range filtered { + // if err := t.IndexOne(s.CreatedAt, s.ID); err != nil { + // return fmt.Errorf("IndexBeforeAndIncluding: error indexing status with id %s: %s", s.ID, err) + // } + // } + + return nil +} + +func (t *timeline) IndexBehind(statusID string, include bool, amount int) error { + // filtered := []*gtsmodel.Status{} + // offsetStatus := statusID + + // grabloop: + // for len(filtered) < amount { + // statuses, err := t.db.GetStatusesWhereFollowing(t.accountID, amount, offsetStatus, include, false) + // if err != nil { + // if _, ok := err.(db.ErrNoEntries); !ok { + // return fmt.Errorf("IndexBehindAndIncluding: error getting statuses from db: %s", err) + // } + // break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail + // } + + // for _, s := range statuses { + // relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(s) + // if err != nil { + // continue + // } + // visible, err := t.db.StatusVisible(s, t.account, relevantAccounts) + // if err != nil { + // continue + // } + // if visible { + // filtered = append(filtered, s) + // } + // offsetStatus = s.ID + // } + // } + + // for _, s := range filtered { + // if err := t.IndexOne(s.CreatedAt, s.ID); err != nil { + // return fmt.Errorf("IndexBehindAndIncluding: error indexing status with id %s: %s", s.ID, err) + // } + // } + + return nil +} + +func (t *timeline) IndexOneByID(statusID string) error { + return nil +} + +func (t *timeline) IndexOne(statusCreatedAt time.Time, statusID string) error { + t.Lock() + defer t.Unlock() + + postIndexEntry := &postIndexEntry{ + createdAt: statusCreatedAt, + statusID: statusID, + } + + return t.postIndex.insertIndexed(postIndexEntry) +} + +func (t *timeline) IndexAndPrepareOne(statusCreatedAt time.Time, statusID string) error { + t.Lock() + defer t.Unlock() + + postIndexEntry := &postIndexEntry{ + createdAt: statusCreatedAt, + statusID: statusID, + } + + if err := t.postIndex.insertIndexed(postIndexEntry); err != nil { + return fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %s", err) + } + + if err := t.prepare(statusID); err != nil { + return fmt.Errorf("IndexAndPrepareOne: error preparing: %s", err) + } + + return nil +} diff --git a/internal/timeline/manager.go b/internal/timeline/manager.go index 80e5caf..b26526f 100644 --- a/internal/timeline/manager.go +++ b/internal/timeline/manager.go @@ -150,7 +150,7 @@ func (m *manager) HomeTimeline(timelineAccountID string, maxID string, sinceID s } else if maxID != "" { statuses, err = t.GetXBehindID(limit, maxID) } else if sinceID != "" { - statuses, err = t.GetXBeforeID(limit, sinceID) + statuses, err = t.GetXBeforeID(limit, sinceID, true) } else { statuses, err = t.GetXFromTop(limit) } @@ -180,7 +180,7 @@ func (m *manager) GetOldestIndexedID(timelineAccountID string) (string, error) { func (m *manager) PrepareXFromTop(timelineAccountID string, limit int) error { t := m.getOrCreateTimeline(timelineAccountID) - return t.PrepareXFromTop(limit) + return t.PrepareFromTop(limit) } func (m *manager) WipeStatusFromTimeline(timelineAccountID string, statusID string) (int, error) { diff --git a/internal/timeline/postindex.go b/internal/timeline/postindex.go index 86afcb5..609a196 100644 --- a/internal/timeline/postindex.go +++ b/internal/timeline/postindex.go @@ -16,7 +16,6 @@ type postIndexEntry struct { } func (p *postIndex) insertIndexed(i *postIndexEntry) error { - if p.data == nil { p.data = &list.List{} } @@ -27,20 +26,33 @@ func (p *postIndex) insertIndexed(i *postIndexEntry) error { return nil } - // we need to iterate through the index to make sure we put this post in the appropriate place according to when it was created + var insertMark *list.Element + // We need to iterate through the index to make sure we put this post in the appropriate place according to when it was created. + // We also need to make sure we're not inserting a duplicate post -- this can happen sometimes and it's not nice UX (*shudder*). for e := p.data.Front(); e != nil; e = e.Next() { entry, ok := e.Value.(*postIndexEntry) if !ok { - return errors.New("Remove: could not parse e as a postIndexEntry") + return errors.New("index: could not parse e as a postIndexEntry") } // if the post to index is newer than e, insert it before e in the list - if i.createdAt.After(entry.createdAt) { - p.data.InsertBefore(i, e) + if insertMark == nil { + if i.createdAt.After(entry.createdAt) { + insertMark = e + } + } + + // make sure we don't insert a duplicate + if entry.statusID == i.statusID { return nil } } + if insertMark != nil { + p.data.InsertBefore(i, insertMark) + return nil + } + // if we reach this point it's the oldest post we've seen so put it at the back p.data.PushBack(i) return nil diff --git a/internal/timeline/prepare.go b/internal/timeline/prepare.go new file mode 100644 index 0000000..11aa170 --- /dev/null +++ b/internal/timeline/prepare.go @@ -0,0 +1,180 @@ +package timeline + +import ( + "errors" + "fmt" + + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" +) + +func (t *timeline) PrepareBehind(statusID string, include bool, amount int) error { + t.Lock() + defer t.Unlock() + + var prepared int + var preparing bool +prepareloop: + for e := t.postIndex.data.Front(); e != nil; e = e.Next() { + entry, ok := e.Value.(*postIndexEntry) + if !ok { + return errors.New("PrepareBehind: could not parse e as a postIndexEntry") + } + + if !preparing { + // we haven't hit the position we need to prepare from yet + if entry.statusID == statusID { + preparing = true + if !include { + continue + } + } + } + + if preparing { + if err := t.prepare(entry.statusID); err != nil { + // there's been an error + if _, ok := err.(db.ErrNoEntries); !ok { + // it's a real error + return fmt.Errorf("PrepareBehind: error preparing status with id %s: %s", entry.statusID, err) + } + // the status just doesn't exist (anymore) so continue to the next one + continue + } + if prepared == amount { + // we're done + break prepareloop + } + prepared = prepared + 1 + } + } + + return nil +} + +func (t *timeline) PrepareBefore(statusID string, include bool, amount int) error { + t.Lock() + defer t.Unlock() + + var prepared int + var preparing bool +prepareloop: + for e := t.postIndex.data.Back(); e != nil; e = e.Prev() { + entry, ok := e.Value.(*postIndexEntry) + if !ok { + return errors.New("PrepareBefore: could not parse e as a postIndexEntry") + } + + if !preparing { + // we haven't hit the position we need to prepare from yet + if entry.statusID == statusID { + preparing = true + if !include { + continue + } + } + } + + if preparing { + if err := t.prepare(entry.statusID); err != nil { + // there's been an error + if _, ok := err.(db.ErrNoEntries); !ok { + // it's a real error + return fmt.Errorf("PrepareBefore: error preparing status with id %s: %s", entry.statusID, err) + } + // the status just doesn't exist (anymore) so continue to the next one + continue + } + if prepared == amount { + // we're done + break prepareloop + } + prepared = prepared + 1 + } + } + + return nil +} + +func (t *timeline) PrepareFromTop(amount int) error { + t.Lock() + defer t.Unlock() + + t.preparedPosts.data.Init() + + var prepared int +prepareloop: + for e := t.postIndex.data.Front(); e != nil; e = e.Next() { + entry, ok := e.Value.(*postIndexEntry) + if !ok { + return errors.New("PrepareFromTop: could not parse e as a postIndexEntry") + } + + if err := t.prepare(entry.statusID); err != nil { + // there's been an error + if _, ok := err.(db.ErrNoEntries); !ok { + // it's a real error + return fmt.Errorf("PrepareFromTop: error preparing status with id %s: %s", entry.statusID, err) + } + // the status just doesn't exist (anymore) so continue to the next one + continue + } + + prepared = prepared + 1 + if prepared == amount { + // we're done + break prepareloop + } + } + + return nil +} + +func (t *timeline) prepare(statusID string) error { + + // start by getting the status out of the database according to its indexed ID + gtsStatus := >smodel.Status{} + if err := t.db.GetByID(statusID, gtsStatus); err != nil { + return err + } + + // if the account pointer hasn't been set on this timeline already, set it lazily here + if t.account == nil { + timelineOwnerAccount := >smodel.Account{} + if err := t.db.GetByID(t.accountID, timelineOwnerAccount); err != nil { + return err + } + t.account = timelineOwnerAccount + } + + // to convert the status we need relevant accounts from it, so pull them out here + relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(gtsStatus) + if err != nil { + return err + } + + // check if this is a boost... + var reblogOfStatus *gtsmodel.Status + if gtsStatus.BoostOfID != "" { + s := >smodel.Status{} + if err := t.db.GetByID(gtsStatus.BoostOfID, s); err != nil { + return err + } + reblogOfStatus = s + } + + // serialize the status (or, at least, convert it to a form that's ready to be serialized) + apiModelStatus, err := t.tc.StatusToMasto(gtsStatus, relevantAccounts.StatusAuthor, t.account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus) + if err != nil { + return err + } + + // shove it in prepared posts as a prepared posts entry + preparedPostsEntry := &preparedPostsEntry{ + createdAt: gtsStatus.CreatedAt, + statusID: statusID, + prepared: apiModelStatus, + } + + return t.preparedPosts.insertPrepared(preparedPostsEntry) +} diff --git a/internal/timeline/preparedposts.go b/internal/timeline/preparedposts.go index 50a85a2..159e6b4 100644 --- a/internal/timeline/preparedposts.go +++ b/internal/timeline/preparedposts.go @@ -29,7 +29,9 @@ func (p *preparedPosts) insertPrepared(i *preparedPostsEntry) error { return nil } - // we need to iterate through the index to make sure we put this post in the appropriate place according to when it was created + var insertMark *list.Element + // We need to iterate through the index to make sure we put this post in the appropriate place according to when it was created. + // We also need to make sure we're not inserting a duplicate post -- this can happen sometimes and it's not nice UX (*shudder*). for e := p.data.Front(); e != nil; e = e.Next() { entry, ok := e.Value.(*preparedPostsEntry) if !ok { @@ -37,12 +39,23 @@ func (p *preparedPosts) insertPrepared(i *preparedPostsEntry) error { } // if the post to index is newer than e, insert it before e in the list - if i.createdAt.After(entry.createdAt) { - p.data.InsertBefore(i, e) + if insertMark == nil { + if i.createdAt.After(entry.createdAt) { + insertMark = e + } + } + + // make sure we don't insert a duplicate + if entry.statusID == i.statusID { return nil } } + if insertMark != nil { + p.data.InsertBefore(i, insertMark) + return nil + } + // if we reach this point it's the oldest post we've seen so put it at the back p.data.PushBack(i) return nil diff --git a/internal/timeline/remove.go b/internal/timeline/remove.go new file mode 100644 index 0000000..2f340d3 --- /dev/null +++ b/internal/timeline/remove.go @@ -0,0 +1,50 @@ +package timeline + +import ( + "container/list" + "errors" +) + +func (t *timeline) Remove(statusID string) (int, error) { + t.Lock() + defer t.Unlock() + var removed int + + // remove entr(ies) from the post index + removeIndexes := []*list.Element{} + if t.postIndex != nil && t.postIndex.data != nil { + for e := t.postIndex.data.Front(); e != nil; e = e.Next() { + entry, ok := e.Value.(*postIndexEntry) + if !ok { + return removed, errors.New("Remove: could not parse e as a postIndexEntry") + } + if entry.statusID == statusID { + removeIndexes = append(removeIndexes, e) + } + } + } + for _, e := range removeIndexes { + t.postIndex.data.Remove(e) + removed = removed + 1 + } + + // remove entr(ies) from prepared posts + removePrepared := []*list.Element{} + if t.preparedPosts != nil && t.preparedPosts.data != nil { + for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() { + entry, ok := e.Value.(*preparedPostsEntry) + if !ok { + return removed, errors.New("Remove: could not parse e as a preparedPostsEntry") + } + if entry.statusID == statusID { + removePrepared = append(removePrepared, e) + } + } + } + for _, e := range removePrepared { + t.preparedPosts.data.Remove(e) + removed = removed + 1 + } + + return removed, nil +} diff --git a/internal/timeline/timeline.go b/internal/timeline/timeline.go index a662632..2f2aade 100644 --- a/internal/timeline/timeline.go +++ b/internal/timeline/timeline.go @@ -19,9 +19,7 @@ package timeline import ( - "container/list" "errors" - "fmt" "sync" "time" @@ -52,7 +50,7 @@ type Timeline interface { // This will NOT include the status with the given ID. // // This corresponds to an api call to /timelines/home?since_id=WHATEVER - GetXBeforeID(amount int, sinceID string) ([]*apimodel.Status, error) + GetXBeforeID(amount int, sinceID string, startFromTop bool) ([]*apimodel.Status, error) // GetXBetweenID returns x amount of posts from the given maxID, up to the given id, from newest to oldest. // This will NOT include the status with the given IDs. // @@ -80,9 +78,10 @@ type Timeline interface { */ // PrepareXFromTop instructs the timeline to prepare x amount of posts from the top of the timeline. - PrepareXFromTop(amount int) error - // PrepareXFromPosition instrucst the timeline to prepare the next amount of entries for serialization, from position onwards. - PrepareXFromPosition(amount int, position int) error + PrepareFromTop(amount int) error + // PrepareBehind instructs the timeline to prepare the next amount of entries for serialization, from position onwards. + // If include is true, then the given status ID will also be prepared, otherwise only entries behind it will be prepared. + PrepareBehind(statusID string, include bool, amount int) error // IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property, // and then immediately prepares it. IndexAndPrepareOne(statusCreatedAt time.Time, statusID string) error @@ -124,280 +123,6 @@ func NewTimeline(accountID string, db db.DB, typeConverter typeutils.TypeConvert } } -func (t *timeline) PrepareXFromPosition(amount int, desiredPosition int) error { - t.Lock() - defer t.Unlock() - - var position int - var prepared int - var preparing bool - for e := t.postIndex.data.Front(); e != nil; e = e.Next() { - entry, ok := e.Value.(*postIndexEntry) - if !ok { - return errors.New("PrepareXFromTop: could not parse e as a postIndexEntry") - } - - if !preparing { - // we haven't hit the position we need to prepare from yet - position = position + 1 - if position == desiredPosition { - preparing = true - continue - } - } else { - if err := t.prepare(entry.statusID); err != nil { - return fmt.Errorf("PrepareXFromTop: error preparing status with id %s: %s", entry.statusID, err) - } - prepared = prepared + 1 - if prepared >= amount { - // we're done - break - } - } - } - - return nil -} - -func (t *timeline) PrepareXFromTop(amount int) error { - t.Lock() - defer t.Unlock() - - t.preparedPosts.data.Init() - - var prepared int - for e := t.postIndex.data.Front(); e != nil; e = e.Next() { - entry, ok := e.Value.(*postIndexEntry) - if !ok { - return errors.New("PrepareXFromTop: could not parse e as a postIndexEntry") - } - - if err := t.prepare(entry.statusID); err != nil { - return fmt.Errorf("PrepareXFromTop: error preparing status with id %s: %s", entry.statusID, err) - } - - prepared = prepared + 1 - if prepared >= amount { - // we're done - break - } - } - - return nil -} - -func (t *timeline) GetXFromTop(amount int) ([]*apimodel.Status, error) { - // make a slice of statuses with the length we need to return - statuses := make([]*apimodel.Status, 0, amount) - - // if there are no prepared posts, just return the empty slice - if t.preparedPosts.data == nil { - t.preparedPosts.data = &list.List{} - } - - // make sure we have enough posts prepared to return - if t.preparedPosts.data.Len() < amount { - if err := t.PrepareXFromTop(amount); err != nil { - return nil, err - } - } - - // work through the prepared posts from the top and return - var served int - for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() { - entry, ok := e.Value.(*preparedPostsEntry) - if !ok { - return nil, errors.New("GetXFromTop: could not parse e as a preparedPostsEntry") - } - statuses = append(statuses, entry.prepared) - served = served + 1 - if served >= amount { - break - } - } - - return statuses, nil -} - -func (t *timeline) GetXBehindID(amount int, behindID string) ([]*apimodel.Status, error) { - // make a slice of statuses with the length we need to return - statuses := make([]*apimodel.Status, 0, amount) - - // if there are no prepared posts, just return the empty slice - if t.preparedPosts.data == nil { - t.preparedPosts.data = &list.List{} - } - - // find the position of id - var position int - for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() { - entry, ok := e.Value.(*preparedPostsEntry) - if !ok { - return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry") - } - if entry.statusID == behindID { - break - } - position = position + 1 - } - - // make sure we have enough posts prepared behind it to return what we're being asked for - if t.preparedPosts.data.Len() < amount+position { - if err := t.PrepareXFromPosition(amount, position); err != nil { - return nil, err - } - } - - // iterate through the modified list until we hit the fromID again - var serving bool - var served int - for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() { - entry, ok := e.Value.(*preparedPostsEntry) - if !ok { - return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry") - } - - if !serving { - // start serving if we've hit the id we're looking for - if entry.statusID == behindID { - serving = true - continue - } - } else { - // serve up to the amount requested - statuses = append(statuses, entry.prepared) - served = served + 1 - if served >= amount { - break - } - } - } - - return statuses, nil -} - -func (t *timeline) GetXBeforeID(amount int, beforeID string) ([]*apimodel.Status, error) { - // make a slice of statuses with the length we need to return - statuses := make([]*apimodel.Status, 0, amount) - - // if there are no prepared posts, just return the empty slice - if t.preparedPosts.data == nil { - t.preparedPosts.data = &list.List{} - } - - // iterate through the modified list until we hit the fromID again - var served int -serveloop: - for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() { - entry, ok := e.Value.(*preparedPostsEntry) - if !ok { - return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry") - } - - if entry.statusID == beforeID { - // we're good - break serveloop - } - - // serve up to the amount requested - statuses = append(statuses, entry.prepared) - served = served + 1 - if served >= amount { - break - } - } - - return statuses, nil -} - -func (t *timeline) GetXBetweenID(amount int, maxID string, sinceID string) ([]*apimodel.Status, error) { - // make a slice of statuses with the length we need to return - statuses := make([]*apimodel.Status, 0, amount) - - // if there are no prepared posts, just return the empty slice - if t.preparedPosts.data == nil { - t.preparedPosts.data = &list.List{} - } - - return statuses, nil -} - -func (t *timeline) IndexOne(statusCreatedAt time.Time, statusID string) error { - t.Lock() - defer t.Unlock() - - postIndexEntry := &postIndexEntry{ - createdAt: statusCreatedAt, - statusID: statusID, - } - - return t.postIndex.insertIndexed(postIndexEntry) -} - -func (t *timeline) IndexAndPrepareOne(statusCreatedAt time.Time, statusID string) error { - t.Lock() - defer t.Unlock() - - postIndexEntry := &postIndexEntry{ - createdAt: statusCreatedAt, - statusID: statusID, - } - - if err := t.postIndex.insertIndexed(postIndexEntry); err != nil { - return fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %s", err) - } - - if err := t.prepare(statusID); err != nil { - return fmt.Errorf("IndexAndPrepareOne: error preparing: %s", err) - } - - return nil -} - -func (t *timeline) Remove(statusID string) (int, error) { - t.Lock() - defer t.Unlock() - var removed int - - // remove entr(ies) from the post index - removeIndexes := []*list.Element{} - if t.postIndex != nil && t.postIndex.data != nil { - for e := t.postIndex.data.Front(); e != nil; e = e.Next() { - entry, ok := e.Value.(*postIndexEntry) - if !ok { - return removed, errors.New("Remove: could not parse e as a postIndexEntry") - } - if entry.statusID == statusID { - removeIndexes = append(removeIndexes, e) - } - } - } - for _, e := range removeIndexes { - t.postIndex.data.Remove(e) - removed = removed + 1 - } - - // remove entr(ies) from prepared posts - removePrepared := []*list.Element{} - if t.preparedPosts != nil && t.preparedPosts.data != nil { - for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() { - entry, ok := e.Value.(*preparedPostsEntry) - if !ok { - return removed, errors.New("Remove: could not parse e as a preparedPostsEntry") - } - if entry.statusID == statusID { - removePrepared = append(removePrepared, e) - } - } - } - for _, e := range removePrepared { - t.preparedPosts.data.Remove(e) - removed = removed + 1 - } - - return removed, nil -} - func (t *timeline) Reset() error { return nil } @@ -430,52 +155,3 @@ func (t *timeline) OldestIndexedPostID() (string, error) { } return entry.statusID, nil } - -func (t *timeline) prepare(statusID string) error { - - // start by getting the status out of the database according to its indexed ID - gtsStatus := >smodel.Status{} - if err := t.db.GetByID(statusID, gtsStatus); err != nil { - return err - } - - // if the account pointer hasn't been set on this timeline already, set it lazily here - if t.account == nil { - timelineOwnerAccount := >smodel.Account{} - if err := t.db.GetByID(t.accountID, timelineOwnerAccount); err != nil { - return err - } - t.account = timelineOwnerAccount - } - - // to convert the status we need relevant accounts from it, so pull them out here - relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(gtsStatus) - if err != nil { - return err - } - - // check if this is a boost... - var reblogOfStatus *gtsmodel.Status - if gtsStatus.BoostOfID != "" { - s := >smodel.Status{} - if err := t.db.GetByID(gtsStatus.BoostOfID, s); err != nil { - return err - } - reblogOfStatus = s - } - - // serialize the status (or, at least, convert it to a form that's ready to be serialized) - apiModelStatus, err := t.tc.StatusToMasto(gtsStatus, relevantAccounts.StatusAuthor, t.account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus) - if err != nil { - return err - } - - // shove it in prepared posts as a prepared posts entry - preparedPostsEntry := &preparedPostsEntry{ - createdAt: gtsStatus.CreatedAt, - statusID: statusID, - prepared: apiModelStatus, - } - - return t.preparedPosts.insertPrepared(preparedPostsEntry) -}