i can't even

internal/db/db.go

@@ -255,7 +255,8 @@ type DB interface {
 	// This slice will be unfiltered, not taking account of blocks and whatnot, so filter it before serving it back to a user.
 	WhoBoostedStatus(status *gtsmodel.Status) ([]*gtsmodel.Account, error)
 
-	GetStatusesWhereFollowing(accountID string, limit int, offsetStatusID string) ([]*gtsmodel.Status, error)
+	// GetStatusesWhereFollowing returns a slice of statuses from accounts that are followed by the given account id.
+	GetStatusesWhereFollowing(accountID string, limit int, maxID string, minID string, sinceID string) ([]*gtsmodel.Status, error)
 
 	// GetPublicTimelineForAccount fetches the account's PUBLIC timline -- ie., posts and replies that are public.
 	// It will use the given filters and try to return as many statuses as possible up to the limit.

internal/db/pg/pg.go

@@ -788,7 +788,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 
 	// if target account is suspended then don't show the status
 	if !targetAccount.SuspendedAt.IsZero() {
-		l.Debug("target account suspended at is not zero")
+		l.Trace("target account suspended at is not zero")
 		return false, nil
 	}
 
@@ -807,7 +807,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 		// if target user is disabled, not yet approved, or not confirmed then don't show the status
 		// (although in the latter two cases it's unlikely they posted a status yet anyway, but you never know!)
 		if targetUser.Disabled || !targetUser.Approved || targetUser.ConfirmedAt.IsZero() {
-			l.Debug("target user is disabled, not approved, or not confirmed")
+			l.Trace("target user is disabled, not approved, or not confirmed")
 			return false, nil
 		}
 	}
@@ -818,14 +818,14 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 		if targetStatus.Visibility == gtsmodel.VisibilityPublic {
 			return true, nil
 		}
-		l.Debug("requesting account is nil but the target status isn't public")
+		l.Trace("requesting account is nil but the target status isn't public")
 		return false, nil
 	}
 
 	// if requesting account is suspended then don't show the status -- although they probably shouldn't have gotten
 	// this far (ie., been authed) in the first place: this is just for safety.
 	if !requestingAccount.SuspendedAt.IsZero() {
-		l.Debug("requesting account is suspended")
+		l.Trace("requesting account is suspended")
 		return false, nil
 	}
 
@@ -843,7 +843,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 		}
 		// okay, user exists, so make sure it has full privileges/is confirmed/approved
 		if requestingUser.Disabled || !requestingUser.Approved || requestingUser.ConfirmedAt.IsZero() {
-			l.Debug("requesting account is local but corresponding user is either disabled, not approved, or not confirmed")
+			l.Trace("requesting account is local but corresponding user is either disabled, not approved, or not confirmed")
 			return false, nil
 		}
 	}
@@ -860,7 +860,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 		return false, err
 	} else if blocked {
 		// don't allow the status to be viewed if a block exists in *either* direction between these two accounts, no creepy stalking please
-		l.Debug("a block exists between requesting account and target account")
+		l.Trace("a block exists between requesting account and target account")
 		return false, nil
 	}
 
@@ -871,7 +871,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 			if blocked, err := ps.Blocked(relevantAccounts.ReplyToAccount.ID, requestingAccount.ID); err != nil {
 				return false, err
 			} else if blocked {
-				l.Debug("a block exists between requesting account and reply to account")
+				l.Trace("a block exists between requesting account and reply to account")
 				return false, nil
 			}
 
@@ -882,7 +882,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 					return false, err
 				}
 				if !followsRepliedAccount {
-					l.Debug("target status is a followers-only reply to an account that is not followed by the requesting account")
+					l.Trace("target status is a followers-only reply to an account that is not followed by the requesting account")
 					return false, nil
 				}
 			}
@@ -893,7 +893,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 			if blocked, err := ps.Blocked(relevantAccounts.BoostedAccount.ID, requestingAccount.ID); err != nil {
 				return false, err
 			} else if blocked {
-				l.Debug("a block exists between requesting account and boosted account")
+				l.Trace("a block exists between requesting account and boosted account")
 				return false, nil
 			}
 		}
@@ -903,7 +903,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 			if blocked, err := ps.Blocked(relevantAccounts.BoostedReplyToAccount.ID, requestingAccount.ID); err != nil {
 				return false, err
 			} else if blocked {
-				l.Debug("a block exists between requesting account and boosted reply to account")
+				l.Trace("a block exists between requesting account and boosted reply to account")
 				return false, nil
 			}
 		}
@@ -913,7 +913,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 			if blocked, err := ps.Blocked(a.ID, requestingAccount.ID); err != nil {
 				return false, err
 			} else if blocked {
-				l.Debug("a block exists between requesting account and a mentioned account")
+				l.Trace("a block exists between requesting account and a mentioned account")
 				return false, nil
 			}
 		}
@@ -939,7 +939,7 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 			return false, err
 		}
 		if !follows {
-			l.Debug("requested status is followers only but requesting account is not a follower")
+			l.Trace("requested status is followers only but requesting account is not a follower")
 			return false, nil
 		}
 		return true, nil
@@ -950,12 +950,12 @@ func (ps *postgresService) StatusVisible(targetStatus *gtsmodel.Status, requesti
 			return false, err
 		}
 		if !mutuals {
-			l.Debug("requested status is mutuals only but accounts aren't mufos")
+			l.Trace("requested status is mutuals only but accounts aren't mufos")
 			return false, nil
 		}
 		return true, nil
 	case gtsmodel.VisibilityDirect:
-		l.Debug("requesting account requests a status it's not mentioned in")
+		l.Trace("requesting account requests a status it's not mentioned in")
 		return false, nil // it's not mentioned -_-
 	}
 
@@ -1133,22 +1133,32 @@ func (ps *postgresService) WhoBoostedStatus(status *gtsmodel.Status) ([]*gtsmode
 	return accounts, nil
 }
 
-func (ps *postgresService) GetStatusesWhereFollowing(accountID string, limit int, offsetStatusID string) ([]*gtsmodel.Status, error) {
+func (ps *postgresService) GetStatusesWhereFollowing(accountID string, limit int, offsetStatusID string, includeOffsetStatus bool, ascending bool) ([]*gtsmodel.Status, error) {
 	statuses := []*gtsmodel.Status{}
 
 	q := ps.conn.Model(&statuses)
 
 	q = q.ColumnExpr("status.*").
 		Join("JOIN follows AS f ON f.target_account_id = status.account_id").
-		Where("f.account_id = ?", accountID).
+		Where("f.account_id = ?", accountID)
-		Order("status.created_at DESC")
 
+	if ascending {
+		q = q.Order("status.created_at")
+	} else {
+		q = q.Order("status.created_at DESC")
+	}
+
+	s := &gtsmodel.Status{}
 	if offsetStatusID != "" {
-		s := &gtsmodel.Status{}
 		if err := ps.conn.Model(s).Where("id = ?", offsetStatusID).Select(); err != nil {
 			return nil, err
 		}
-		q = q.Where("status.created_at < ?", s.CreatedAt)
+
+		if ascending {
+			q = q.Where("status.created_at > ?", s.CreatedAt)
+		} else {
+			q = q.Where("status.created_at < ?", s.CreatedAt)
+		}
 	}
 
 	if limit > 0 {
@@ -1162,6 +1172,14 @@ func (ps *postgresService) GetStatusesWhereFollowing(accountID string, limit int
 		}
 	}
 
+	if includeOffsetStatus {
+		if ascending {
+			statuses = append([]*gtsmodel.Status{s}, statuses...)
+		} else {
+			statuses = append(statuses, s)
+		}
+	}
+
 	return statuses, nil
 }
 

internal/processing/timeline.go

@@ -32,9 +32,53 @@ import (
 
 func (p *processor) HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, gtserror.WithCode) {
 
-	statuses, err := p.timelineManager.HomeTimeline(authed.Account.ID, maxID, sinceID, minID, limit, local)
+	statuses := []*apimodel.Status{}
-	if err != nil {
+
-		return nil, gtserror.NewErrorInternalError(err)
+grabloop:
+	for len(statuses) < limit {
+		gtsStatuses, err := p.db.GetStatusesWhereFollowing(authed.Account.ID, limit, maxID, false, false)
+
+		if err != nil {
+			if _, ok := err.(db.ErrNoEntries); !ok {
+				return nil, gtserror.NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error getting statuses from db: %s", err))
+			}
+			break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
+		}
+
+		for _, s := range gtsStatuses {
+			relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(s)
+			if err != nil {
+				continue
+			}
+			visible, err := p.db.StatusVisible(s, authed.Account, relevantAccounts)
+			if err != nil {
+				continue
+			}
+
+			if visible {
+				// check if this is a boost...
+				var reblogOfStatus *gtsmodel.Status
+				if s.BoostOfID != "" {
+					s := &gtsmodel.Status{}
+					if err := p.db.GetByID(s.BoostOfID, s); err != nil {
+						continue
+					}
+					reblogOfStatus = s
+				}
+
+				// serialize the status (or, at least, convert it to a form that's ready to be serialized)
+				apiModelStatus, err := p.tc.StatusToMasto(s, relevantAccounts.StatusAuthor, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus)
+				if err != nil {
+					continue
+				}
+
+				statuses = append(statuses, apiModelStatus)
+				if len(statuses) == limit {
+					// we have enough
+					break grabloop
+				}
+			}
+		}
 	}
 
 	return statuses, nil
@@ -163,7 +207,7 @@ func (p *processor) initTimelineFor(account *gtsmodel.Account, wg *sync.WaitGrou
 
 	desiredIndexLength := p.timelineManager.GetDesiredIndexLength()
 
-	statuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength, "")
+	statuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength, "", false, false)
 	if err != nil {
 		l.Error(fmt.Errorf("initTimelineFor: error getting statuses: %s", err))
 		return
@@ -180,7 +224,7 @@ func (p *processor) initTimelineFor(account *gtsmodel.Account, wg *sync.WaitGrou
 		}
 
 		if rearmostStatusID != "" {
-			moreStatuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength/2, rearmostStatusID)
+			moreStatuses, err := p.db.GetStatusesWhereFollowing(account.ID, desiredIndexLength/2, rearmostStatusID, false, false)
 			if err != nil {
 				l.Error(fmt.Errorf("initTimelineFor: error getting more statuses: %s", err))
 				return

internal/timeline/get.go

@@ -0,0 +1,250 @@
+package timeline
+
+import (
+	"container/list"
+	"errors"
+	"fmt"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+)
+
+func (t *timeline) GetXFromTop(amount int) ([]*apimodel.Status, error) {
+	// make a slice of statuses with the length we need to return
+	statuses := make([]*apimodel.Status, 0, amount)
+
+	// if there are no prepared posts, just return the empty slice
+	if t.preparedPosts.data == nil {
+		t.preparedPosts.data = &list.List{}
+	}
+
+	// make sure we have enough posts prepared to return
+	if t.preparedPosts.data.Len() < amount {
+		if err := t.PrepareFromTop(amount); err != nil {
+			return nil, err
+		}
+	}
+
+	// work through the prepared posts from the top and return
+	var served int
+	for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+		entry, ok := e.Value.(*preparedPostsEntry)
+		if !ok {
+			return nil, errors.New("GetXFromTop: could not parse e as a preparedPostsEntry")
+		}
+		statuses = append(statuses, entry.prepared)
+		served = served + 1
+		if served >= amount {
+			break
+		}
+	}
+
+	return statuses, nil
+}
+
+func (t *timeline) GetXBehindID(amount int, behindID string) ([]*apimodel.Status, error) {
+	// make a slice of statuses with the length we need to return
+	statuses := make([]*apimodel.Status, 0, amount)
+
+	// if there are no prepared posts, just return the empty slice
+	if t.preparedPosts.data == nil {
+		t.preparedPosts.data = &list.List{}
+	}
+
+	// iterate through the modified list until we hit the mark we're looking for
+	var position int
+	var behindIDMark *list.Element
+findMarkLoop:
+	for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+		position = position + 1
+		entry, ok := e.Value.(*preparedPostsEntry)
+		if !ok {
+			return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
+		}
+
+		if entry.statusID == behindID {
+			fmt.Printf("\n\n\n GETXBEHINDID: FOUND BEHINDID %s WITH POSITION %d AND CREATEDAT %s \n\n\n", behindID, position, entry.createdAt.String())
+			behindIDMark = e
+			break findMarkLoop
+		}
+	}
+
+	// we didn't find it, so we need to make sure it's indexed and prepared and then try again
+	if behindIDMark == nil {
+		if err := t.IndexBehind(behindID, true, amount); err != nil {
+			return nil, fmt.Errorf("GetXBehindID: error indexing behind and including ID %s", behindID)
+		}
+		if err := t.PrepareBehind(behindID, true, amount); err != nil {
+			return nil, fmt.Errorf("GetXBehindID: error preparing behind and including ID %s", behindID)
+		}
+		return t.GetXBehindID(amount, behindID)
+	}
+
+	// make sure we have enough posts prepared behind it to return what we're being asked for
+	if t.preparedPosts.data.Len() < amount+position {
+		fmt.Printf("\n\n\n GETXBEHINDID: PREPARED POSTS LENGTH %d WAS LESS THAN AMOUNT %d PLUS POSITION %d", t.preparedPosts.data.Len(), amount, position)
+		if err := t.PrepareBehind(behindID, false, amount); err != nil {
+			return nil, err
+		}
+		fmt.Printf("\n\n\n GETXBEHINDID: PREPARED POSTS LENGTH IS NOW %d", t.preparedPosts.data.Len())
+	}
+
+	// start serving from the entry right after the mark
+	var served int
+serveloop:
+	for e := behindIDMark.Next(); e != nil; e = e.Next() {
+		entry, ok := e.Value.(*preparedPostsEntry)
+		if !ok {
+			return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
+		}
+
+		fmt.Printf("\n\n\n GETXBEHINDID: SERVING STATUS ID %s WITH CREATEDAT %s \n\n\n", entry.statusID, entry.createdAt.String())
+		// serve up to the amount requested
+		statuses = append(statuses, entry.prepared)
+		served = served + 1
+		if served >= amount {
+			break serveloop
+		}
+	}
+
+	return statuses, nil
+}
+
+func (t *timeline) GetXBeforeID(amount int, beforeID string, startFromTop bool) ([]*apimodel.Status, error) {
+	// make a slice of statuses with the length we need to return
+	statuses := make([]*apimodel.Status, 0, amount)
+
+	if t.preparedPosts.data == nil {
+		t.preparedPosts.data = &list.List{}
+	}
+
+	// iterate through the modified list until we hit the mark we're looking for
+	var beforeIDMark *list.Element
+findMarkLoop:
+	for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+		entry, ok := e.Value.(*preparedPostsEntry)
+		if !ok {
+			return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
+		}
+
+		if entry.statusID == beforeID {
+			beforeIDMark = e
+			break findMarkLoop
+		}
+	}
+
+	// we didn't find it, so we need to make sure it's indexed and prepared and then try again
+	if beforeIDMark == nil {
+		if err := t.IndexBefore(beforeID, true, amount); err != nil {
+			return nil, fmt.Errorf("GetXBeforeID: error indexing before and including ID %s", beforeID)
+		}
+		if err := t.PrepareBefore(beforeID, true, amount); err != nil {
+			return nil, fmt.Errorf("GetXBeforeID: error preparing before and including ID %s", beforeID)
+		}
+		return t.GetXBeforeID(amount, beforeID, startFromTop)
+	}
+
+	var served int
+
+	if startFromTop {
+		// start serving from the front/top and keep going until we hit mark or get x amount statuses
+	serveloopFromTop:
+		for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+			entry, ok := e.Value.(*preparedPostsEntry)
+			if !ok {
+				return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
+			}
+
+			if entry.statusID == beforeID {
+				break serveloopFromTop
+			}
+
+			// serve up to the amount requested
+			statuses = append(statuses, entry.prepared)
+			served = served + 1
+			if served >= amount {
+				break serveloopFromTop
+			}
+		}
+
+	} else if startFromTop {
+		// start serving from the entry right before the mark
+	serveloopFromBottom:
+		for e := beforeIDMark.Prev(); e != nil; e = e.Prev() {
+			entry, ok := e.Value.(*preparedPostsEntry)
+			if !ok {
+				return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
+			}
+
+			// serve up to the amount requested
+			statuses = append(statuses, entry.prepared)
+			served = served + 1
+			if served >= amount {
+				break serveloopFromBottom
+			}
+		}
+	}
+
+	return statuses, nil
+}
+
+func (t *timeline) GetXBetweenID(amount int, behindID string, beforeID string) ([]*apimodel.Status, error) {
+	// make a slice of statuses with the length we need to return
+	statuses := make([]*apimodel.Status, 0, amount)
+
+	// if there are no prepared posts, just return the empty slice
+	if t.preparedPosts.data == nil {
+		t.preparedPosts.data = &list.List{}
+	}
+
+	// iterate through the modified list until we hit the mark we're looking for
+	var position int
+	var behindIDMark *list.Element
+findMarkLoop:
+	for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+		position = position + 1
+		entry, ok := e.Value.(*preparedPostsEntry)
+		if !ok {
+			return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
+		}
+
+		if entry.statusID == behindID {
+			behindIDMark = e
+			break findMarkLoop
+		}
+	}
+
+	// we didn't find it
+	if behindIDMark == nil {
+		return nil, fmt.Errorf("GetXBetweenID: couldn't find status with ID %s", behindID)
+	}
+
+	// make sure we have enough posts prepared behind it to return what we're being asked for
+	if t.preparedPosts.data.Len() < amount+position {
+		if err := t.PrepareBehind(behindID, false, amount); err != nil {
+			return nil, err
+		}
+	}
+
+	// start serving from the entry right after the mark
+	var served int
+serveloop:
+	for e := behindIDMark.Next(); e != nil; e = e.Next() {
+		entry, ok := e.Value.(*preparedPostsEntry)
+		if !ok {
+			return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
+		}
+
+		if entry.statusID == beforeID {
+			break serveloop
+		}
+
+		// serve up to the amount requested
+		statuses = append(statuses, entry.prepared)
+		served = served + 1
+		if served >= amount {
+			break serveloop
+		}
+	}
+
+	return statuses, nil
+}

internal/timeline/index.go

@@ -0,0 +1,120 @@
+package timeline
+
+import (
+	"fmt"
+	"time"
+)
+
+func (t *timeline) IndexBefore(statusID string, include bool, amount int) error {
+	// 	filtered := []*gtsmodel.Status{}
+	// 	offsetStatus := statusID
+
+	// grabloop:
+	// 	for len(filtered) < amount {
+	// 		statuses, err := t.db.GetStatusesWhereFollowing(t.accountID, amount, offsetStatus, include, true)
+	// 		if err != nil {
+	// 			if _, ok := err.(db.ErrNoEntries); !ok {
+	// 				return fmt.Errorf("IndexBeforeAndIncluding: error getting statuses from db: %s", err)
+	// 			}
+	// 			break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
+	// 		}
+
+	// 		for _, s := range statuses {
+	// 			relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(s)
+	// 			if err != nil {
+	// 				continue
+	// 			}
+	// 			visible, err := t.db.StatusVisible(s, t.account, relevantAccounts)
+	// 			if err != nil {
+	// 				continue
+	// 			}
+	// 			if visible {
+	// 				filtered = append(filtered, s)
+	// 			}
+	// 			offsetStatus = s.ID
+	// 		}
+	// 	}
+
+	// 	for _, s := range filtered {
+	// 		if err := t.IndexOne(s.CreatedAt, s.ID); err != nil {
+	// 			return fmt.Errorf("IndexBeforeAndIncluding: error indexing status with id %s: %s", s.ID, err)
+	// 		}
+	// 	}
+
+	return nil
+}
+
+func (t *timeline) IndexBehind(statusID string, include bool, amount int) error {
+	// 	filtered := []*gtsmodel.Status{}
+	// 	offsetStatus := statusID
+
+	// grabloop:
+	// 	for len(filtered) < amount {
+	// 		statuses, err := t.db.GetStatusesWhereFollowing(t.accountID, amount, offsetStatus, include, false)
+	// 		if err != nil {
+	// 			if _, ok := err.(db.ErrNoEntries); !ok {
+	// 				return fmt.Errorf("IndexBehindAndIncluding: error getting statuses from db: %s", err)
+	// 			}
+	// 			break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
+	// 		}
+
+	// 		for _, s := range statuses {
+	// 			relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(s)
+	// 			if err != nil {
+	// 				continue
+	// 			}
+	// 			visible, err := t.db.StatusVisible(s, t.account, relevantAccounts)
+	// 			if err != nil {
+	// 				continue
+	// 			}
+	// 			if visible {
+	// 				filtered = append(filtered, s)
+	// 			}
+	// 			offsetStatus = s.ID
+	// 		}
+	// 	}
+
+	// 	for _, s := range filtered {
+	// 		if err := t.IndexOne(s.CreatedAt, s.ID); err != nil {
+	// 			return fmt.Errorf("IndexBehindAndIncluding: error indexing status with id %s: %s", s.ID, err)
+	// 		}
+	// 	}
+
+	return nil
+}
+
+func (t *timeline) IndexOneByID(statusID string) error {
+	return nil
+}
+
+func (t *timeline) IndexOne(statusCreatedAt time.Time, statusID string) error {
+	t.Lock()
+	defer t.Unlock()
+
+	postIndexEntry := &postIndexEntry{
+		createdAt: statusCreatedAt,
+		statusID:  statusID,
+	}
+
+	return t.postIndex.insertIndexed(postIndexEntry)
+}
+
+func (t *timeline) IndexAndPrepareOne(statusCreatedAt time.Time, statusID string) error {
+	t.Lock()
+	defer t.Unlock()
+
+	postIndexEntry := &postIndexEntry{
+		createdAt: statusCreatedAt,
+		statusID:  statusID,
+	}
+
+	if err := t.postIndex.insertIndexed(postIndexEntry); err != nil {
+		return fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %s", err)
+	}
+
+	if err := t.prepare(statusID); err != nil {
+		return fmt.Errorf("IndexAndPrepareOne: error preparing: %s", err)
+	}
+
+	return nil
+}

internal/timeline/manager.go

@@ -150,7 +150,7 @@ func (m *manager) HomeTimeline(timelineAccountID string, maxID string, sinceID s
 	} else if maxID != "" {
 		statuses, err = t.GetXBehindID(limit, maxID)
 	} else if sinceID != "" {
-		statuses, err = t.GetXBeforeID(limit, sinceID)
+		statuses, err = t.GetXBeforeID(limit, sinceID, true)
 	} else {
 		statuses, err = t.GetXFromTop(limit)
 	}
@@ -180,7 +180,7 @@ func (m *manager) GetOldestIndexedID(timelineAccountID string) (string, error) {
 func (m *manager) PrepareXFromTop(timelineAccountID string, limit int) error {
 	t := m.getOrCreateTimeline(timelineAccountID)
 
-	return t.PrepareXFromTop(limit)
+	return t.PrepareFromTop(limit)
 }
 
 func (m *manager) WipeStatusFromTimeline(timelineAccountID string, statusID string) (int, error) {

internal/timeline/postindex.go

@@ -16,7 +16,6 @@ type postIndexEntry struct {
 }
 
 func (p *postIndex) insertIndexed(i *postIndexEntry) error {
-
 	if p.data == nil {
 		p.data = &list.List{}
 	}
@@ -27,20 +26,33 @@ func (p *postIndex) insertIndexed(i *postIndexEntry) error {
 		return nil
 	}
 
-	// we need to iterate through the index to make sure we put this post in the appropriate place according to when it was created
+	var insertMark *list.Element
+	// We need to iterate through the index to make sure we put this post in the appropriate place according to when it was created.
+	// We also need to make sure we're not inserting a duplicate post -- this can happen sometimes and it's not nice UX (*shudder*).
 	for e := p.data.Front(); e != nil; e = e.Next() {
 		entry, ok := e.Value.(*postIndexEntry)
 		if !ok {
-			return errors.New("Remove: could not parse e as a postIndexEntry")
+			return errors.New("index: could not parse e as a postIndexEntry")
 		}
 
 		// if the post to index is newer than e, insert it before e in the list
-		if i.createdAt.After(entry.createdAt) {
+		if insertMark == nil {
-			p.data.InsertBefore(i, e)
+			if i.createdAt.After(entry.createdAt) {
+				insertMark = e
+			}
+		}
+
+		// make sure we don't insert a duplicate
+		if entry.statusID == i.statusID {
 			return nil
 		}
 	}
 
+	if insertMark != nil {
+		p.data.InsertBefore(i, insertMark)
+		return nil
+	}
+
 	// if we reach this point it's the oldest post we've seen so put it at the back
 	p.data.PushBack(i)
 	return nil

internal/timeline/prepare.go

@@ -0,0 +1,180 @@
+package timeline
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (t *timeline) PrepareBehind(statusID string, include bool, amount int) error {
+	t.Lock()
+	defer t.Unlock()
+
+	var prepared int
+	var preparing bool
+prepareloop:
+	for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
+		entry, ok := e.Value.(*postIndexEntry)
+		if !ok {
+			return errors.New("PrepareBehind: could not parse e as a postIndexEntry")
+		}
+
+		if !preparing {
+			// we haven't hit the position we need to prepare from yet
+			if entry.statusID == statusID {
+				preparing = true
+				if !include {
+					continue
+				}
+			}
+		}
+
+		if preparing {
+			if err := t.prepare(entry.statusID); err != nil {
+				// there's been an error
+				if _, ok := err.(db.ErrNoEntries); !ok {
+					// it's a real error
+					return fmt.Errorf("PrepareBehind: error preparing status with id %s: %s", entry.statusID, err)
+				}
+				// the status just doesn't exist (anymore) so continue to the next one
+				continue
+			}
+			if prepared == amount {
+				// we're done
+				break prepareloop
+			}
+			prepared = prepared + 1
+		}
+	}
+
+	return nil
+}
+
+func (t *timeline) PrepareBefore(statusID string, include bool, amount int) error {
+	t.Lock()
+	defer t.Unlock()
+
+	var prepared int
+	var preparing bool
+prepareloop:
+	for e := t.postIndex.data.Back(); e != nil; e = e.Prev() {
+		entry, ok := e.Value.(*postIndexEntry)
+		if !ok {
+			return errors.New("PrepareBefore: could not parse e as a postIndexEntry")
+		}
+
+		if !preparing {
+			// we haven't hit the position we need to prepare from yet
+			if entry.statusID == statusID {
+				preparing = true
+				if !include {
+					continue
+				}
+			}
+		}
+
+		if preparing {
+			if err := t.prepare(entry.statusID); err != nil {
+				// there's been an error
+				if _, ok := err.(db.ErrNoEntries); !ok {
+					// it's a real error
+					return fmt.Errorf("PrepareBefore: error preparing status with id %s: %s", entry.statusID, err)
+				}
+				// the status just doesn't exist (anymore) so continue to the next one
+				continue
+			}
+			if prepared == amount {
+				// we're done
+				break prepareloop
+			}
+			prepared = prepared + 1
+		}
+	}
+
+	return nil
+}
+
+func (t *timeline) PrepareFromTop(amount int) error {
+	t.Lock()
+	defer t.Unlock()
+
+	t.preparedPosts.data.Init()
+
+	var prepared int
+prepareloop:
+	for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
+		entry, ok := e.Value.(*postIndexEntry)
+		if !ok {
+			return errors.New("PrepareFromTop: could not parse e as a postIndexEntry")
+		}
+
+		if err := t.prepare(entry.statusID); err != nil {
+			// there's been an error
+			if _, ok := err.(db.ErrNoEntries); !ok {
+				// it's a real error
+				return fmt.Errorf("PrepareFromTop: error preparing status with id %s: %s", entry.statusID, err)
+			}
+			// the status just doesn't exist (anymore) so continue to the next one
+			continue
+		}
+
+		prepared = prepared + 1
+		if prepared == amount {
+			// we're done
+			break prepareloop
+		}
+	}
+
+	return nil
+}
+
+func (t *timeline) prepare(statusID string) error {
+
+	// start by getting the status out of the database according to its indexed ID
+	gtsStatus := &gtsmodel.Status{}
+	if err := t.db.GetByID(statusID, gtsStatus); err != nil {
+		return err
+	}
+
+	// if the account pointer hasn't been set on this timeline already, set it lazily here
+	if t.account == nil {
+		timelineOwnerAccount := &gtsmodel.Account{}
+		if err := t.db.GetByID(t.accountID, timelineOwnerAccount); err != nil {
+			return err
+		}
+		t.account = timelineOwnerAccount
+	}
+
+	// to convert the status we need relevant accounts from it, so pull them out here
+	relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(gtsStatus)
+	if err != nil {
+		return err
+	}
+
+	// check if this is a boost...
+	var reblogOfStatus *gtsmodel.Status
+	if gtsStatus.BoostOfID != "" {
+		s := &gtsmodel.Status{}
+		if err := t.db.GetByID(gtsStatus.BoostOfID, s); err != nil {
+			return err
+		}
+		reblogOfStatus = s
+	}
+
+	// serialize the status (or, at least, convert it to a form that's ready to be serialized)
+	apiModelStatus, err := t.tc.StatusToMasto(gtsStatus, relevantAccounts.StatusAuthor, t.account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus)
+	if err != nil {
+		return err
+	}
+
+	// shove it in prepared posts as a prepared posts entry
+	preparedPostsEntry := &preparedPostsEntry{
+		createdAt: gtsStatus.CreatedAt,
+		statusID:  statusID,
+		prepared:  apiModelStatus,
+	}
+
+	return t.preparedPosts.insertPrepared(preparedPostsEntry)
+}

internal/timeline/preparedposts.go

@@ -29,7 +29,9 @@ func (p *preparedPosts) insertPrepared(i *preparedPostsEntry) error {
 		return nil
 	}
 
-	// we need to iterate through the index to make sure we put this post in the appropriate place according to when it was created
+	var insertMark *list.Element
+	// We need to iterate through the index to make sure we put this post in the appropriate place according to when it was created.
+	// We also need to make sure we're not inserting a duplicate post -- this can happen sometimes and it's not nice UX (*shudder*).
 	for e := p.data.Front(); e != nil; e = e.Next() {
 		entry, ok := e.Value.(*preparedPostsEntry)
 		if !ok {
@@ -37,12 +39,23 @@ func (p *preparedPosts) insertPrepared(i *preparedPostsEntry) error {
 		}
 
 		// if the post to index is newer than e, insert it before e in the list
-		if i.createdAt.After(entry.createdAt) {
+		if insertMark == nil {
-			p.data.InsertBefore(i, e)
+			if i.createdAt.After(entry.createdAt) {
+				insertMark = e
+			}
+		}
+
+		// make sure we don't insert a duplicate
+		if entry.statusID == i.statusID {
 			return nil
 		}
 	}
 
+	if insertMark != nil {
+		p.data.InsertBefore(i, insertMark)
+		return nil
+	}
+
 	// if we reach this point it's the oldest post we've seen so put it at the back
 	p.data.PushBack(i)
 	return nil

internal/timeline/remove.go

@@ -0,0 +1,50 @@
+package timeline
+
+import (
+	"container/list"
+	"errors"
+)
+
+func (t *timeline) Remove(statusID string) (int, error) {
+	t.Lock()
+	defer t.Unlock()
+	var removed int
+
+	// remove entr(ies) from the post index
+	removeIndexes := []*list.Element{}
+	if t.postIndex != nil && t.postIndex.data != nil {
+		for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
+			entry, ok := e.Value.(*postIndexEntry)
+			if !ok {
+				return removed, errors.New("Remove: could not parse e as a postIndexEntry")
+			}
+			if entry.statusID == statusID {
+				removeIndexes = append(removeIndexes, e)
+			}
+		}
+	}
+	for _, e := range removeIndexes {
+		t.postIndex.data.Remove(e)
+		removed = removed + 1
+	}
+
+	// remove entr(ies) from prepared posts
+	removePrepared := []*list.Element{}
+	if t.preparedPosts != nil && t.preparedPosts.data != nil {
+		for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
+			entry, ok := e.Value.(*preparedPostsEntry)
+			if !ok {
+				return removed, errors.New("Remove: could not parse e as a preparedPostsEntry")
+			}
+			if entry.statusID == statusID {
+				removePrepared = append(removePrepared, e)
+			}
+		}
+	}
+	for _, e := range removePrepared {
+		t.preparedPosts.data.Remove(e)
+		removed = removed + 1
+	}
+
+	return removed, nil
+}

internal/timeline/timeline.go

@@ -19,9 +19,7 @@
 package timeline
 
 import (
-	"container/list"
 	"errors"
-	"fmt"
 	"sync"
 	"time"
 
@@ -52,7 +50,7 @@ type Timeline interface {
 	// This will NOT include the status with the given ID.
 	//
 	// This corresponds to an api call to /timelines/home?since_id=WHATEVER
-	GetXBeforeID(amount int, sinceID string) ([]*apimodel.Status, error)
+	GetXBeforeID(amount int, sinceID string, startFromTop bool) ([]*apimodel.Status, error)
 	// GetXBetweenID returns x amount of posts from the given maxID, up to the given id, from newest to oldest.
 	// This will NOT include the status with the given IDs.
 	//
@@ -80,9 +78,10 @@ type Timeline interface {
 	*/
 
 	// PrepareXFromTop instructs the timeline to prepare x amount of posts from the top of the timeline.
-	PrepareXFromTop(amount int) error
+	PrepareFromTop(amount int) error
-	// PrepareXFromPosition instrucst the timeline to prepare the next amount of entries for serialization, from position onwards.
+	// PrepareBehind instructs the timeline to prepare the next amount of entries for serialization, from position onwards.
-	PrepareXFromPosition(amount int, position int) error
+	// If include is true, then the given status ID will also be prepared, otherwise only entries behind it will be prepared.
+	PrepareBehind(statusID string, include bool, amount int) error
 	// IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property,
 	// and then immediately prepares it.
 	IndexAndPrepareOne(statusCreatedAt time.Time, statusID string) error
@@ -124,280 +123,6 @@ func NewTimeline(accountID string, db db.DB, typeConverter typeutils.TypeConvert
 	}
 }
 
-func (t *timeline) PrepareXFromPosition(amount int, desiredPosition int) error {
-	t.Lock()
-	defer t.Unlock()
-
-	var position int
-	var prepared int
-	var preparing bool
-	for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
-		entry, ok := e.Value.(*postIndexEntry)
-		if !ok {
-			return errors.New("PrepareXFromTop: could not parse e as a postIndexEntry")
-		}
-
-		if !preparing {
-			// we haven't hit the position we need to prepare from yet
-			position = position + 1
-			if position == desiredPosition {
-				preparing = true
-				continue
-			}
-		} else {
-			if err := t.prepare(entry.statusID); err != nil {
-				return fmt.Errorf("PrepareXFromTop: error preparing status with id %s: %s", entry.statusID, err)
-			}
-			prepared = prepared + 1
-			if prepared >= amount {
-				// we're done
-				break
-			}
-		}
-	}
-
-	return nil
-}
-
-func (t *timeline) PrepareXFromTop(amount int) error {
-	t.Lock()
-	defer t.Unlock()
-
-	t.preparedPosts.data.Init()
-
-	var prepared int
-	for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
-		entry, ok := e.Value.(*postIndexEntry)
-		if !ok {
-			return errors.New("PrepareXFromTop: could not parse e as a postIndexEntry")
-		}
-
-		if err := t.prepare(entry.statusID); err != nil {
-			return fmt.Errorf("PrepareXFromTop: error preparing status with id %s: %s", entry.statusID, err)
-		}
-
-		prepared = prepared + 1
-		if prepared >= amount {
-			// we're done
-			break
-		}
-	}
-
-	return nil
-}
-
-func (t *timeline) GetXFromTop(amount int) ([]*apimodel.Status, error) {
-	// make a slice of statuses with the length we need to return
-	statuses := make([]*apimodel.Status, 0, amount)
-
-	// if there are no prepared posts, just return the empty slice
-	if t.preparedPosts.data == nil {
-		t.preparedPosts.data = &list.List{}
-	}
-
-	// make sure we have enough posts prepared to return
-	if t.preparedPosts.data.Len() < amount {
-		if err := t.PrepareXFromTop(amount); err != nil {
-			return nil, err
-		}
-	}
-
-	// work through the prepared posts from the top and return
-	var served int
-	for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
-		entry, ok := e.Value.(*preparedPostsEntry)
-		if !ok {
-			return nil, errors.New("GetXFromTop: could not parse e as a preparedPostsEntry")
-		}
-		statuses = append(statuses, entry.prepared)
-		served = served + 1
-		if served >= amount {
-			break
-		}
-	}
-
-	return statuses, nil
-}
-
-func (t *timeline) GetXBehindID(amount int, behindID string) ([]*apimodel.Status, error) {
-	// make a slice of statuses with the length we need to return
-	statuses := make([]*apimodel.Status, 0, amount)
-
-	// if there are no prepared posts, just return the empty slice
-	if t.preparedPosts.data == nil {
-		t.preparedPosts.data = &list.List{}
-	}
-
-	// find the position of id
-	var position int
-	for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
-		entry, ok := e.Value.(*preparedPostsEntry)
-		if !ok {
-			return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
-		}
-		if entry.statusID == behindID {
-			break
-		}
-		position = position + 1
-	}
-
-	// make sure we have enough posts prepared behind it to return what we're being asked for
-	if t.preparedPosts.data.Len() < amount+position {
-		if err := t.PrepareXFromPosition(amount, position); err != nil {
-			return nil, err
-		}
-	}
-
-	// iterate through the modified list until we hit the fromID again
-	var serving bool
-	var served int
-	for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
-		entry, ok := e.Value.(*preparedPostsEntry)
-		if !ok {
-			return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
-		}
-
-		if !serving {
-			// start serving if we've hit the id we're looking for
-			if entry.statusID == behindID {
-				serving = true
-				continue
-			}
-		} else {
-			// serve up to the amount requested
-			statuses = append(statuses, entry.prepared)
-			served = served + 1
-			if served >= amount {
-				break
-			}
-		}
-	}
-
-	return statuses, nil
-}
-
-func (t *timeline) GetXBeforeID(amount int, beforeID string) ([]*apimodel.Status, error) {
-	// make a slice of statuses with the length we need to return
-	statuses := make([]*apimodel.Status, 0, amount)
-
-	// if there are no prepared posts, just return the empty slice
-	if t.preparedPosts.data == nil {
-		t.preparedPosts.data = &list.List{}
-	}
-
-	// iterate through the modified list until we hit the fromID again
-	var served int
-serveloop:
-	for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
-		entry, ok := e.Value.(*preparedPostsEntry)
-		if !ok {
-			return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
-		}
-
-		if entry.statusID == beforeID {
-			// we're good
-			break serveloop
-		}
-
-		// serve up to the amount requested
-		statuses = append(statuses, entry.prepared)
-		served = served + 1
-		if served >= amount {
-			break
-		}
-	}
-
-	return statuses, nil
-}
-
-func (t *timeline) GetXBetweenID(amount int, maxID string, sinceID string) ([]*apimodel.Status, error) {
-	// make a slice of statuses with the length we need to return
-	statuses := make([]*apimodel.Status, 0, amount)
-
-	// if there are no prepared posts, just return the empty slice
-	if t.preparedPosts.data == nil {
-		t.preparedPosts.data = &list.List{}
-	}
-
-	return statuses, nil
-}
-
-func (t *timeline) IndexOne(statusCreatedAt time.Time, statusID string) error {
-	t.Lock()
-	defer t.Unlock()
-
-	postIndexEntry := &postIndexEntry{
-		createdAt: statusCreatedAt,
-		statusID:  statusID,
-	}
-
-	return t.postIndex.insertIndexed(postIndexEntry)
-}
-
-func (t *timeline) IndexAndPrepareOne(statusCreatedAt time.Time, statusID string) error {
-	t.Lock()
-	defer t.Unlock()
-
-	postIndexEntry := &postIndexEntry{
-		createdAt: statusCreatedAt,
-		statusID:  statusID,
-	}
-
-	if err := t.postIndex.insertIndexed(postIndexEntry); err != nil {
-		return fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %s", err)
-	}
-
-	if err := t.prepare(statusID); err != nil {
-		return fmt.Errorf("IndexAndPrepareOne: error preparing: %s", err)
-	}
-
-	return nil
-}
-
-func (t *timeline) Remove(statusID string) (int, error) {
-	t.Lock()
-	defer t.Unlock()
-	var removed int
-
-	// remove entr(ies) from the post index
-	removeIndexes := []*list.Element{}
-	if t.postIndex != nil && t.postIndex.data != nil {
-		for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
-			entry, ok := e.Value.(*postIndexEntry)
-			if !ok {
-				return removed, errors.New("Remove: could not parse e as a postIndexEntry")
-			}
-			if entry.statusID == statusID {
-				removeIndexes = append(removeIndexes, e)
-			}
-		}
-	}
-	for _, e := range removeIndexes {
-		t.postIndex.data.Remove(e)
-		removed = removed + 1
-	}
-
-	// remove entr(ies) from prepared posts
-	removePrepared := []*list.Element{}
-	if t.preparedPosts != nil && t.preparedPosts.data != nil {
-		for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
-			entry, ok := e.Value.(*preparedPostsEntry)
-			if !ok {
-				return removed, errors.New("Remove: could not parse e as a preparedPostsEntry")
-			}
-			if entry.statusID == statusID {
-				removePrepared = append(removePrepared, e)
-			}
-		}
-	}
-	for _, e := range removePrepared {
-		t.preparedPosts.data.Remove(e)
-		removed = removed + 1
-	}
-
-	return removed, nil
-}
-
 func (t *timeline) Reset() error {
 	return nil
 }
@@ -430,52 +155,3 @@ func (t *timeline) OldestIndexedPostID() (string, error) {
 	}
 	return entry.statusID, nil
 }
-
-func (t *timeline) prepare(statusID string) error {
-
-	// start by getting the status out of the database according to its indexed ID
-	gtsStatus := &gtsmodel.Status{}
-	if err := t.db.GetByID(statusID, gtsStatus); err != nil {
-		return err
-	}
-
-	// if the account pointer hasn't been set on this timeline already, set it lazily here
-	if t.account == nil {
-		timelineOwnerAccount := &gtsmodel.Account{}
-		if err := t.db.GetByID(t.accountID, timelineOwnerAccount); err != nil {
-			return err
-		}
-		t.account = timelineOwnerAccount
-	}
-
-	// to convert the status we need relevant accounts from it, so pull them out here
-	relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(gtsStatus)
-	if err != nil {
-		return err
-	}
-
-	// check if this is a boost...
-	var reblogOfStatus *gtsmodel.Status
-	if gtsStatus.BoostOfID != "" {
-		s := &gtsmodel.Status{}
-		if err := t.db.GetByID(gtsStatus.BoostOfID, s); err != nil {
-			return err
-		}
-		reblogOfStatus = s
-	}
-
-	// serialize the status (or, at least, convert it to a form that's ready to be serialized)
-	apiModelStatus, err := t.tc.StatusToMasto(gtsStatus, relevantAccounts.StatusAuthor, t.account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus)
-	if err != nil {
-		return err
-	}
-
-	// shove it in prepared posts as a prepared posts entry
-	preparedPostsEntry := &preparedPostsEntry{
-		createdAt: gtsStatus.CreatedAt,
-		statusID:  statusID,
-		prepared:  apiModelStatus,
-	}
-
-	return t.preparedPosts.insertPrepared(preparedPostsEntry)
-}