autonomic-cooperative/gotosocial
0
0
Fork 0
Code Releases Activity

bit more progress

Browse Source
This commit is contained in:
tsmethurst 2021-06-03 10:57:24 +02:00
parent c23075cac2
commit 5acca2a735
11 changed files with 99 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/db/db.go
View File

@ -143,7 +143,7 @@ type DB interface {
// GetFollowersByAccountID is a shortcut for the common action of fetching a list of accounts that accountID is followed by.
// The given slice 'followers' will be set to the result of the query, whatever it is.
// In case of no entries, a 'no entries' error will be returned
GetFollowersByAccountID(accountID string, followers *[]gtsmodel.Follow) error
GetFollowersByAccountID(accountID string, followers *[]gtsmodel.Follow, localOnly bool) error
// GetFavesByAccountID is a shortcut for the common action of fetching a list of faves made by the given accountID.
// The given slice 'faves' will be set to the result of the query, whatever it is.

7
internal/db/pg/pg.go
View File

@ -459,7 +459,12 @@ func (ps *postgresService) GetFollowingByAccountID(accountID string, following *
return nil
}
func (ps *postgresService) GetFollowersByAccountID(accountID string, followers *[]gtsmodel.Follow) error {
func (ps *postgresService) GetFollowersByAccountID(accountID string, followers *[]gtsmodel.Follow, localOnly bool) error {
q := ps.conn.Model(followers).Where("target_account_id = ?", accountID)
if err := ps.conn.Model(followers).Where("target_account_id = ?", accountID).Select(); err != nil {
if err == pg.ErrNoRows {
return nil

11
internal/federation/federatingdb/create.go
View File

@ -101,16 +101,19 @@ func (f *federatingDB) Create(ctx context.Context, asType vocab.Type) error {
}
if err := f.db.Put(status); err != nil {
if _, ok := err.(db.ErrAlreadyExists); ok {
// the status already exists in the database, which means we've already handled everything else,
// so we can just return nil here and be done with it.
return nil
}
// an actual error has happened
return fmt.Errorf("database error inserting status: %s", err)
}
fromFederatorChan <- gtsmodel.FromFederator{
APObjectType: gtsmodel.ActivityStreamsNote,
APActivityType: gtsmodel.ActivityStreamsCreate,
GTSModel: status,
ReceivingAccount: targetAcct,
APObjectType: gtsmodel.ActivityStreamsNote,
APActivityType: gtsmodel.ActivityStreamsCreate,
GTSModel: status,
ReceivingAccount: targetAcct,
}
}
}

2
internal/federation/federatingdb/followers.go
View File

@ -43,7 +43,7 @@ func (f *federatingDB) Followers(c context.Context, actorIRI *url.URL) (follower
}
acctFollowers := []gtsmodel.Follow{}
if err := f.db.GetFollowersByAccountID(acct.ID, &acctFollowers); err != nil {
if err := f.db.GetFollowersByAccountID(acct.ID, &acctFollowers, false); err != nil {
return nil, fmt.Errorf("db error getting followers for account id %s: %s", acct.ID, err)
}

8
internal/gtsmodel/messages.go
View File

@ -25,8 +25,8 @@ type FromClientAPI struct {
// FromFederator wraps a message that travels from the federator into the processor
type FromFederator struct {
APObjectType string
APActivityType string
GTSModel interface{}
ReceivingAccount *Account
APObjectType string
APActivityType string
GTSModel interface{}
ReceivingAccount *Account
}

10
internal/processing/fromcommon.go
View File

@ -211,3 +211,13 @@ func (p *processor) notifyAnnounce(status *gtsmodel.Status) error {
return nil
}
func (p *processor) timelineStatus(status *gtsmodel.Status) error {
followers := &[]gtsmodel.Follow{}
if err := p.db.GetFollowersByAccountID(status.AccountID, followers); err != nil {
}
// filter out so we only have the local ones
localFollowers := &[]gtsmodel.Account{}
}

5
internal/processing/fromfederator.go
View File

@ -56,9 +56,14 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
return fmt.Errorf("error updating dereferenced status in the db: %s", err)
}
if err := p.timelineStatus(incomingStatus); err != nil {
return err
}
if err := p.notifyStatus(incomingStatus); err != nil {
return err
}
case gtsmodel.ActivityStreamsProfile:
// CREATE AN ACCOUNT
incomingAccount, ok := federatorMsg.GTSModel.(*gtsmodel.Account)

84
internal/processing/timeline/manager.go
View File

@ -34,15 +34,37 @@ const (
desiredPostIndexLength = 400
)
// Manager abstracts functions for creating timelines for multiple accounts, and adding, removing, and fetching entries from those timelines.
//
// By the time a status hits the manager interface, it should already have been filtered and it should be established that the status indeed
// belongs in the home timeline of the given account ID.
//
// The manager makes a distinction between *indexed* posts and *prepared* posts.
//
// Indexed posts consist of just that post's ID (in the database) and the time it was created. An indexed post takes up very little memory, so
// it's not a huge priority to keep trimming the indexed posts list.
//
// Prepared posts consist of the post's database ID, the time it was created, AND the apimodel representation of that post, for quick serialization.
// Prepared posts of course take up more memory than indexed posts, so they should be regularly pruned if they're not being actively served.
type Manager interface {
// Ingest takes one status and indexes it into the timeline for the given account ID.
//
// It should already be established before calling this function that the status/post actually belongs in the timeline!
Ingest(status *gtsmodel.Status, timelineAccountID string) error
// HomeTimeline returns limit n amount of entries from the home timeline of the given account ID, in descending chronological order.
// If maxID is provided, it will return entries from that maxID onwards, inclusive.
HomeTimeline(timelineAccountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error)
// GetIndexedLength returns the amount of posts/statuses that have been *indexed* for the given account ID.
GetIndexedLength(timelineAccountID string) int
// GetDesiredIndexLength returns the amount of posts that we, ideally, index for each user.
GetDesiredIndexLength() int
// GetOldestIndexedID returns the status ID for the oldest post that we have indexed for the given account.
GetOldestIndexedID(timelineAccountID string) (string, error)
// PrepareXFromTop prepares limit n amount of posts, based on their indexed representations, from the top of the index.
PrepareXFromTop(timelineAccountID string, limit int) error
}
// NewManager returns a new timeline manager with the given database, typeconverter, config, and log.
func NewManager(db db.DB, tc typeutils.TypeConverter, config *config.Config, log *logrus.Logger) Manager {
return &manager{
accountTimelines: sync.Map{},
@ -68,17 +90,9 @@ func (m *manager) Ingest(status *gtsmodel.Status, timelineAccountID string) erro
"statusID": status.ID,
})
var t Timeline
i, ok := m.accountTimelines.Load(timelineAccountID)
if !ok {
t = NewTimeline(timelineAccountID, m.db, m.tc)
m.accountTimelines.Store(timelineAccountID, t)
} else {
t = i.(Timeline)
}
t := m.getOrCreateTimeline(timelineAccountID)
l.Trace("ingesting status")
return t.IndexOne(status.CreatedAt, status.ID)
}
@ -89,17 +103,9 @@ func (m *manager) Remove(statusID string, timelineAccountID string) error {
"statusID": statusID,
})
var t Timeline
i, ok := m.accountTimelines.Load(timelineAccountID)
if !ok {
t = NewTimeline(timelineAccountID, m.db, m.tc)
m.accountTimelines.Store(timelineAccountID, t)
} else {
t = i.(Timeline)
}
t := m.getOrCreateTimeline(timelineAccountID)
l.Trace("removing status")
return t.Remove(statusID)
}
@ -109,19 +115,12 @@ func (m *manager) HomeTimeline(timelineAccountID string, maxID string, sinceID s
"timelineAccountID": timelineAccountID,
})
var t Timeline
i, ok := m.accountTimelines.Load(timelineAccountID)
if !ok {
t = NewTimeline(timelineAccountID, m.db, m.tc)
m.accountTimelines.Store(timelineAccountID, t)
} else {
t = i.(Timeline)
}
t := m.getOrCreateTimeline(timelineAccountID)
var err error
var statuses []*apimodel.Status
if maxID != "" {
statuses, err = t.GetXBehindID(limit, maxID)
statuses, err = t.GetXFromID(limit, maxID)
} else {
statuses, err = t.GetXFromTop(limit)
}
@ -133,14 +132,7 @@ func (m *manager) HomeTimeline(timelineAccountID string, maxID string, sinceID s
}
func (m *manager) GetIndexedLength(timelineAccountID string) int {
var t Timeline
i, ok := m.accountTimelines.Load(timelineAccountID)
if !ok {
t = NewTimeline(timelineAccountID, m.db, m.tc)
m.accountTimelines.Store(timelineAccountID, t)
} else {
t = i.(Timeline)
}
t := m.getOrCreateTimeline(timelineAccountID)
return t.PostIndexLength()
}
@ -150,27 +142,29 @@ func (m *manager) GetDesiredIndexLength() int {
}
func (m *manager) GetOldestIndexedID(timelineAccountID string) (string, error) {
var t Timeline
i, ok := m.accountTimelines.Load(timelineAccountID)
if !ok {
t = NewTimeline(timelineAccountID, m.db, m.tc)
m.accountTimelines.Store(timelineAccountID, t)
} else {
t = i.(Timeline)
}
t := m.getOrCreateTimeline(timelineAccountID)
return t.OldestIndexedPostID()
}
func (m *manager) PrepareXFromTop(timelineAccountID string, limit int) error {
t := m.getOrCreateTimeline(timelineAccountID)
return t.PrepareXFromTop(limit)
}
func (m *manager) getOrCreateTimeline(timelineAccountID string) Timeline {
var t Timeline
i, ok := m.accountTimelines.Load(timelineAccountID)
if !ok {
t = NewTimeline(timelineAccountID, m.db, m.tc)
m.accountTimelines.Store(timelineAccountID, t)
} else {
t = i.(Timeline)
t, ok = i.(Timeline)
if !ok {
panic("couldn't parse entry as Timeline, this should never happen so panic")
}
}
return t.PrepareXFromTop(limit)
return t
}

2
internal/processing/timeline/postindex.go
View File

@ -15,7 +15,7 @@ type postIndexEntry struct {
statusID string
}
func (p *postIndex) index(i *postIndexEntry) error {
func (p *postIndex) insertIndexed(i *postIndexEntry) error {
if p.data == nil {
p.data = &list.List{}

64
internal/processing/timeline/timeline.go
View File

@ -32,20 +32,18 @@ import (
)
const (
fromLatest = "FROM_LATEST"
preparedPostsMaxLength = desiredPostIndexLength
)
type Timeline interface {
// GetXFromTop returns x amount of posts from the top of the timeline, from newest to oldest.
GetXFromTop(amount int) ([]*apimodel.Status, error)
// GetXFromTop returns x amount of posts from the given id onwards, from newest to oldest.
GetXBehindID(amount int, fromID string) ([]*apimodel.Status, error)
// GetXFromID returns x amount of posts from the given id onwards, from newest to oldest.
// This will include the status with the given ID.
GetXFromID(amount int, fromID string) ([]*apimodel.Status, error)
// IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property.
IndexOne(statusCreatedAt time.Time, statusID string) error
// IndexMany instructs the timeline to index all the given posts.
IndexMany([]*apimodel.Status) error
// Remove removes a status from the timeline.
Remove(statusID string) error
// OldestIndexedPostID returns the id of the rearmost (ie., the oldest) indexed post, or an error if something goes wrong.
@ -111,7 +109,6 @@ func (t *timeline) PrepareXFromIndex(amount int, index int) error {
prepared = prepared + 1
if prepared >= amount {
// we're done
fmt.Printf("\n\n\nprepared %d entries\n\n\n", prepared)
break
}
}
@ -180,7 +177,7 @@ func (t *timeline) GetXFromTop(amount int) ([]*apimodel.Status, error) {
return statuses, nil
}
func (t *timeline) GetXBehindID(amount int, fromID string) ([]*apimodel.Status, error) {
func (t *timeline) GetXFromID(amount int, fromID string) ([]*apimodel.Status, error) {
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
@ -197,15 +194,13 @@ func (t *timeline) GetXBehindID(amount int, fromID string) ([]*apimodel.Status,
return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
}
if entry.statusID == fromID {
fmt.Printf("\n\n\nfromid %s is at position %d\n\n\n", fromID, position)
break
}
position = position + 1
}
// make sure we have enough posts prepared to return
// make sure we have enough posts prepared behind it to return what we're being asked for
if t.preparedPosts.data.Len() < amount+position {
if err := t.PrepareXFromIndex(amount, position); err != nil {
return nil, err
}
@ -221,14 +216,14 @@ func (t *timeline) GetXBehindID(amount int, fromID string) ([]*apimodel.Status,
}
if !serving {
// we're not serving yet but we might on the next time round if we hit our from id
// start serving if we've hit the id we're looking for
if entry.statusID == fromID {
fmt.Printf("\n\n\nwe've hit fromid %s at position %d, will now serve\n\n\n", fromID, position)
serving = true
continue
}
} else {
// we're serving now!
}
if serving {
// serve up to the amount requested
statuses = append(statuses, entry.prepared)
served = served + 1
if served >= amount {
@ -248,7 +243,8 @@ func (t *timeline) IndexOne(statusCreatedAt time.Time, statusID string) error {
createdAt: statusCreatedAt,
statusID: statusID,
}
return t.postIndex.index(postIndexEntry)
return t.postIndex.insertIndexed(postIndexEntry)
}
func (t *timeline) Remove(statusID string) error {
@ -263,7 +259,7 @@ func (t *timeline) Remove(statusID string) error {
}
if entry.statusID == statusID {
t.postIndex.data.Remove(e)
break // bail once we found it
break // bail once we found and removed it
}
}
@ -275,34 +271,13 @@ func (t *timeline) Remove(statusID string) error {
}
if entry.statusID == statusID {
t.preparedPosts.data.Remove(e)
break // bail once we found it
break // bail once we found and removed it
}
}
return nil
}
func (t *timeline) IndexMany(statuses []*apimodel.Status) error {
t.Lock()
defer t.Unlock()
// add statuses to the index
for _, s := range statuses {
createdAt, err := time.Parse(s.CreatedAt, time.RFC3339)
if err != nil {
return fmt.Errorf("IndexMany: could not parse time %s on status id %s: %s", s.CreatedAt, s.ID, err)
}
postIndexEntry := &postIndexEntry{
createdAt: createdAt,
statusID: s.ID,
}
if err := t.postIndex.index(postIndexEntry); err != nil {
return err
}
}
return nil
}
func (t *timeline) Reset() error {
return nil
}
@ -318,11 +293,14 @@ func (t *timeline) PostIndexLength() int {
func (t *timeline) OldestIndexedPostID() (string, error) {
var id string
if t.postIndex == nil || t.postIndex.data == nil {
// return an empty string if postindex hasn't been initialized yet
return id, nil
}
e := t.postIndex.data.Back()
if e == nil {
// return an empty string if there's no back entry (ie., the index list hasn't been initialized yet)
return id, nil
}
@ -330,16 +308,18 @@ func (t *timeline) OldestIndexedPostID() (string, error) {
if !ok {
return id, errors.New("OldestIndexedPostID: could not parse e as a postIndexEntry")
}
return entry.statusID, nil
}
func (t *timeline) prepare(statusID string) error {
// start by getting the status out of the database according to its indexed ID
gtsStatus := &gtsmodel.Status{}
if err := t.db.GetByID(statusID, gtsStatus); err != nil {
return err
}
// if the account pointer hasn't been set on this timeline already, set it lazily here
if t.account == nil {
timelineOwnerAccount := &gtsmodel.Account{}
if err := t.db.GetByID(t.accountID, timelineOwnerAccount); err != nil {
@ -348,11 +328,13 @@ func (t *timeline) prepare(statusID string) error {
t.account = timelineOwnerAccount
}
// to convert the status we need relevant accounts from it, so pull them out here
relevantAccounts, err := t.db.PullRelevantAccountsFromStatus(gtsStatus)
if err != nil {
return err
}
// check if this is a boost...
var reblogOfStatus *gtsmodel.Status
if gtsStatus.BoostOfID != "" {
s := &gtsmodel.Status{}
@ -362,11 +344,13 @@ func (t *timeline) prepare(statusID string) error {
reblogOfStatus = s
}
// serialize the status (or, at least, convert it to a form that's ready to be serialized)
apiModelStatus, err := t.tc.StatusToMasto(gtsStatus, relevantAccounts.StatusAuthor, t.account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, reblogOfStatus)
if err != nil {
return err
}
// shove it in prepared posts as a prepared posts entry
preparedPostsEntry := &preparedPostsEntry{
createdAt: gtsStatus.CreatedAt,
statusID: statusID,

2
internal/typeutils/internaltofrontend.go
View File

@ -64,7 +64,7 @@ func (c *converter) AccountToMastoSensitive(a *gtsmodel.Account) (*model.Account
func (c *converter) AccountToMastoPublic(a *gtsmodel.Account) (*model.Account, error) {
// count followers
followers := []gtsmodel.Follow{}
if err := c.db.GetFollowersByAccountID(a.ID, &followers); err != nil {
if err := c.db.GetFollowersByAccountID(a.ID, &followers, false); err != nil {
if _, ok := err.(db.ErrNoEntries); !ok {
return nil, fmt.Errorf("error getting followers: %s", err)
}