From 86fd23ccd973de50eb7569735d3c8f6f9f9e04e0 Mon Sep 17 00:00:00 2001 From: tsmethurst Date: Sat, 5 Jun 2021 18:51:14 +0200 Subject: [PATCH] some more hacking away at the timeline code phew --- internal/db/pg/pg.go | 9 ++--- .../processing/synchronous/status/create.go | 1 - .../processing/synchronous/status/util.go | 6 +++ internal/timeline/manager.go | 6 ++- internal/timeline/timeline.go | 39 ++++++++++++++----- internal/typeutils/internal.go | 2 +- internal/util/regexes.go | 4 +- internal/util/statustools.go | 27 ++----------- 8 files changed, 49 insertions(+), 45 deletions(-) diff --git a/internal/db/pg/pg.go b/internal/db/pg/pg.go index b029b10..788a330 100644 --- a/internal/db/pg/pg.go +++ b/internal/db/pg/pg.go @@ -244,10 +244,6 @@ func (ps *postgresService) GetWhere(where []db.Where, i interface{}) error { return nil } -// func (ps *postgresService) GetWhereMany(i interface{}, where ...model.Where) error { -// return nil -// } - func (ps *postgresService) GetAll(i interface{}) error { if err := ps.conn.Model(i).Select(); err != nil { if err == pg.ErrNoRows { @@ -1257,6 +1253,8 @@ func (ps *postgresService) GetNotificationsForAccount(accountID string, limit in CONVERSION FUNCTIONS */ +// TODO: move these to the type converter, it's bananas that they're here and not there + func (ps *postgresService) MentionStringsToMentions(targetAccounts []string, originAccountID string, statusID string) ([]*gtsmodel.Mention, error) { ogAccount := >smodel.Account{} if err := ps.conn.Model(ogAccount).Where("id = ?", originAccountID).Select(); err != nil { @@ -1341,10 +1339,11 @@ func (ps *postgresService) TagStringsToTags(tags []string, originAccountID strin tag := >smodel.Tag{} // we can use selectorinsert here to create the new tag if it doesn't exist already // inserted will be true if this is a new tag we just created - if err := ps.conn.Model(tag).Where("name = ?", t).Select(); err != nil { + if err := ps.conn.Model(tag).Where("LOWER(?) = LOWER(?)", pg.Ident("name"), t).Select(); err != nil { if err == pg.ErrNoRows { // tag doesn't exist yet so populate it tag.ID = uuid.NewString() + tag.URL = fmt.Sprintf("%s://%s/tags/%s", ps.config.Protocol, ps.config.Host, t) tag.Name = t tag.FirstSeenFromAccountID = originAccountID tag.CreatedAt = time.Now() diff --git a/internal/processing/synchronous/status/create.go b/internal/processing/synchronous/status/create.go index a5aa7a4..2b5e0cf 100644 --- a/internal/processing/synchronous/status/create.go +++ b/internal/processing/synchronous/status/create.go @@ -21,7 +21,6 @@ func (p *processor) Create(account *gtsmodel.Account, application *gtsmodel.Appl ID: thisStatusID, URI: thisStatusURI, URL: thisStatusURL, - Content: util.HTMLFormat(form.Status), CreatedAt: time.Now(), UpdatedAt: time.Now(), Local: true, diff --git a/internal/processing/synchronous/status/util.go b/internal/processing/synchronous/status/util.go index 0f2e9f6..582dd4b 100644 --- a/internal/processing/synchronous/status/util.go +++ b/internal/processing/synchronous/status/util.go @@ -248,6 +248,12 @@ func (p *processor) processContent(form *apimodel.AdvancedStatusCreateForm, acco } } + // format tags nicely + for _, tag := range status.GTSTags { + tagContent := fmt.Sprintf(``, tag.URL, tag.Name) + content = strings.ReplaceAll(content, fmt.Sprintf("#%s", tag.Name), tagContent) + } + // replace newlines with breaks content = strings.ReplaceAll(content, "\n", "
") diff --git a/internal/timeline/manager.go b/internal/timeline/manager.go index 65259b1..4ee70d2 100644 --- a/internal/timeline/manager.go +++ b/internal/timeline/manager.go @@ -143,8 +143,10 @@ func (m *manager) HomeTimeline(timelineAccountID string, maxID string, sinceID s var err error var statuses []*apimodel.Status - if maxID != "" { - statuses, err = t.GetXFromIDOnwards(limit, maxID) + if maxID != "" && sinceID != "" { + statuses, err = t.GetXBetweenID(limit, maxID, sinceID) + } else if maxID != "" { + statuses, err = t.GetXBehindID(limit, maxID) } else if sinceID != "" { statuses, err = t.GetXBeforeID(limit, sinceID) } else { diff --git a/internal/timeline/timeline.go b/internal/timeline/timeline.go index c8c2b90..7c6e319 100644 --- a/internal/timeline/timeline.go +++ b/internal/timeline/timeline.go @@ -43,12 +43,21 @@ type Timeline interface { // GetXFromTop returns x amount of posts from the top of the timeline, from newest to oldest. GetXFromTop(amount int) ([]*apimodel.Status, error) - // GetXFromIDOnwards returns x amount of posts from the given id onwards, from newest to oldest. - // This will include the status with the given ID. - GetXFromIDOnwards(amount int, fromID string) ([]*apimodel.Status, error) + // GetXBehindID returns x amount of posts from the given id onwards, from newest to oldest. + // This will NOT include the status with the given ID. + // + // This corresponds to an api call to /timelines/home?max_id=WHATEVER + GetXBehindID(amount int, fromID string) ([]*apimodel.Status, error) // GetXBeforeID returns x amount of posts up to the given id, from newest to oldest. // This will NOT include the status with the given ID. + // + // This corresponds to an api call to /timelines/home?since_id=WHATEVER GetXBeforeID(amount int, sinceID string) ([]*apimodel.Status, error) + // GetXBetweenID returns x amount of posts from the given maxID, up to the given id, from newest to oldest. + // This will NOT include the status with the given IDs. + // + // This corresponds to an api call to /timelines/home?since_id=WHATEVER&max_id=WHATEVER_ELSE + GetXBetweenID(amount int, maxID string, sinceID string) ([]*apimodel.Status, error) /* INDEXING FUNCTIONS @@ -126,10 +135,11 @@ func (t *timeline) PrepareXFromPosition(amount int, desiredPosition int) error { if !preparing { // we haven't hit the position we need to prepare from yet + position = position + 1 if position == desiredPosition { preparing = true + continue } - position = position + 1 } else { if err := t.prepare(entry.statusID); err != nil { return fmt.Errorf("PrepareXFromTop: error preparing status with id %s: %s", entry.statusID, err) @@ -205,7 +215,7 @@ func (t *timeline) GetXFromTop(amount int) ([]*apimodel.Status, error) { return statuses, nil } -func (t *timeline) GetXFromIDOnwards(amount int, fromID string) ([]*apimodel.Status, error) { +func (t *timeline) GetXBehindID(amount int, behindID string) ([]*apimodel.Status, error) { // make a slice of statuses with the length we need to return statuses := make([]*apimodel.Status, 0, amount) @@ -221,7 +231,7 @@ func (t *timeline) GetXFromIDOnwards(amount int, fromID string) ([]*apimodel.Sta if !ok { return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry") } - if entry.statusID == fromID { + if entry.statusID == behindID { break } position = position + 1 @@ -245,12 +255,11 @@ func (t *timeline) GetXFromIDOnwards(amount int, fromID string) ([]*apimodel.Sta if !serving { // start serving if we've hit the id we're looking for - if entry.statusID == fromID { + if entry.statusID == behindID { serving = true + continue } - } - - if serving { + } else { // serve up to the amount requested statuses = append(statuses, entry.prepared) served = served + 1 @@ -297,6 +306,16 @@ servloop: return statuses, nil } +func (t *timeline) GetXBetweenID(amount int, maxID string, sinceID string) ([]*apimodel.Status, error) { + // make a slice of statuses with the length we need to return + statuses := make([]*apimodel.Status, 0, amount) + + // if there are no prepared posts, just return the empty slice + if t.preparedPosts.data == nil { + t.preparedPosts.data = &list.List{} + } +} + func (t *timeline) IndexOne(statusCreatedAt time.Time, statusID string) error { t.Lock() defer t.Unlock() diff --git a/internal/typeutils/internal.go b/internal/typeutils/internal.go index 626509b..2342f5f 100644 --- a/internal/typeutils/internal.go +++ b/internal/typeutils/internal.go @@ -56,7 +56,7 @@ func (c *converter) StatusToBoost(s *gtsmodel.Status, boostingAccount *gtsmodel. Emojis: []string{}, // the below fields will be taken from the target status - Content: util.HTMLFormat(s.Content), + Content: s.Content, ContentWarning: s.ContentWarning, ActivityStreamsType: s.ActivityStreamsType, Sensitive: s.Sensitive, diff --git a/internal/util/regexes.go b/internal/util/regexes.go index 27b01d2..1dcef25 100644 --- a/internal/util/regexes.go +++ b/internal/util/regexes.go @@ -41,11 +41,11 @@ var ( mentionNameRegex = regexp.MustCompile(mentionNameRegexString) // mention regex can be played around with here: https://regex101.com/r/qwM9D3/1 - mentionFinderRegexString = `(?: |^|\W)?(@[a-zA-Z0-9_]+(?:@[a-zA-Z0-9_\-\.]+)?)(?:[^a-zA-Z0-9]|\W)` + mentionFinderRegexString = `(?: |^|\W)(@[a-zA-Z0-9_]+(?:@[a-zA-Z0-9_\-\.]+)?)(?:[^a-zA-Z0-9]|\W|$)?` mentionFinderRegex = regexp.MustCompile(mentionFinderRegexString) // hashtag regex can be played with here: https://regex101.com/r/Vhy8pg/1 - hashtagFinderRegexString = fmt.Sprintf(`(?: |^|\W)?#([a-zA-Z0-9]{1,%d})(?:\b|\r)`, maximumHashtagLength) + hashtagFinderRegexString = fmt.Sprintf(`(?:\b)?#(\w{1,%d})(?:\b)`, maximumHashtagLength) hashtagFinderRegex = regexp.MustCompile(hashtagFinderRegexString) // emoji shortcode regex can be played with here: https://regex101.com/r/zMDRaG/1 diff --git a/internal/util/statustools.go b/internal/util/statustools.go index 8f9cb79..b51f2c8 100644 --- a/internal/util/statustools.go +++ b/internal/util/statustools.go @@ -35,7 +35,7 @@ func DeriveMentionsFromStatus(status string) []string { for _, m := range mentionFinderRegex.FindAllStringSubmatch(status, -1) { mentionedAccounts = append(mentionedAccounts, m[1]) } - return lower(unique(mentionedAccounts)) + return unique(mentionedAccounts) } // DeriveHashtagsFromStatus takes a plaintext (ie., not html-formatted) status, @@ -47,7 +47,7 @@ func DeriveHashtagsFromStatus(status string) []string { for _, m := range hashtagFinderRegex.FindAllStringSubmatch(status, -1) { tags = append(tags, m[1]) } - return lower(unique(tags)) + return unique(tags) } // DeriveEmojisFromStatus takes a plaintext (ie., not html-formatted) status, @@ -59,7 +59,7 @@ func DeriveEmojisFromStatus(status string) []string { for _, m := range emojiFinderRegex.FindAllStringSubmatch(status, -1) { emojis = append(emojis, m[1]) } - return lower(unique(emojis)) + return unique(emojis) } // ExtractMentionParts extracts the username test_user and the domain example.org @@ -94,24 +94,3 @@ func unique(s []string) []string { } return list } - -// lower lowercases all strings in a given string slice -func lower(s []string) []string { - new := []string{} - for _, i := range s { - new = append(new, strings.ToLower(i)) - } - return new -} - -// HTMLFormat takes a plaintext formatted status string, and converts it into -// a nice HTML-formatted string. -// -// This includes: -// - Replacing line-breaks with

-// - Replacing URLs with hrefs. -// - Replacing mentions with links to that account's URL as stored in the database. -func HTMLFormat(status string) string { - // TODO: write proper HTML formatting logic for a status - return status -}