updates to models and tags

This commit is contained in:
tsmethurst 2021-04-15 19:10:04 +02:00
parent 32629a378d
commit f91ba5b304
16 changed files with 290 additions and 70 deletions

View File

@ -25,8 +25,8 @@ import (
)
// ClientAPIModule represents a chunk of code (usually contained in a single package) that adds a set
// of functionalities and side effects to a router, by mapping routes and handlers onto it--in other words, a REST API ;)
// A ClientAPIMpdule corresponds roughly to one main path of the gotosocial REST api, for example /api/v1/accounts/ or /oauth/
// of functionalities and/or side effects to a router, by mapping routes and/or middlewares onto it--in other words, a REST API ;)
// A ClientAPIMpdule with routes corresponds roughly to one main path of the gotosocial REST api, for example /api/v1/accounts/ or /oauth/
type ClientAPIModule interface {
Route(s router.Router) error
CreateTables(db db.DB) error

View File

@ -0,0 +1,27 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package security
import "github.com/gin-gonic/gin"
// flocBlock prevents google chrome cohort tracking by writing the Permissions-Policy header after all other parts of the request have been completed.
// See: https://plausible.io/blog/google-floc
func (m *module) flocBlock(c *gin.Context) {
c.Header("Permissions-Policy", "interest-cohort=()")
}

View File

@ -0,0 +1,50 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package security
import (
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/apimodule"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/router"
)
// module implements the apiclient interface
type module struct {
config *config.Config
log *logrus.Logger
}
// New returns a new security module
func New(config *config.Config, log *logrus.Logger) apimodule.ClientAPIModule {
return &module{
config: config,
log: log,
}
}
func (m *module) Route(s router.Router) error {
s.AttachMiddleware(m.flocBlock)
return nil
}
func (m *module) CreateTables(db db.DB) error {
return nil
}

View File

@ -85,6 +85,7 @@ func (m *statusModule) CreateTables(db db.DB) error {
models := []interface{}{
&gtsmodel.User{},
&gtsmodel.Account{},
&gtsmodel.Block{},
&gtsmodel.Follow{},
&gtsmodel.FollowRequest{},
&gtsmodel.Status{},

View File

@ -135,39 +135,21 @@ func (m *statusModule) statusCreatePOSTHandler(c *gin.Context) {
return
}
// convert mentions to *gtsmodel.Mention
menchies, err := m.db.MentionStringsToMentions(util.DeriveMentions(form.Status), authed.Account.ID, thisStatusID)
if err != nil {
l.Debugf("error generating mentions from status: %s", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "error generating mentions from status"})
// handle mentions
if err := m.parseMentions(form, authed.Account.ID, newStatus); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
for _, menchie := range menchies {
if err := m.db.Put(menchie); err != nil {
l.Debugf("error putting mentions in db: %s", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "db error while generating mentions from status"})
return
}
}
newStatus.GTSMentions = menchies
// convert tags to *gtsmodel.Tag
tags, err := m.db.TagStringsToTags(util.DeriveHashtags(form.Status), authed.Account.ID, thisStatusID)
if err != nil {
l.Debugf("error generating hashtags from status: %s", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "error generating hashtags from status"})
if err := m.parseTags(form, authed.Account.ID, newStatus); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
newStatus.GTSTags = tags
// convert emojis to *gtsmodel.Emoji
emojis, err := m.db.EmojiStringsToEmojis(util.DeriveEmojis(form.Status), authed.Account.ID, thisStatusID)
if err != nil {
l.Debugf("error generating emojis from status: %s", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "error generating emojis from status"})
if err := m.parseEmojis(form, authed.Account.ID, newStatus); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
newStatus.GTSEmojis = emojis
/*
FROM THIS POINT ONWARDS WE ARE HAPPY WITH THE STATUS -- it is valid and we will try to create it
@ -196,8 +178,9 @@ func (m *statusModule) statusCreatePOSTHandler(c *gin.Context) {
Activity: newStatus,
}
// now we need to build up the mastodon-style status object to return to the submitter
/*
FROM THIS POINT ONWARDS WE ARE JUST CREATING THE FRONTEND REPRESENTATION OF THE STATUS TO RETURN TO THE SUBMITTER
*/
mastoVis := util.ParseMastoVisFromGTSVis(newStatus.Visibility)
mastoAccount, err := m.mastoConverter.AccountToMastoPublic(authed.Account)
@ -232,6 +215,16 @@ func (m *statusModule) statusCreatePOSTHandler(c *gin.Context) {
return
}
mastoTags := []mastotypes.Tag{}
for _, gtst := range newStatus.GTSTags {
mt, err := m.mastoConverter.TagToMasto(gtst)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
return
}
mastoTags = append(mastoTags, mt)
}
mastoEmojis := []mastotypes.Emoji{}
for _, gtse := range newStatus.GTSEmojis {
me, err := m.mastoConverter.EmojiToMasto(gtse)
@ -258,7 +251,7 @@ func (m *statusModule) statusCreatePOSTHandler(c *gin.Context) {
Account: mastoAccount,
MediaAttachments: mastoAttachments,
Mentions: mastoMentions,
Tags: nil,
Tags: mastoTags,
Emojis: mastoEmojis,
Text: form.Status,
}
@ -448,8 +441,8 @@ func (m *statusModule) parseMediaIDs(form *advancedStatusCreateForm, thisAccount
return nil
}
GTSMediaAttachments := []*gtsmodel.MediaAttachment{}
Attachments := []string{}
gtsMediaAttachments := []*gtsmodel.MediaAttachment{}
attachments := []string{}
for _, mediaID := range form.MediaIDs {
// check these attachments exist
a := &gtsmodel.MediaAttachment{}
@ -464,11 +457,11 @@ func (m *statusModule) parseMediaIDs(form *advancedStatusCreateForm, thisAccount
if a.StatusID != "" || a.ScheduledStatusID != "" {
return fmt.Errorf("media with id %s is already attached to a status", mediaID)
}
GTSMediaAttachments = append(GTSMediaAttachments, a)
Attachments = append(Attachments, a.ID)
gtsMediaAttachments = append(gtsMediaAttachments, a)
attachments = append(attachments, a.ID)
}
status.GTSMediaAttachments = GTSMediaAttachments
status.Attachments = Attachments
status.GTSMediaAttachments = gtsMediaAttachments
status.Attachments = attachments
return nil
}
@ -483,3 +476,57 @@ func parseLanguage(form *advancedStatusCreateForm, accountDefaultLanguage string
}
return nil
}
func (m *statusModule) parseMentions(form *advancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
menchies := []string{}
gtsMenchies, err := m.db.MentionStringsToMentions(util.DeriveMentions(form.Status), accountID, status.ID)
if err != nil {
return fmt.Errorf("error generating mentions from status: %s", err)
}
for _, menchie := range gtsMenchies {
if err := m.db.Put(menchie); err != nil {
return fmt.Errorf("error putting mentions in db: %s", err)
}
menchies = append(menchies, menchie.ID)
}
// add full populated gts menchies to the status for passing them around conveniently
status.GTSMentions = gtsMenchies
// add just the ids of the mentioned accounts to the status for putting in the db
status.Mentions = menchies
return nil
}
func (m *statusModule) parseTags(form *advancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
tags := []string{}
gtsTags, err := m.db.TagStringsToTags(util.DeriveHashtags(form.Status), accountID, status.ID)
if err != nil {
return fmt.Errorf("error generating hashtags from status: %s", err)
}
for _, tag := range gtsTags {
if err := m.db.Upsert(tag, "name"); err != nil {
return fmt.Errorf("error putting tags in db: %s", err)
}
tags = append(tags, tag.ID)
}
// add full populated gts tags to the status for passing them around conveniently
status.GTSTags = gtsTags
// add just the ids of the used tags to the status for putting in the db
status.Tags = tags
return nil
}
func (m *statusModule) parseEmojis(form *advancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
emojis := []string{}
gtsEmojis, err := m.db.EmojiStringsToEmojis(util.DeriveEmojis(form.Status), accountID, status.ID)
if err != nil {
return fmt.Errorf("error generating emojis from status: %s", err)
}
for _, e := range gtsEmojis {
emojis = append(emojis, e.ID)
}
// add full populated gts emojis to the status for passing them around conveniently
status.GTSEmojis = gtsEmojis
// add just the ids of the used emojis to the status for putting in the db
status.Emojis = emojis
return nil
}

View File

@ -92,6 +92,11 @@ type DB interface {
// The given interface i will be set to the result of the query, whatever it is. Use a pointer or a slice.
Put(i interface{}) error
// Upsert stores or updates i based on the given conflict column, as in https://www.postgresqltutorial.com/postgresql-upsert/
// It is up to the implementation to figure out how to store it, and using what key.
// The given interface i will be set to the result of the query, whatever it is. Use a pointer or a slice.
Upsert(i interface{}, conflictColumn string) error
// UpdateByID updates i with id id.
// The given interface i will be set to the result of the query, whatever it is. Use a pointer or a slice.
UpdateByID(id string, i interface{}) error
@ -192,15 +197,16 @@ type DB interface {
// checks in the database for the mentioned accounts, and returns a slice of mentions generated based on the given parameters.
//
// Note: this func doesn't/shouldn't do any manipulation of the accounts in the DB, it's just for checking
// if they exist in the db and conveniently returning them.
// if they exist in the db and conveniently returning them if they do.
MentionStringsToMentions(targetAccounts []string, originAccountID string, statusID string) ([]*gtsmodel.Mention, error)
// TagStringsToTags takes a slice of deduplicated, lowercase tags in the form "somehashtag", which have been
// used in a status. It takes the id of the account that wrote the status, and the id of the status itself, and then
// returns a slice of *model.Tag corresponding to the given tags.
// returns a slice of *model.Tag corresponding to the given tags. If the tag already exists in database, that tag
// will be returned. Otherwise a pointer to a new tag struct will be created and returned.
//
// Note: this func doesn't/shouldn't do any manipulation of the tags in the DB, it's just for checking
// if they exist in the db and conveniently returning them.
// if they exist in the db already, and conveniently returning them, or creating new tag structs.
TagStringsToTags(tags []string, originAccountID string, statusID string) ([]*gtsmodel.Tag, error)
// EmojiStringsToEmojis takes a slice of deduplicated, lowercase emojis in the form ":emojiname:", which have been
@ -208,7 +214,7 @@ type DB interface {
// returns a slice of *model.Emoji corresponding to the given emojis.
//
// Note: this func doesn't/shouldn't do any manipulation of the emoji in the DB, it's just for checking
// if they exist in the db and conveniently returning them.
// if they exist in the db and conveniently returning them if they do.
EmojiStringsToEmojis(emojis []string, originAccountID string, statusID string) ([]*gtsmodel.Emoji, error)
}

View File

@ -25,15 +25,15 @@ type Mention struct {
// ID of this mention in the database
ID string `pg:"type:uuid,default:gen_random_uuid(),pk,notnull,unique"`
// ID of the status this mention originates from
StatusID string
StatusID string `pg:",notnull"`
// When was this mention created?
CreatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
// When was this mention last updated?
UpdatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
// Who created this mention?
OriginAccountID string
OriginAccountID string `pg:",notnull"`
// Who does this mention target?
TargetAccountID string
TargetAccountID string `pg:",notnull"`
// Prevent this mention from generating a notification?
Silent bool
}

View File

@ -31,7 +31,13 @@ type Status struct {
// the html-formatted content of this status
Content string
// Database IDs of any media attachments associated with this status
Attachments []string
Attachments []string `pg:",array"`
// Database IDs of any tags used in this status
Tags []string `pg:",array"`
// Database IDs of any mentions in this status
Mentions []string `pg:",array"`
// Database IDs of any emojis used in this status
Emojis []string `pg:",array"`
// when was this status created?
CreatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
// when was this status updated?

View File

@ -20,17 +20,22 @@ package gtsmodel
import "time"
// Tag represents a hashtag for gathering public statuses together
type Tag struct {
ID string `pg:"type:uuid,default:gen_random_uuid(),pk,notnull"`
Name string `pg:"unique,notnull"`
CreatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
UpdatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
Useable bool
Trendable bool
Listable bool
ReviewedAt time.Time
RequestedReviewAt time.Time
LastStatusAt time.Time
MaxScore float32
MaxScoreAt time.Time
// id of this tag in the database
ID string `pg:",unique,type:uuid,default:gen_random_uuid(),pk,notnull"`
// name of this tag -- the tag without the hash part
Name string `pg:",unique,pk,notnull"`
// Which account ID is the first one we saw using this tag?
FirstSeenFromAccountID string
// when was this tag created
CreatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
// when was this tag last updated
UpdatedAt time.Time `pg:"type:timestamp,notnull,default:now()"`
// can our instance users use this tag?
Useable bool `pg:",notnull,default:true"`
// can our instance users look up this tag?
Listable bool `pg:",notnull,default:true"`
// when was this tag last used?
LastStatusAt time.Time `pg:"type:timestamp,notnull,default:now()"`
}

View File

@ -34,6 +34,7 @@ import (
"github.com/go-pg/pg/extra/pgdebug"
"github.com/go-pg/pg/v10"
"github.com/go-pg/pg/v10/orm"
"github.com/google/uuid"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
@ -273,8 +274,18 @@ func (ps *postgresService) Put(i interface{}) error {
return err
}
func (ps *postgresService) Upsert(i interface{}, conflictColumn string) error {
if _, err := ps.conn.Model(i).OnConflict(fmt.Sprintf("(%s) DO UPDATE", conflictColumn)).Insert(); err != nil {
if err == pg.ErrNoRows {
return ErrNoEntries{}
}
return err
}
return nil
}
func (ps *postgresService) UpdateByID(id string, i interface{}) error {
if _, err := ps.conn.Model(i).OnConflict("(id) DO UPDATE").Insert(); err != nil {
if _, err := ps.conn.Model(i).Where("id = ?", id).OnConflict("(id) DO UPDATE").Insert(); err != nil {
if err == pg.ErrNoRows {
return ErrNoEntries{}
}
@ -765,15 +776,33 @@ func (ps *postgresService) MentionStringsToMentions(targetAccounts []string, ori
return menchies, nil
}
// for now this function doesn't really use the database, but it's here because:
// A) it probably will later and
// B) it's v. similar to MentionStringsToMentions
func (ps *postgresService) TagStringsToTags(tags []string, originAccountID string, statusID string) ([]*gtsmodel.Tag, error) {
newTags := []*gtsmodel.Tag{}
for _, t := range tags {
newTags = append(newTags, &gtsmodel.Tag{
Name: t,
})
tag := &gtsmodel.Tag{}
// we can use selectorinsert here to create the new tag if it doesn't exist already
// inserted will be true if this is a new tag we just created
if err := ps.conn.Model(tag).Where("name = ?", t).Select(); err != nil {
if err == pg.ErrNoRows {
// tag doesn't exist yet so populate it
tag.ID = uuid.NewString()
tag.Name = t
tag.FirstSeenFromAccountID = originAccountID
tag.CreatedAt = time.Now()
tag.UpdatedAt = time.Now()
tag.Useable = true
tag.Listable = true
} else {
return nil, fmt.Errorf("error getting tag with name %s: %s", t, err)
}
}
// bail already if the tag isn't useable
if !tag.Useable {
continue
}
tag.LastStatusAt = time.Now()
newTags = append(newTags, tag)
}
return newTags, nil
}

View File

@ -34,6 +34,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/apimodule/auth"
"github.com/superseriousbusiness/gotosocial/internal/apimodule/fileserver"
mediaModule "github.com/superseriousbusiness/gotosocial/internal/apimodule/media"
"github.com/superseriousbusiness/gotosocial/internal/apimodule/security"
"github.com/superseriousbusiness/gotosocial/internal/apimodule/status"
"github.com/superseriousbusiness/gotosocial/internal/cache"
"github.com/superseriousbusiness/gotosocial/internal/config"
@ -83,9 +84,14 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr
fileServerModule := fileserver.New(c, dbService, storageBackend, log)
adminModule := admin.New(c, dbService, mediaHandler, mastoConverter, log)
statusModule := status.New(c, dbService, oauthServer, mediaHandler, mastoConverter, distributor, log)
securityModule := security.New(c, log)
apiModules := []apimodule.ClientAPIModule{
authModule, // this one has to go first so the other modules use its middleware
// modules with middleware go first
securityModule,
authModule,
// now everything else
accountModule,
appsModule,
mm,

View File

@ -60,6 +60,9 @@ type Converter interface {
// EmojiToMasto converts a gts model emoji into its mastodon (frontend) representation for serialization on the API.
EmojiToMasto(e *gtsmodel.Emoji) (mastotypes.Emoji, error)
// TagToMasto converts a gts model tag into its mastodon (frontend) representation for serialization on the API.
TagToMasto(t *gtsmodel.Tag) (mastotypes.Tag, error)
}
type converter struct {
@ -290,7 +293,7 @@ func (c *converter) MentionToMasto(m *gtsmodel.Mention) (mastotypes.Mention, err
}
return mastotypes.Mention{
ID: m.ID,
ID: target.ID,
Username: target.Username,
URL: target.URL,
Acct: acct,
@ -306,3 +309,12 @@ func (c *converter) EmojiToMasto(e *gtsmodel.Emoji) (mastotypes.Emoji, error) {
Category: e.CategoryID,
}, nil
}
func (c *converter) TagToMasto(t *gtsmodel.Tag) (mastotypes.Tag, error) {
tagURL := fmt.Sprintf("%s://%s/tags/%s", c.config.Protocol, c.config.Host, t.Name)
return mastotypes.Tag{
Name: t.Name,
URL: tagURL, // we don't serve URLs with collections of tagged statuses (FOR NOW) so this is purely for mastodon compatibility ¯\_(ツ)_/¯
}, nil
}

View File

@ -20,4 +20,8 @@ package mastotypes
// Tag represents a hashtag used within the content of a status. See https://docs.joinmastodon.org/entities/tag/
type Tag struct {
// The value of the hashtag after the # sign.
Name string `json:"name"`
// A link to the hashtag on the instance.
URL string `json:"url"`
}

View File

@ -83,7 +83,17 @@ func (r *router) AttachMiddleware(middleware gin.HandlerFunc) {
// New returns a new Router with the specified configuration, using the given logrus logger.
func New(config *config.Config, logger *logrus.Logger) (Router, error) {
engine := gin.New()
lvl, err := logrus.ParseLevel(config.LogLevel)
if err != nil {
return nil, fmt.Errorf("couldn't parse log level %s to set router level: %s", config.LogLevel, err)
}
switch lvl {
case logrus.TraceLevel, logrus.DebugLevel:
gin.SetMode(gin.DebugMode)
default:
gin.SetMode(gin.ReleaseMode)
}
engine := gin.Default()
// create a new session store middleware
store, err := sessionStore()

View File

@ -5,10 +5,9 @@ set -eux
SERVER_URL="http://localhost:8080"
REDIRECT_URI="${SERVER_URL}"
CLIENT_NAME="Test Application Name"
REGISTRATION_REASON="Testing whether or not this dang diggity thing works!"
REGISTRATION_EMAIL="test@example.org"
REGISTRATION_USERNAME="test_user"
REGISTRATION_USERNAME="${1}"
REGISTRATION_EMAIL="${2}"
REGISTRATION_PASSWORD="very safe password 123"
REGISTRATION_AGREEMENT="true"
REGISTRATION_LOCALE="en"

View File

@ -681,8 +681,11 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
ID: "502ccd6f-0edf-48d7-9016-2dfa4d3714cd",
URI: "http://localhost:8080/users/admin/statuses/502ccd6f-0edf-48d7-9016-2dfa4d3714cd",
URL: "http://localhost:8080/@admin/statuses/502ccd6f-0edf-48d7-9016-2dfa4d3714cd",
Content: "hello world! first post on the instance!",
Content: "hello world! #welcome ! first post on the instance :rainbow: !",
Attachments: []string{"b052241b-f30f-4dc6-92fc-2bad0be1f8d8"},
Tags: []string{"a7e8f5ca-88a1-4652-8079-a187eab8d56e"},
Mentions: []string{},
Emojis: []string{"a96ec4f3-1cae-47e4-a508-f9d66a6b221b"},
CreatedAt: time.Now().Add(-71 * time.Hour),
UpdatedAt: time.Now().Add(-71 * time.Hour),
Local: true,
@ -865,3 +868,18 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
},
}
}
func NewTestTags() map[string]*gtsmodel.Tag {
return map[string]*gtsmodel.Tag{
"welcome": {
ID: "a7e8f5ca-88a1-4652-8079-a187eab8d56e",
Name: "welcome",
FirstSeenFromAccountID: "",
CreatedAt: time.Now().Add(-71 * time.Hour),
UpdatedAt: time.Now().Add(-71 * time.Hour),
Useable: true,
Listable: true,
LastStatusAt: time.Now().Add(-71 * time.Hour),
},
}
}