autonomic-cooperative
/
gotosocial
0
0
Fork 0
Code Releases Activity

bleep bloop

This commit is contained in:
tsmethurst 2021-04-09 23:55:57 +02:00
parent a30a1a267b
commit b713ccac9f
15 changed files with 337 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/gotosocial/main.go
View File

@ -143,6 +143,18 @@ func main() {
Value: 5242880, // 5mb
EnvVars: []string{envNames.MediaMaxVideoSize},
},
&cli.IntFlag{
Name: flagNames.MediaMinDescriptionChars,
Usage: "Min required chars for an image description",
Value: 0,
EnvVars: []string{envNames.MediaMinDescriptionChars},
},
&cli.IntFlag{
Name: flagNames.MediaMaxDescriptionChars,
Usage: "Max permitted chars for an image description",
Value: 500,
EnvVars: []string{envNames.MediaMaxDescriptionChars},
},
// STORAGE FLAGS
&cli.StringFlag{

6
internal/apimodule/media/media.go
View File

@ -32,7 +32,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/router"
)
const mediaPath = "/api/v1/media"
const basePath = "/api/v1/media"
type mediaModule struct {
mediaHandler media.MediaHandler
@ -46,7 +46,7 @@ type mediaModule struct {
func New(db db.DB, mediaHandler media.MediaHandler, mastoConverter mastotypes.Converter, config *config.Config, log *logrus.Logger) apimodule.ClientAPIModule {
return &mediaModule{
mediaHandler: mediaHandler,
config: config,
config: config,
db: db,
mastoConverter: mastoConverter,
log: log,
@ -55,7 +55,7 @@ func New(db db.DB, mediaHandler media.MediaHandler, mastoConverter mastotypes.Co
// Route satisfies the RESTAPIModule interface
func (m *mediaModule) Route(s router.Router) error {
s.AttachHandler(http.MethodPost, mediaPath, m.mediaCreatePOSTHandler)
s.AttachHandler(http.MethodPost, basePath, m.mediaCreatePOSTHandler)
return nil
}

31
internal/apimodule/media/mediacreate.go
View File

@ -67,14 +67,13 @@ func (m *mediaModule) mediaCreatePOSTHandler(c *gin.Context) {
return
}
// open the attachment and extract the bytes from it
f, err := form.File.Open()
if err != nil {
l.Debugf("error opening attachment: %s", err)
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not open provided attachment: %s", err)})
return
}
// extract the bytes
buf := new(bytes.Buffer)
size, err := io.Copy(buf, f)
if err != nil {
@ -88,6 +87,7 @@ func (m *mediaModule) mediaCreatePOSTHandler(c *gin.Context) {
return
}
// allow the mediaHandler to work its magic of processing the attachment bytes, and putting them in whatever storage backend we're using
attachment, err := m.mediaHandler.ProcessAttachment(buf.Bytes(), authed.Account.ID)
if err != nil {
l.Debugf("error reading attachment: %s", err)
@ -95,7 +95,14 @@ func (m *mediaModule) mediaCreatePOSTHandler(c *gin.Context) {
return
}
// now we need to add extra fields that the attachment processor doesn't know (from the form)
// TODO: handle this inside mediaHandler.ProcessAttachment (just pass more params to it)
// first description
attachment.Description = form.Description
// now parse the focus parameter
// TODO: tidy this up into a separate function and just return an error so all the c.JSON and return calls are obviated
var focusx, focusy float32
if form.Focus != "" {
spl := strings.Split(form.Focus, ",")
@ -106,12 +113,12 @@ func (m *mediaModule) mediaCreatePOSTHandler(c *gin.Context) {
}
xStr := spl[0]
yStr := spl[1]
if xStr == "" || xStr == "" {
if xStr == "" || yStr == "" {
l.Debugf("improperly formatted focus %s", form.Focus)
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("improperly formatted focus %s", form.Focus)})
return
}
fx, err := strconv.ParseFloat(xStr[:4], 32)
fx, err := strconv.ParseFloat(xStr, 32)
if err != nil {
l.Debugf("improperly formatted focus %s: %s", form.Focus, err)
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("improperly formatted focus %s", form.Focus)})
@ -123,7 +130,7 @@ func (m *mediaModule) mediaCreatePOSTHandler(c *gin.Context) {
return
}
focusx = float32(fx)
fy, err := strconv.ParseFloat(yStr[:4], 32)
fy, err := strconv.ParseFloat(yStr, 32)
if err != nil {
l.Debugf("improperly formatted focus %s: %s", form.Focus, err)
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("improperly formatted focus %s", form.Focus)})
@ -136,10 +143,11 @@ func (m *mediaModule) mediaCreatePOSTHandler(c *gin.Context) {
}
focusy = float32(fy)
}
attachment.FileMeta.Focus.X = focusx
attachment.FileMeta.Focus.Y = focusy
// prepare the frontend representation now -- if there are any errors here at least we can bail without
// having already put something in the database and then having to clean it up again (eugh)
mastoAttachment, err := m.mastoConverter.AttachmentToMasto(attachment)
if err != nil {
l.Debugf("error parsing media attachment to frontend type: %s", err)
@ -147,12 +155,14 @@ func (m *mediaModule) mediaCreatePOSTHandler(c *gin.Context) {
return
}
// now we can confidently put the attachment in the database
if err := m.db.Put(attachment); err != nil {
l.Debugf("error storing media attachment in db: %s", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("error storing media attachment in db: %s", err)})
return
}
// and return its frontend representation
c.JSON(http.StatusAccepted, mastoAttachment)
}
@ -162,7 +172,7 @@ func validateCreateMedia(form *mastotypes.AttachmentRequest, config *config.Medi
return errors.New("no attachment given")
}
// a very superficial check to see if no limits are exceeded
// a very superficial check to see if no size limits are exceeded
// we still don't actually know which media types we're dealing with but the other handlers will go into more detail there
maxSize := config.MaxVideoSize
if config.MaxImageSize > maxSize {
@ -171,5 +181,12 @@ func validateCreateMedia(form *mastotypes.AttachmentRequest, config *config.Medi
if form.File.Size > int64(maxSize) {
return fmt.Errorf("file size limit exceeded: limit is %d bytes but attachment was %d bytes", maxSize, form.File.Size)
}
if len(form.Description) < config.MinDescriptionChars || len(form.Description) > config.MaxDescriptionChars {
return fmt.Errorf("image description length must be between %d and %d characters (inclusive), but provided image description was %d chars", config.MinDescriptionChars, config.MaxDescriptionChars, len(form.Description))
}
// TODO: validate focus here
return nil
}

197
internal/apimodule/media/mediacreate_test.go Normal file
View File

@ -0,0 +1,197 @@
package media
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/mastotypes"
mastomodel "github.com/superseriousbusiness/gotosocial/internal/mastotypes/mastomodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type MediaCreateTestSuite struct {
suite.Suite
config *config.Config
mockOauthServer *oauth.MockServer
mockStorage *storage.MockStorage
mediaHandler media.MediaHandler
mastoConverter mastotypes.Converter
testTokens map[string]*oauth.Token
testClients map[string]*oauth.Client
testApplications map[string]*gtsmodel.Application
testUsers map[string]*gtsmodel.User
testAccounts map[string]*gtsmodel.Account
log *logrus.Logger
db db.DB
mediaModule *mediaModule
}
/*
TEST INFRASTRUCTURE
*/
// SetupSuite sets some variables on the suite that we can use as consts (more or less) throughout
func (suite *MediaCreateTestSuite) SetupSuite() {
// some of our subsequent entities need a log so create this here
log := logrus.New()
log.SetLevel(logrus.TraceLevel)
suite.log = log
// Direct config to local postgres instance
c := config.Empty()
c.Protocol = "http"
c.Host = "localhost"
c.DBConfig = &config.DBConfig{
Type: "postgres",
Address: "localhost",
Port: 5432,
User: "postgres",
Password: "postgres",
Database: "postgres",
ApplicationName: "gotosocial",
}
c.MediaConfig = &config.MediaConfig{
MaxImageSize: 2 << 20,
}
c.StorageConfig = &config.StorageConfig{
Backend: "local",
BasePath: "/tmp",
ServeProtocol: "http",
ServeHost: "localhost",
ServeBasePath: "/fileserver/media",
}
c.StatusesConfig = &config.StatusesConfig{
MaxChars: 500,
CWMaxChars: 50,
PollMaxOptions: 4,
PollOptionMaxChars: 50,
MaxMediaFiles: 4,
}
suite.config = c
// use an actual database for this, because it's just easier than mocking one out
database, err := db.New(context.Background(), c, log)
if err != nil {
suite.FailNow(err.Error())
}
suite.db = database
suite.mockOauthServer = &oauth.MockServer{}
suite.mockStorage = &storage.MockStorage{}
suite.mockStorage.On("StoreFileAt", mock.AnythingOfType("string"), mock.AnythingOfType("[]uint8")).Return(nil) // just pretend to store
suite.mediaHandler = media.New(suite.config, suite.db, suite.mockStorage, log)
suite.mastoConverter = mastotypes.New(suite.config, suite.db)
suite.mediaModule = New(suite.db, suite.mediaHandler, suite.mastoConverter, suite.config, suite.log).(*mediaModule)
}
func (suite *MediaCreateTestSuite) TearDownSuite() {
if err := suite.db.Stop(context.Background()); err != nil {
logrus.Panicf("error closing db connection: %s", err)
}
}
func (suite *MediaCreateTestSuite) SetupTest() {
if err := testrig.StandardDBSetup(suite.db); err != nil {
panic(err)
}
suite.testTokens = testrig.TestTokens()
suite.testClients = testrig.TestClients()
suite.testApplications = testrig.TestApplications()
suite.testUsers = testrig.TestUsers()
suite.testAccounts = testrig.TestAccounts()
}
// TearDownTest drops tables to make sure there's no data in the db
func (suite *MediaCreateTestSuite) TearDownTest() {
if err := testrig.StandardDBTeardown(suite.db); err != nil {
panic(err)
}
}
/*
ACTUAL TESTS
*/
func (suite *MediaCreateTestSuite) TestStatusCreatePOSTImageHandlerSuccessful() {
t := suite.testTokens["local_account_1"]
oauthToken := oauth.PGTokenToOauthToken(t)
// setup
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
ctx.Set(oauth.SessionAuthorizedToken, oauthToken)
ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"])
ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"])
buf, w, err := testrig.CreateMultipartFormData("file", "../../media/test/test-jpeg.jpg", map[string]string{
"description": "this is a test image -- a cool background from somewhere",
"focus": "-0.5,0.5",
})
if err != nil {
panic(err)
}
ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", basePath), bytes.NewReader(buf.Bytes())) // the endpoint we're hitting
ctx.Request.Header.Set("Content-Type", w.FormDataContentType())
suite.mediaModule.mediaCreatePOSTHandler(ctx)
// check response
suite.EqualValues(http.StatusAccepted, recorder.Code)
result := recorder.Result()
defer result.Body.Close()
b, err := ioutil.ReadAll(result.Body)
assert.NoError(suite.T(), err)
fmt.Println(string(b))
attachmentReply := &mastomodel.Attachment{}
err = json.Unmarshal(b, attachmentReply)
assert.NoError(suite.T(), err)
assert.Equal(suite.T(), "this is a test image -- a cool background from somewhere", attachmentReply.Description)
assert.Equal(suite.T(), "image", attachmentReply.Type)
assert.EqualValues(suite.T(), mastomodel.MediaMeta{
Original: mastomodel.MediaDimensions{
Width: 1920,
Height: 1080,
Size: "1920x1080",
Aspect: 1.7777778,
},
Small: mastomodel.MediaDimensions{
Width: 256,
Height: 144,
Size: "256x144",
Aspect: 1.7777778,
},
Focus: mastomodel.MediaFocus{
X: -0.5,
Y: 0.5,
},
}, attachmentReply.Meta)
assert.Equal(suite.T(), "LjCZnlvyRkRn_NvzRjWF?urqV@f9", attachmentReply.Blurhash)
assert.NotEmpty(suite.T(), attachmentReply.ID)
assert.NotEmpty(suite.T(), attachmentReply.URL)
assert.NotEmpty(suite.T(), attachmentReply.PreviewURL)
}
func TestMediaCreateTestSuite(t *testing.T) {
suite.Run(t, new(MediaCreateTestSuite))
}

26
internal/config/config.go
View File

@ -155,6 +155,14 @@ func (c *Config) ParseCLIFlags(f KeyedFlags) {
c.MediaConfig.MaxVideoSize = f.Int(fn.MediaMaxVideoSize)
}
if c.MediaConfig.MinDescriptionChars == 0 || f.IsSet(fn.MediaMinDescriptionChars) {
c.MediaConfig.MinDescriptionChars = f.Int(fn.MediaMinDescriptionChars)
}
if c.MediaConfig.MaxDescriptionChars == 0 || f.IsSet(fn.MediaMaxDescriptionChars) {
c.MediaConfig.MaxDescriptionChars = f.Int(fn.MediaMaxDescriptionChars)
}
// storage flags
if c.StorageConfig.Backend == "" || f.IsSet(fn.StorageBackend) {
c.StorageConfig.Backend = f.String(fn.StorageBackend)
@ -224,8 +232,10 @@ type Flags struct {
AccountsOpenRegistration string
AccountsRequireApproval string
MediaMaxImageSize string
MediaMaxVideoSize string
MediaMaxImageSize string
MediaMaxVideoSize string
MediaMinDescriptionChars string
MediaMaxDescriptionChars string
StorageBackend string
StorageBasePath string
@ -262,8 +272,10 @@ func GetFlagNames() Flags {
AccountsOpenRegistration: "accounts-open-registration",
AccountsRequireApproval: "accounts-require-approval",
MediaMaxImageSize: "media-max-image-size",
MediaMaxVideoSize: "media-max-video-size",
MediaMaxImageSize: "media-max-image-size",
MediaMaxVideoSize: "media-max-video-size",
MediaMinDescriptionChars: "media-min-description-chars",
MediaMaxDescriptionChars: "media-max-description-chars",
StorageBackend: "storage-backend",
StorageBasePath: "storage-base-path",
@ -301,8 +313,10 @@ func GetEnvNames() Flags {
AccountsOpenRegistration: "GTS_ACCOUNTS_OPEN_REGISTRATION",
AccountsRequireApproval: "GTS_ACCOUNTS_REQUIRE_APPROVAL",
MediaMaxImageSize: "GTS_MEDIA_MAX_IMAGE_SIZE",
MediaMaxVideoSize: "GTS_MEDIA_MAX_VIDEO_SIZE",
MediaMaxImageSize: "GTS_MEDIA_MAX_IMAGE_SIZE",
MediaMaxVideoSize: "GTS_MEDIA_MAX_VIDEO_SIZE",
MediaMinDescriptionChars: "GTS_MEDIA_MIN_DESCRIPTION_CHARS",
MediaMaxDescriptionChars: "GTS_MEDIA_MAX_DESCRIPTION_CHARS",
StorageBackend: "GTS_STORAGE_BACKEND",
StorageBasePath: "GTS_STORAGE_BASE_PATH",

4
internal/config/media.go
View File

@ -24,4 +24,8 @@ type MediaConfig struct {
MaxImageSize int `yaml:"maxImageSize"`
// Max size of uploaded video in bytes
MaxVideoSize int `yaml:"maxVideoSize"`
// Minimum amount of chars required in an image description
MinDescriptionChars int `yaml:"minDescriptionChars"`
// Max amount of chars allowed in an image description
MaxDescriptionChars int `yaml:"maxDescriptionChars"`
}

2
internal/db/gtsmodel/poll.go
View File

@ -17,5 +17,3 @@
*/
package gtsmodel

26
internal/mastotypes/converter.go
View File

@ -236,23 +236,23 @@ func (c *converter) AppToMastoPublic(a *gtsmodel.Application) (*mastotypes.Appli
func (c *converter) AttachmentToMasto(a *gtsmodel.MediaAttachment) (mastotypes.Attachment, error) {
return mastotypes.Attachment{
ID: a.ID,
Type: string(a.Type),
URL: a.URL,
PreviewURL: a.Thumbnail.URL,
RemoteURL: a.RemoteURL,
ID: a.ID,
Type: string(a.Type),
URL: a.URL,
PreviewURL: a.Thumbnail.URL,
RemoteURL: a.RemoteURL,
PreviewRemoteURL: a.Thumbnail.RemoteURL,
Meta: mastotypes.MediaMeta{
Original: mastotypes.MediaDimensions{
Width: a.FileMeta.Original.Width,
Width: a.FileMeta.Original.Width,
Height: a.FileMeta.Original.Height,
Size: fmt.Sprintf("%dx%d", a.FileMeta.Original.Width, a.FileMeta.Original.Height),
Size: fmt.Sprintf("%dx%d", a.FileMeta.Original.Width, a.FileMeta.Original.Height),
Aspect: float32(a.FileMeta.Original.Aspect),
},
Small: mastotypes.MediaDimensions{
Width: a.FileMeta.Small.Width,
Width: a.FileMeta.Small.Width,
Height: a.FileMeta.Small.Height,
Size: fmt.Sprintf("%dx%d", a.FileMeta.Small.Width, a.FileMeta.Small.Height),
Size: fmt.Sprintf("%dx%d", a.FileMeta.Small.Width, a.FileMeta.Small.Height),
Aspect: float32(a.FileMeta.Small.Aspect),
},
Focus: mastotypes.MediaFocus{
@ -261,7 +261,7 @@ func (c *converter) AttachmentToMasto(a *gtsmodel.MediaAttachment) (mastotypes.A
},
},
Description: a.Description,
Blurhash: a.Blurhash,
Blurhash: a.Blurhash,
}, nil
}
@ -284,9 +284,9 @@ func (c *converter) MentionToMasto(m *gtsmodel.Mention) (mastotypes.Mention, err
}
return mastotypes.Mention{
ID: m.ID,
ID: m.ID,
Username: target.Username,
URL: target.URL,
Acct: acct,
URL: target.URL,
Acct: acct,
}, nil
}

2
internal/media/media.go
View File

@ -113,7 +113,7 @@ func (mh *mediaHandler) ProcessAttachment(data []byte, accountID string) (*gtsmo
if err != nil {
return nil, err
}
mainType := strings.Split(contentType, "/")[0]
mainType := strings.Split(contentType, "/")[0]
switch mainType {
case "video":
if !supportedVideoType(contentType) {

5
internal/storage/inmem.go
View File

@ -7,6 +7,11 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/config"
)
// NewInMem returns an in-memory implementation of the Storage interface.
// This is good for testing and whatnot but ***SHOULD ABSOLUTELY NOT EVER
// BE USED IN A PRODUCTION SETTING***, because A) everything will be wiped out
// if you restart the server and B) if you store lots of images your RAM use
// will absolutely go through the roof.
func NewInMem(c *config.Config, log *logrus.Logger) (Storage, error) {
return &inMemStorage{
stored: make(map[string][]byte),

2
internal/storage/local.go
View File

@ -5,6 +5,8 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/config"
)
// NewLocal returns an implementation of the Storage interface that uses
// the local filesystem for storing and retrieving files, attachments, etc.
func NewLocal(c *config.Config, log *logrus.Logger) (Storage, error) {
return &localStorage{}, nil
}

4
internal/storage/storage.go
View File

@ -16,8 +16,12 @@
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Package storage contains an interface and implementations for storing and retrieving files and attachments.
package storage
// Storage is an interface for storing and retrieving blobs
// such as images, videos, and any other attachments/documents
// that shouldn't be stored in a database.
type Storage interface {
StoreFileAt(path string, data []byte) error
RetrieveFileFrom(path string) ([]byte, error)

4
internal/util/status.go
View File

@ -25,8 +25,8 @@ import (
var (
// mention regex can be played around with here: https://regex101.com/r/qwM9D3/1
mentionRegexString = `(?: |^|\W)(@[a-zA-Z0-9_]+(?:@[a-zA-Z0-9_\-\.]+)?)(?: |\n)`
mentionRegex = regexp.MustCompile(mentionRegexString)
mentionRegexString = `(?: |^|\W)(@[a-zA-Z0-9_]+(?:@[a-zA-Z0-9_\-\.]+)?)(?: |\n)`
mentionRegex = regexp.MustCompile(mentionRegexString)
// hashtag regex can be played with here: https://regex101.com/r/Vhy8pg/1
hashtagRegexString = `(?: |^|\W)?#([a-zA-Z0-9]{1,30})(?:\b|\r)`
hashtagRegex = regexp.MustCompile(hashtagRegexString)

4
testrig/models.go
View File

@ -442,9 +442,9 @@ func TestAccounts() map[string]*gtsmodel.Account {
func TestAttachments() map[string]*gtsmodel.MediaAttachment {
return map[string]*gtsmodel.MediaAttachment{
"admin_account_status_1": {
// "admin_account_status_1": {
},
// },
}
}

48
testrig/util.go Normal file
View File

@ -0,0 +1,48 @@
package testrig
import (
"bytes"
"io"
"mime/multipart"
"os"
)
// CreateMultipartFormData is a handy function for taking a fieldname and a filename, and creating a multipart form bytes buffer
// with the file contents set in the given fieldname. The extraFields param can be used to add extra FormFields to the request, as necessary.
// The returned bytes.Buffer b can be used like so:
// httptest.NewRequest(http.MethodPost, "https://example.org/whateverpath", bytes.NewReader(b.Bytes()))
// The returned *multipart.Writer w can be used to set the content type of the request, like so:
// req.Header.Set("Content-Type", w.FormDataContentType())
func CreateMultipartFormData(fieldName string, fileName string, extraFields map[string]string) (bytes.Buffer, *multipart.Writer, error) {
var b bytes.Buffer
var err error
w := multipart.NewWriter(&b)
var fw io.Writer
file, err := os.Open(fileName)
if err != nil {
return b, nil, err
}
if fw, err = w.CreateFormFile(fieldName, file.Name()); err != nil {
return b, nil, err
}
if _, err = io.Copy(fw, file); err != nil {
return b, nil, err
}
if extraFields != nil {
for k, v := range extraFields {
f, err := w.CreateFormField(k)
if err != nil {
return b, nil, err
}
if _, err := io.Copy(f, bytes.NewBufferString(v)); err != nil {
return b, nil, err
}
}
}
if err := w.Close(); err != nil {
return b, nil, err
}
return b, w, nil
}