autonomic-cooperative/gotosocial
0
0
Fork 0
Code Releases Activity

Api/v1/statuses (#11)

Browse Source 
This PR adds:
Statuses

    New status creation.
    View existing status
    Delete a status
    Fave a status
    Unfave a status
    See who's faved a status

Media

    Upload media attachment and store/retrieve it
    Upload custom emoji and store/retrieve it

Fileserver

    Serve files from storage

Testing

    Test models, testrig -- run a GTS test instance and play around with it.
This commit is contained in:
Tobi Smethurst
2021-04-19 19:42:19 +02:00
committed by GitHub
parent 71a49e2b43
commit 32c5fd987a
150 changed files with 9023 additions and 788 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
internal/apimodule/fileserver/fileserver.go
View File

@ -1,20 +1,48 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fileserver
import (
"fmt"
"net/http"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/apimodule"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/db/model"
"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/router"
"github.com/superseriousbusiness/gotosocial/internal/storage"
)
// fileServer implements the RESTAPIModule interface.
const (
AccountIDKey = "account_id"
MediaTypeKey = "media_type"
MediaSizeKey = "media_size"
FileNameKey = "file_name"
FilesPath = "files"
)
// FileServer implements the RESTAPIModule interface.
// The goal here is to serve requested media files if the gotosocial server is configured to use local storage.
type fileServer struct {
type FileServer struct {
config *config.Config
db db.DB
storage storage.Storage
@ -24,34 +52,24 @@ type fileServer struct {
// New returns a new fileServer module
func New(config *config.Config, db db.DB, storage storage.Storage, log *logrus.Logger) apimodule.ClientAPIModule {
storageBase := config.StorageConfig.BasePath // TODO: do this properly
return &fileServer{
return &FileServer{
config: config,
db: db,
storage: storage,
log: log,
storageBase: storageBase,
storageBase: config.StorageConfig.ServeBasePath,
}
}
// Route satisfies the RESTAPIModule interface
func (m *fileServer) Route(s router.Router) error {
// s.AttachHandler(http.MethodPost, appsPath, m.appsPOSTHandler)
func (m *FileServer) Route(s router.Router) error {
s.AttachHandler(http.MethodGet, fmt.Sprintf("%s/:%s/:%s/:%s/:%s", m.storageBase, AccountIDKey, MediaTypeKey, MediaSizeKey, FileNameKey), m.ServeFile)
return nil
}
func (m *fileServer) CreateTables(db db.DB) error {
func (m *FileServer) CreateTables(db db.DB) error {
models := []interface{}{
&model.User{},
&model.Account{},
&model.Follow{},
&model.FollowRequest{},
&model.Status{},
&model.Application{},
&model.EmailDomainBlock{},
&model.MediaAttachment{},
&gtsmodel.MediaAttachment{},
}
for _, m := range models {

243
internal/apimodule/fileserver/servefile.go Normal file
View File

@ -0,0 +1,243 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package fileserver
import (
"bytes"
"net/http"
"strings"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
)
// ServeFile is for serving attachments, headers, and avatars to the requester from instance storage.
//
// Note: to mitigate scraping attempts, no information should be given out on a bad request except "404 page not found".
// Don't give away account ids or media ids or anything like that; callers shouldn't be able to infer anything.
func (m *FileServer) ServeFile(c *gin.Context) {
l := m.log.WithFields(logrus.Fields{
"func": "ServeFile",
"request_uri": c.Request.RequestURI,
"user_agent": c.Request.UserAgent(),
"origin_ip": c.ClientIP(),
})
l.Trace("received request")
// We use request params to check what to pull out of the database/storage so check everything. A request URL should be formatted as follows:
// "https://example.org/fileserver/[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[FILE_NAME]"
// "FILE_NAME" consists of two parts, the attachment's database id, a period, and the file extension.
accountID := c.Param(AccountIDKey)
if accountID == "" {
l.Debug("missing accountID from request")
c.String(http.StatusNotFound, "404 page not found")
return
}
mediaType := c.Param(MediaTypeKey)
if mediaType == "" {
l.Debug("missing mediaType from request")
c.String(http.StatusNotFound, "404 page not found")
return
}
mediaSize := c.Param(MediaSizeKey)
if mediaSize == "" {
l.Debug("missing mediaSize from request")
c.String(http.StatusNotFound, "404 page not found")
return
}
fileName := c.Param(FileNameKey)
if fileName == "" {
l.Debug("missing fileName from request")
c.String(http.StatusNotFound, "404 page not found")
return
}
// Only serve media types that are defined in our internal media module
switch mediaType {
case media.MediaHeader, media.MediaAvatar, media.MediaAttachment:
m.serveAttachment(c, accountID, mediaType, mediaSize, fileName)
return
case media.MediaEmoji:
m.serveEmoji(c, accountID, mediaType, mediaSize, fileName)
return
}
l.Debugf("mediatype %s not recognized", mediaType)
c.String(http.StatusNotFound, "404 page not found")
}
func (m *FileServer) serveAttachment(c *gin.Context, accountID string, mediaType string, mediaSize string, fileName string) {
l := m.log.WithFields(logrus.Fields{
"func": "serveAttachment",
"request_uri": c.Request.RequestURI,
"user_agent": c.Request.UserAgent(),
"origin_ip": c.ClientIP(),
})
// This corresponds to original-sized image as it was uploaded, small (which is the thumbnail), or static
switch mediaSize {
case media.MediaOriginal, media.MediaSmall, media.MediaStatic:
default:
l.Debugf("mediasize %s not recognized", mediaSize)
c.String(http.StatusNotFound, "404 page not found")
return
}
// derive the media id and the file extension from the last part of the request
spl := strings.Split(fileName, ".")
if len(spl) != 2 {
l.Debugf("filename %s not parseable", fileName)
c.String(http.StatusNotFound, "404 page not found")
return
}
wantedMediaID := spl[0]
fileExtension := spl[1]
if wantedMediaID == "" || fileExtension == "" {
l.Debugf("filename %s not parseable", fileName)
c.String(http.StatusNotFound, "404 page not found")
return
}
// now we know the attachment ID that the caller is asking for we can use it to pull the attachment out of the db
attachment := &gtsmodel.MediaAttachment{}
if err := m.db.GetByID(wantedMediaID, attachment); err != nil {
l.Debugf("attachment with id %s not retrievable: %s", wantedMediaID, err)
c.String(http.StatusNotFound, "404 page not found")
return
}
// make sure the given account id owns the requested attachment
if accountID != attachment.AccountID {
l.Debugf("account %s does not own attachment with id %s", accountID, wantedMediaID)
c.String(http.StatusNotFound, "404 page not found")
return
}
// now we can start preparing the response depending on whether we're serving a thumbnail or a larger attachment
var storagePath string
var contentType string
var contentLength int
switch mediaSize {
case media.MediaOriginal:
storagePath = attachment.File.Path
contentType = attachment.File.ContentType
contentLength = attachment.File.FileSize
case media.MediaSmall:
storagePath = attachment.Thumbnail.Path
contentType = attachment.Thumbnail.ContentType
contentLength = attachment.Thumbnail.FileSize
}
// use the path listed on the attachment we pulled out of the database to retrieve the object from storage
attachmentBytes, err := m.storage.RetrieveFileFrom(storagePath)
if err != nil {
l.Debugf("error retrieving from storage: %s", err)
c.String(http.StatusNotFound, "404 page not found")
return
}
l.Errorf("about to serve content length: %d attachment bytes is: %d", int64(contentLength), int64(len(attachmentBytes)))
// finally we can return with all the information we derived above
c.DataFromReader(http.StatusOK, int64(contentLength), contentType, bytes.NewReader(attachmentBytes), map[string]string{})
}
func (m *FileServer) serveEmoji(c *gin.Context, accountID string, mediaType string, mediaSize string, fileName string) {
l := m.log.WithFields(logrus.Fields{
"func": "serveEmoji",
"request_uri": c.Request.RequestURI,
"user_agent": c.Request.UserAgent(),
"origin_ip": c.ClientIP(),
})
// This corresponds to original-sized emoji as it was uploaded, or static
switch mediaSize {
case media.MediaOriginal, media.MediaStatic:
default:
l.Debugf("mediasize %s not recognized", mediaSize)
c.String(http.StatusNotFound, "404 page not found")
return
}
// derive the media id and the file extension from the last part of the request
spl := strings.Split(fileName, ".")
if len(spl) != 2 {
l.Debugf("filename %s not parseable", fileName)
c.String(http.StatusNotFound, "404 page not found")
return
}
wantedEmojiID := spl[0]
fileExtension := spl[1]
if wantedEmojiID == "" || fileExtension == "" {
l.Debugf("filename %s not parseable", fileName)
c.String(http.StatusNotFound, "404 page not found")
return
}
// now we know the attachment ID that the caller is asking for we can use it to pull the attachment out of the db
emoji := &gtsmodel.Emoji{}
if err := m.db.GetByID(wantedEmojiID, emoji); err != nil {
l.Debugf("emoji with id %s not retrievable: %s", wantedEmojiID, err)
c.String(http.StatusNotFound, "404 page not found")
return
}
// make sure the instance account id owns the requested emoji
instanceAccount := &gtsmodel.Account{}
if err := m.db.GetWhere("username", m.config.Host, instanceAccount); err != nil {
l.Debugf("error fetching instance account: %s", err)
c.String(http.StatusNotFound, "404 page not found")
return
}
if accountID != instanceAccount.ID {
l.Debugf("account %s does not own emoji with id %s", accountID, wantedEmojiID)
c.String(http.StatusNotFound, "404 page not found")
return
}
// now we can start preparing the response depending on whether we're serving a thumbnail or a larger attachment
var storagePath string
var contentType string
var contentLength int
switch mediaSize {
case media.MediaOriginal:
storagePath = emoji.ImagePath
contentType = emoji.ImageContentType
contentLength = emoji.ImageFileSize
case media.MediaStatic:
storagePath = emoji.ImageStaticPath
contentType = "image/png"
contentLength = emoji.ImageStaticFileSize
}
// use the path listed on the emoji we pulled out of the database to retrieve the object from storage
emojiBytes, err := m.storage.RetrieveFileFrom(storagePath)
if err != nil {
l.Debugf("error retrieving emoji from storage: %s", err)
c.String(http.StatusNotFound, "404 page not found")
return
}
// finally we can return with all the information we derived above
c.DataFromReader(http.StatusOK, int64(contentLength), contentType, bytes.NewReader(emojiBytes), map[string]string{})
}

157
internal/apimodule/fileserver/test/servefile_test.go Normal file
View File

@ -0,0 +1,157 @@
/*
GoToSocial
Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package test
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/apimodule/fileserver"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/db/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/mastotypes"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/storage"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type ServeFileTestSuite struct {
// standard suite interfaces
suite.Suite
config *config.Config
db db.DB
log *logrus.Logger
storage storage.Storage
mastoConverter mastotypes.Converter
mediaHandler media.MediaHandler
oauthServer oauth.Server
// standard suite models
testTokens map[string]*oauth.Token
testClients map[string]*oauth.Client
testApplications map[string]*gtsmodel.Application
testUsers map[string]*gtsmodel.User
testAccounts map[string]*gtsmodel.Account
testAttachments map[string]*gtsmodel.MediaAttachment
// item being tested
fileServer *fileserver.FileServer
}
/*
TEST INFRASTRUCTURE
*/
func (suite *ServeFileTestSuite) SetupSuite() {
// setup standard items
suite.config = testrig.NewTestConfig()
suite.db = testrig.NewTestDB()
suite.log = testrig.NewTestLog()
suite.storage = testrig.NewTestStorage()
suite.mastoConverter = testrig.NewTestMastoConverter(suite.db)
suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
// setup module being tested
suite.fileServer = fileserver.New(suite.config, suite.db, suite.storage, suite.log).(*fileserver.FileServer)
}
func (suite *ServeFileTestSuite) TearDownSuite() {
if err := suite.db.Stop(context.Background()); err != nil {
logrus.Panicf("error closing db connection: %s", err)
}
}
func (suite *ServeFileTestSuite) SetupTest() {
testrig.StandardDBSetup(suite.db)
testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
suite.testTokens = testrig.NewTestTokens()
suite.testClients = testrig.NewTestClients()
suite.testApplications = testrig.NewTestApplications()
suite.testUsers = testrig.NewTestUsers()
suite.testAccounts = testrig.NewTestAccounts()
suite.testAttachments = testrig.NewTestAttachments()
}
func (suite *ServeFileTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
testrig.StandardStorageTeardown(suite.storage)
}
/*
ACTUAL TESTS
*/
func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() {
targetAttachment, ok := suite.testAttachments["admin_account_status_1_attachment_1"]
assert.True(suite.T(), ok)
assert.NotNil(suite.T(), targetAttachment)
recorder := httptest.NewRecorder()
ctx, _ := gin.CreateTestContext(recorder)
ctx.Request = httptest.NewRequest(http.MethodGet, targetAttachment.URL, nil)
// normally the router would populate these params from the path values,
// but because we're calling the ServeFile function directly, we need to set them manually.
ctx.Params = gin.Params{
gin.Param{
Key: fileserver.AccountIDKey,
Value: targetAttachment.AccountID,
},
gin.Param{
Key: fileserver.MediaTypeKey,
Value: media.MediaAttachment,
},
gin.Param{
Key: fileserver.MediaSizeKey,
Value: media.MediaOriginal,
},
gin.Param{
Key: fileserver.FileNameKey,
Value: fmt.Sprintf("%s.jpeg", targetAttachment.ID),
},
}
// call the function we're testing and check status code
suite.fileServer.ServeFile(ctx)
suite.EqualValues(http.StatusOK, recorder.Code)
b, err := ioutil.ReadAll(recorder.Body)
assert.NoError(suite.T(), err)
assert.NotNil(suite.T(), b)
fileInStorage, err := suite.storage.RetrieveFileFrom(targetAttachment.File.Path)
assert.NoError(suite.T(), err)
assert.NotNil(suite.T(), fileInStorage)
assert.Equal(suite.T(), b, fileInStorage)
}
func TestServeFileTestSuite(t *testing.T) {
suite.Run(t, new(ServeFileTestSuite))
}