Merge branch 'main' of https://github.com/superseriousbusiness/gotosocial into main

Remote instances are now dereferenced when they post to an inbox on a GtS instance.

    Dereferencing will be done first by checking the /api/v1/instance endpoint of an instance.
    If that doesn't work, /.well-known/nodeinfo will be checked.
    If that doesn't work, only a minimal representation of the instance will be stored.

A new field was added to the Instance database model. To create it:

alter table instances add column contact_account_username text;

.drone.yml

@@ -0,0 +1,19 @@
+---
+kind: pipeline
+name: automated container publishing
+steps:
+  - name: publish image
+    image: plugins/docker
+    settings:
+      username:
+        from_secret: docker_reg_username
+      password:
+        from_secret: docker_reg_passwd
+      repo: decentral1se/gotosocial
+      tags: latest
+trigger:
+  branch:
+    - main
+  event:
+    exclude:
+      - pull_request

README.md

@@ -2,6 +2,8 @@
 
 ![patrons](https://img.shields.io/liberapay/patrons/dumpsterqueer.svg?logo=liberapay) ![receives](https://img.shields.io/liberapay/receives/dumpsterqueer.svg?logo=liberapay)
 
+[![Build Status](https://drone.autonomic.zone/api/badges/autonomic-cooperative/gotosocial/status.svg?ref=refs/heads/main)](https://drone.autonomic.zone/autonomic-cooperative/gotosocial)
+
 Federated social media software.
 
 ![Sloth logo made by Freepik from www.flaticon.com](./web/assets/sloth.png)

internal/db/pg/pg.go

@@ -26,7 +26,6 @@ import (
 	"fmt"
 	"net"
 	"net/mail"
-	"regexp"
 	"strings"
 	"time"
 
@@ -48,7 +47,6 @@ type postgresService struct {
 	conn   *pg.DB
 	log    *logrus.Logger
 	cancel context.CancelFunc
-	// federationDB pub.Database
 }
 
 // NewPostgresService returns a postgresService derived from the provided config, which implements the go-fed DB interface.
@@ -120,12 +118,6 @@ func derivePGOptions(c *config.Config) (*pg.Options, error) {
 		return nil, errors.New("no address set")
 	}
 
-	ipv4Regex := regexp.MustCompile(`^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$`)
-	hostnameRegex := regexp.MustCompile(`^(?:[a-z0-9]+(?:-[a-z0-9]+)*\.)+[a-z]{2,}$`)
-	if !hostnameRegex.MatchString(c.DBConfig.Address) && !ipv4Regex.MatchString(c.DBConfig.Address) && c.DBConfig.Address != "localhost" {
-		return nil, fmt.Errorf("address %s was neither an ipv4 address nor a valid hostname", c.DBConfig.Address)
-	}
-
 	// validate username
 	if c.DBConfig.User == "" {
 		return nil, errors.New("no user set")

internal/federation/dereference.go

@@ -14,6 +14,8 @@ import (
 )
 
 func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) {
+	f.startHandshake(username, remoteAccountID)
+	defer f.stopHandshake(username, remoteAccountID)
 
 	transport, err := f.GetTransportForUser(username)
 	if err != nil {

internal/federation/federator.go

@@ -21,6 +21,7 @@ package federation
 import (
 	"net/http"
 	"net/url"
+	"sync"
 
 	"github.com/go-fed/activity/pub"
 	"github.com/sirupsen/logrus"
@@ -58,6 +59,8 @@ type Federator interface {
 	//
 	// If username is an empty string, our instance user's credentials will be used instead.
 	GetTransportForUser(username string) (transport.Transport, error)
+	// Handshaking returns true if the given username is currently in the process of dereferencing the remoteAccountID.
+	Handshaking(username string, remoteAccountID *url.URL) bool
 	pub.CommonBehavior
 	pub.FederatingProtocol
 }
@@ -71,6 +74,8 @@ type federator struct {
 	transportController transport.Controller
 	actor               pub.FederatingActor
 	log                 *logrus.Logger
+	handshakes          map[string][]*url.URL
+	handshakeSync       *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
 }
 
 // NewFederator returns a new federator
@@ -85,6 +90,7 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr
 		typeConverter:       typeConverter,
 		transportController: transportController,
 		log:                 log,
+		handshakeSync:       &sync.Mutex{},
 	}
 	actor := newFederatingActor(f, f, federatingDB, clock)
 	f.actor = actor

internal/federation/handshake.go

@@ -0,0 +1,80 @@
+package federation
+
+import "net/url"
+
+func (f *federator) Handshaking(username string, remoteAccountID *url.URL) bool {
+	f.handshakeSync.Lock()
+	defer f.handshakeSync.Unlock()
+
+	if f.handshakes == nil {
+		// handshakes isn't even initialized yet so we can't be handshaking with anyone
+		return false
+	}
+
+	remoteIDs, ok := f.handshakes[username]
+	if !ok {
+		// user isn't handshaking with anyone, bail
+		return false
+	}
+
+	for _, id := range remoteIDs {
+		if id.String() == remoteAccountID.String() {
+			// we are currently handshaking with the remote account, yep
+			return true
+		}
+	}
+
+	// didn't find it which means we're not handshaking
+	return false
+}
+
+func (f *federator) startHandshake(username string, remoteAccountID *url.URL) {
+	f.handshakeSync.Lock()
+	defer f.handshakeSync.Unlock()
+
+	// lazily initialize handshakes
+	if f.handshakes == nil {
+		f.handshakes = make(map[string][]*url.URL)
+	}
+
+	remoteIDs, ok := f.handshakes[username]
+	if !ok {
+		// there was nothing in there yet, so just add this entry and return
+		f.handshakes[username] = []*url.URL{remoteAccountID}
+		return
+	}
+
+	// add the remote ID to the slice
+	remoteIDs = append(remoteIDs, remoteAccountID)
+	f.handshakes[username] = remoteIDs
+}
+
+func (f *federator) stopHandshake(username string, remoteAccountID *url.URL) {
+	f.handshakeSync.Lock()
+	defer f.handshakeSync.Unlock()
+
+	if f.handshakes == nil {
+		return
+	}
+
+	remoteIDs, ok := f.handshakes[username]
+	if !ok {
+		// there was nothing in there yet anyway so just bail
+		return
+	}
+
+	newRemoteIDs := []*url.URL{}
+	for _, id := range remoteIDs {
+		if id.String() != remoteAccountID.String() {
+			newRemoteIDs = append(newRemoteIDs, id)
+		}
+	}
+
+	if len(newRemoteIDs) == 0 {
+		// there are no handshakes so just remove this user entry from the map and save a few bytes
+		delete(f.handshakes, username)
+	} else {
+		// there are still other handshakes ongoing
+		f.handshakes[username] = newRemoteIDs
+	}
+}

internal/processing/federation.go

@@ -26,7 +26,6 @@ import (
 
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
-	"github.com/sirupsen/logrus"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
@@ -35,23 +34,16 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
-// authenticateAndDereferenceFediRequest authenticates the HTTP signature of an incoming federation request, using the given
+// dereferenceFediRequest authenticates the HTTP signature of an incoming federation request, using the given
 // username to perform the validation. It will *also* dereference the originator of the request and return it as a gtsmodel account
 // for further processing. NOTE that this function will have the side effect of putting the dereferenced account into the database,
 // and passing it into the processor through a channel for further asynchronous processing.
-func (p *processor) authenticateAndDereferenceFediRequest(username string, r *http.Request) (*gtsmodel.Account, error) {
+func (p *processor) dereferenceFediRequest(username string, requestingAccountURI *url.URL) (*gtsmodel.Account, error) {
-
-	// first authenticate
-	requestingAccountURI, err := p.federator.AuthenticateFederatedRequest(username, r)
-	if err != nil {
-		return nil, fmt.Errorf("couldn't authenticate request for username %s: %s", username, err)
-	}
-
 	// OK now we can do the dereferencing part
 	// we might already have an entry for this account so check that first
 	requestingAccount := &gtsmodel.Account{}
 
-	err = p.db.GetWhere([]db.Where{{Key: "uri", Value: requestingAccountURI.String()}}, requestingAccount)
+	err := p.db.GetWhere([]db.Where{{Key: "uri", Value: requestingAccountURI.String()}}, requestingAccount)
 	if err == nil {
 		// we do have it yay, return it
 		return requestingAccount, nil
@@ -98,12 +90,6 @@ func (p *processor) authenticateAndDereferenceFediRequest(username string, r *ht
 }
 
 func (p *processor) GetFediUser(requestedUsername string, request *http.Request) (interface{}, gtserror.WithCode) {
-	l := p.log.WithFields(logrus.Fields{
-		"func":              "GetFediUser",
-		"requestedUsername": requestedUsername,
-		"requestURL":        request.URL.String(),
-	})
-
 	// get the account the request is referring to
 	requestedAccount := &gtsmodel.Account{}
 	if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil {
@@ -113,16 +99,21 @@ func (p *processor) GetFediUser(requestedUsername string, request *http.Request)
 	var requestedPerson vocab.ActivityStreamsPerson
 	var err error
 	if util.IsPublicKeyPath(request.URL) {
-		l.Debug("serving from public key path")
 		// if it's a public key path, we don't need to authenticate but we'll only serve the bare minimum user profile needed for the public key
 		requestedPerson, err = p.tc.AccountToASMinimal(requestedAccount)
 		if err != nil {
 			return nil, gtserror.NewErrorInternalError(err)
 		}
 	} else if util.IsUserPath(request.URL) {
-		l.Debug("serving from user path")
 		// if it's a user path, we want to fully authenticate the request before we serve any data, and then we can serve a more complete profile
-		requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+		requestingAccountURI, err := p.federator.AuthenticateFederatedRequest(requestedUsername, request)
+		if err != nil {
+			return nil, gtserror.NewErrorNotAuthorized(err)
+		}
+
+		// if we're already handshaking/dereferencing a remote account, we can skip the dereferencing part
+		if !p.federator.Handshaking(requestedUsername, requestingAccountURI) {
+			requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
 			if err != nil {
 				return nil, gtserror.NewErrorNotAuthorized(err)
 			}
@@ -135,6 +126,8 @@ func (p *processor) GetFediUser(requestedUsername string, request *http.Request)
 			if blocked {
 				return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID))
 			}
+		}
+
 		requestedPerson, err = p.tc.AccountToAS(requestedAccount)
 		if err != nil {
 			return nil, gtserror.NewErrorInternalError(err)
@@ -159,7 +152,12 @@ func (p *processor) GetFediFollowers(requestedUsername string, request *http.Req
 	}
 
 	// authenticate the request
-	requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+	requestingAccountURI, err := p.federator.AuthenticateFederatedRequest(requestedUsername, request)
+	if err != nil {
+		return nil, gtserror.NewErrorNotAuthorized(err)
+	}
+
+	requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
 	if err != nil {
 		return nil, gtserror.NewErrorNotAuthorized(err)
 	}
@@ -199,7 +197,12 @@ func (p *processor) GetFediFollowing(requestedUsername string, request *http.Req
 	}
 
 	// authenticate the request
-	requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+	requestingAccountURI, err := p.federator.AuthenticateFederatedRequest(requestedUsername, request)
+	if err != nil {
+		return nil, gtserror.NewErrorNotAuthorized(err)
+	}
+
+	requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
 	if err != nil {
 		return nil, gtserror.NewErrorNotAuthorized(err)
 	}
@@ -239,7 +242,12 @@ func (p *processor) GetFediStatus(requestedUsername string, requestedStatusID st
 	}
 
 	// authenticate the request
-	requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+	requestingAccountURI, err := p.federator.AuthenticateFederatedRequest(requestedUsername, request)
+	if err != nil {
+		return nil, gtserror.NewErrorNotAuthorized(err)
+	}
+
+	requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
 	if err != nil {
 		return nil, gtserror.NewErrorNotAuthorized(err)
 	}

internal/transport/derefinstance.go

@@ -13,6 +13,7 @@ import (
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 func (t *transport) DereferenceInstance(c context.Context, iri *url.URL) (*gtsmodel.Instance, error) {
@@ -26,7 +27,7 @@ func (t *transport) DereferenceInstance(c context.Context, iri *url.URL) (*gtsmo
 	//
 	// This will only work with Mastodon-api compatible instances: Mastodon, some Pleroma instances, GoToSocial.
 	l.Debugf("trying to dereference instance %s by /api/v1/instance", iri.Host)
-	i, err = dereferenceByAPIV1Instance(t, c, iri)
+	i, err = dereferenceByAPIV1Instance(c, t, iri)
 	if err == nil {
 		l.Debugf("successfully dereferenced instance using /api/v1/instance")
 		return i, nil
@@ -36,17 +37,28 @@ func (t *transport) DereferenceInstance(c context.Context, iri *url.URL) (*gtsmo
 	// If that doesn't work, try to dereference using /.well-known/nodeinfo.
 	// This will involve two API calls and return less info overall, but should be more widely compatible.
 	l.Debugf("trying to dereference instance %s by /.well-known/nodeinfo", iri.Host)
-	i, err = dereferenceByNodeInfo(t, c, iri)
+	i, err = dereferenceByNodeInfo(c, t, iri)
 	if err == nil {
 		l.Debugf("successfully dereferenced instance using /.well-known/nodeinfo")
 		return i, nil
 	}
 	l.Debugf("couldn't dereference instance using /.well-known/nodeinfo: %s", err)
 
-	return nil, fmt.Errorf("couldn't dereference instance %s using either /api/v1/instance or /.well-known/nodeinfo", iri.Host)
+	// we couldn't dereference the instance using any of the known methods, so just return a minimal representation
+	l.Debugf("returning minimal representation of instance %s", iri.Host)
+	id, err := id.NewRandomULID()
+	if err != nil {
+		return nil, fmt.Errorf("error creating new id for instance %s: %s", iri.Host, err)
+	}
+
+	return &gtsmodel.Instance{
+		ID:     id,
+		Domain: iri.Host,
+		URI:    iri.String(),
+	}, nil
 }
 
-func dereferenceByAPIV1Instance(t *transport, c context.Context, iri *url.URL) (*gtsmodel.Instance, error) {
+func dereferenceByAPIV1Instance(c context.Context, t *transport, iri *url.URL) (*gtsmodel.Instance, error) {
 	l := t.log.WithField("func", "dereferenceByAPIV1Instance")
 
 	cleanIRI := &url.URL{
@@ -119,8 +131,93 @@ func dereferenceByAPIV1Instance(t *transport, c context.Context, iri *url.URL) (
 	return i, nil
 }
 
-func dereferenceByNodeInfo(t *transport, c context.Context, iri *url.URL) (*gtsmodel.Instance, error) {
+func dereferenceByNodeInfo(c context.Context, t *transport, iri *url.URL) (*gtsmodel.Instance, error) {
-	l := t.log.WithField("func", "dereferenceByNodeInfo")
+	niIRI, err := callNodeInfoWellKnown(c, t, iri)
+	if err != nil {
+		return nil, fmt.Errorf("dereferenceByNodeInfo: error during initial call to well-known nodeinfo: %s", err)
+	}
+
+	ni, err := callNodeInfo(c, t, niIRI)
+	if err != nil {
+		return nil, fmt.Errorf("dereferenceByNodeInfo: error doing second call to nodeinfo uri %s: %s", niIRI.String(), err)
+	}
+
+	// we got a response of some kind! take what we can from it...
+	id, err := id.NewRandomULID()
+	if err != nil {
+		return nil, fmt.Errorf("dereferenceByNodeInfo: error creating new id for instance %s: %s", iri.Host, err)
+	}
+
+	// this is the bare minimum instance we'll return, and we'll add more stuff to it if we can
+	i := &gtsmodel.Instance{
+		ID:     id,
+		Domain: iri.Host,
+		URI:    iri.String(),
+	}
+
+	var title string
+	if i, present := ni.Metadata["nodeName"]; present {
+		// it's present, check it's a string
+		if v, ok := i.(string); ok {
+			// it is a string!
+			title = v
+		}
+	}
+	i.Title = title
+
+	var shortDescription string
+	if i, present := ni.Metadata["nodeDescription"]; present {
+		// it's present, check it's a string
+		if v, ok := i.(string); ok {
+			// it is a string!
+			shortDescription = v
+		}
+	}
+	i.ShortDescription = shortDescription
+
+	var contactEmail string
+	var contactAccountUsername string
+	if i, present := ni.Metadata["maintainer"]; present {
+		// it's present, check it's a map
+		if v, ok := i.(map[string]string); ok {
+			// see if there's an email in the map
+			if email, present := v["email"]; present {
+				if err := util.ValidateEmail(email); err == nil {
+					// valid email address
+					contactEmail = email
+				}
+			}
+			// see if there's a 'name' in the map
+			if name, present := v["name"]; present {
+				// name could be just a username, or could be a mention string eg @whatever@aaaa.com
+				username, _, err := util.ExtractMentionParts(name)
+				if err == nil {
+					// it was a mention string
+					contactAccountUsername = username
+				} else {
+					// not a mention string
+					contactAccountUsername = name
+				}
+			}
+		}
+	}
+	i.ContactEmail = contactEmail
+	i.ContactAccountUsername = contactAccountUsername
+
+	var software string
+	if ni.Software.Name != "" {
+		software = ni.Software.Name
+	}
+	if ni.Software.Version != "" {
+		software = software + " " + ni.Software.Version
+	}
+	i.Version = software
+
+	return i, nil
+}
+
+func callNodeInfoWellKnown(c context.Context, t *transport, iri *url.URL) (*url.URL, error) {
+	l := t.log.WithField("func", "callNodeInfoWellKnown")
 
 	cleanIRI := &url.URL{
 		Scheme: iri.Scheme,
@@ -150,7 +247,7 @@ func dereferenceByNodeInfo(t *transport, c context.Context, iri *url.URL) (*gtsm
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("GET request to %s failed (%d): %s", cleanIRI.String(), resp.StatusCode, resp.Status)
+		return nil, fmt.Errorf("callNodeInfoWellKnown: GET request to %s failed (%d): %s", cleanIRI.String(), resp.StatusCode, resp.Status)
 	}
 	b, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
@@ -158,28 +255,72 @@ func dereferenceByNodeInfo(t *transport, c context.Context, iri *url.URL) (*gtsm
 	}
 
 	if len(b) == 0 {
-		return nil, errors.New("dereferenceByNodeInfo: response bytes was len 0")
+		return nil, errors.New("callNodeInfoWellKnown: response bytes was len 0")
 	}
 
 	wellKnownResp := &apimodel.WellKnownResponse{}
 	if err := json.Unmarshal(b, wellKnownResp); err != nil {
-		return nil, fmt.Errorf("dereferenceByNodeInfo: could not unmarshal server response as WellKnownResponse: %s", err)
+		return nil, fmt.Errorf("callNodeInfoWellKnown: could not unmarshal server response as WellKnownResponse: %s", err)
 	}
 
 	// look through the links for the first one that matches the nodeinfo schema, this is what we need
 	var nodeinfoHref *url.URL
 	for _, l := range wellKnownResp.Links {
-		if l.Href == "" || !strings.HasPrefix(l.Rel, "http://nodeinfo.diaspora.software/ns/schema") {
+		if l.Href == "" || !strings.HasPrefix(l.Rel, "http://nodeinfo.diaspora.software/ns/schema/2") {
 			continue
 		}
 		nodeinfoHref, err = url.Parse(l.Href)
 		if err != nil {
-			return nil, fmt.Errorf("dereferenceByNodeInfo: couldn't parse url %s: %s", l.Href, err)
+			return nil, fmt.Errorf("callNodeInfoWellKnown: couldn't parse url %s: %s", l.Href, err)
 		}
 	}
 	if nodeinfoHref == nil {
-		return nil, errors.New("could not find nodeinfo rel in well known response")
+		return nil, errors.New("callNodeInfoWellKnown: could not find nodeinfo rel in well known response")
 	}
-aaaaaaaaaaaaaaaaa // do the second query
+
-	return nil, errors.New("not yet implemented")
+	return nodeinfoHref, nil
+}
+
+func callNodeInfo(c context.Context, t *transport, iri *url.URL) (*apimodel.Nodeinfo, error) {
+	l := t.log.WithField("func", "callNodeInfo")
+
+	l.Debugf("performing GET to %s", iri.String())
+	req, err := http.NewRequest("GET", iri.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(c)
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
+	req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
+	req.Header.Set("Host", iri.Host)
+	t.getSignerMu.Lock()
+	err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
+	t.getSignerMu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	resp, err := t.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("callNodeInfo: GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
+	}
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(b) == 0 {
+		return nil, errors.New("callNodeInfo: response bytes was len 0")
+	}
+
+	niResp := &apimodel.Nodeinfo{}
+	if err := json.Unmarshal(b, niResp); err != nil {
+		return nil, fmt.Errorf("callNodeInfo: could not unmarshal server response as Nodeinfo: %s", err)
+	}
+
+	return niResp, nil
 }