Merge branch 'main' of https://github.com/superseriousbusiness/gotosocial into main

Remote instances are now dereferenced when they post to an inbox on a GtS instance.

    Dereferencing will be done first by checking the /api/v1/instance endpoint of an instance.
    If that doesn't work, /.well-known/nodeinfo will be checked.
    If that doesn't work, only a minimal representation of the instance will be stored.

A new field was added to the Instance database model. To create it:

alter table instances add column contact_account_username text;

internal/api/s2s/nodeinfo/nodeinfo.go

@@ -32,7 +32,7 @@ const (
 	// NodeInfoWellKnownPath is the base path for serving responses to nodeinfo lookup requests.
 	NodeInfoWellKnownPath = ".well-known/nodeinfo"
 	// NodeInfoBasePath is the path for serving nodeinfo responses.
-	NodeInfoBasePath      = "/nodeinfo/2.0"
+	NodeInfoBasePath = "/nodeinfo/2.0"
 )
 
 // Module implements the FederationModule interface

internal/federation/authenticate.go

@@ -35,8 +35,6 @@ import (
 	"github.com/go-fed/httpsig"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/transport"
-	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
 /*
@@ -99,11 +97,14 @@ func getPublicKeyFromResponse(c context.Context, b []byte, keyID *url.URL) (voca
 
 // AuthenticateFederatedRequest authenticates any kind of incoming federated request from a remote server. This includes things like
 // GET requests for dereferencing our users or statuses etc, and POST requests for delivering new Activities. The function returns
-// the URL of the owner of the public key used in the http signature.
+// the URL of the owner of the public key used in the requesting http signature.
 //
-// Authenticate in this case is defined as just making sure that the http request is actually signed by whoever claims
-// to have signed it, by fetching the public key from the signature and checking it against the remote public key. This function
-// *does not* check whether the request is authorized, only whether it's authentic.
+// Authenticate in this case is defined as making sure that the http request is actually signed by whoever claims
+// to have signed it, by fetching the public key from the signature and checking it against the remote public key.
+//
+// To avoid making unnecessary http calls towards blocked domains, this function *does* bail early if an instance-level domain block exists
+// for the request from the incoming domain. However, it does not check whether individual blocks exist between the requesting user or domain
+// and the requested user: this should be done elsewhere.
 //
 // The provided username will be used to generate a transport for making remote requests/derefencing the public key ID of the request signature.
 // Ideally you should pass in the username of the user *being requested*, so that the remote server can decide how to handle the request based on who's making it.
@@ -114,7 +115,12 @@ func getPublicKeyFromResponse(c context.Context, b []byte, keyID *url.URL) (voca
 //
 // Also note that this function *does not* dereference the remote account that the signature key is associated with.
 // Other functions should use the returned URL to dereference the remote account, if required.
-func (f *federator) AuthenticateFederatedRequest(username string, r *http.Request) (*url.URL, error) {
+func (f *federator) AuthenticateFederatedRequest(requestedUsername string, r *http.Request) (*url.URL, error) {
+
+	var publicKey interface{}
+	var pkOwnerURI *url.URL
+	var err error
+
 	// set this extra field for signature validation
 	r.Header.Set("host", f.config.Host)
 
@@ -130,11 +136,19 @@ func (f *federator) AuthenticateFederatedRequest(username string, r *http.Reques
 		return nil, fmt.Errorf("could not parse key id into a url: %s", err)
 	}
 
-	var publicKey interface{}
-	var pkOwnerURI *url.URL
+	// if the domain is blocked we want to make as few calls towards it as possible, so already bail here if that's the case!
+	blockedDomain, err := f.blockedDomain(requestingPublicKeyID.Host)
+	if err != nil {
+		return nil, fmt.Errorf("could not tell if domain %s was blocked or not: %s", requestingPublicKeyID.Host, err)
+	}
+	if blockedDomain {
+		return nil, fmt.Errorf("host %s was domain blocked, aborting auth", requestingPublicKeyID.Host)
+	}
+
 	requestingRemoteAccount := &gtsmodel.Account{}
 	requestingLocalAccount := &gtsmodel.Account{}
-	if strings.EqualFold(requestingPublicKeyID.Host, f.config.Host) {
+	requestingHost := requestingPublicKeyID.Host
+	if strings.EqualFold(requestingHost, f.config.Host) {
 		// LOCAL ACCOUNT REQUEST
 		// the request is coming from INSIDE THE HOUSE so skip the remote dereferencing
 		if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingLocalAccount); err != nil {
@@ -157,7 +171,7 @@ func (f *federator) AuthenticateFederatedRequest(username string, r *http.Reques
 		// REMOTE ACCOUNT REQUEST WITHOUT KEY CACHED LOCALLY
 		// the request is remote and we don't have the public key yet,
 		// so we need to authenticate the request properly by dereferencing the remote key
-		transport, err := f.GetTransportForUser(username)
+		transport, err := f.GetTransportForUser(requestedUsername)
 		if err != nil {
 			return nil, fmt.Errorf("transport err: %s", err)
 		}
@@ -212,152 +226,19 @@ func (f *federator) AuthenticateFederatedRequest(username string, r *http.Reques
 	return pkOwnerURI, nil
 }
 
-func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) {
-
-	transport, err := f.GetTransportForUser(username)
-	if err != nil {
-		return nil, fmt.Errorf("transport err: %s", err)
+func (f *federator) blockedDomain(host string) (bool, error) {
+	b := &gtsmodel.DomainBlock{}
+	err := f.db.GetWhere([]db.Where{{Key: "domain", Value: host, CaseInsensitive: true}}, b)
+	if err == nil {
+		// block exists
+		return true, nil
 	}
 
-	b, err := transport.Dereference(context.Background(), remoteAccountID)
-	if err != nil {
-		return nil, fmt.Errorf("error deferencing %s: %s", remoteAccountID.String(), err)
+	if _, ok := err.(db.ErrNoEntries); ok {
+		// there are no entries so there's no block
+		return false, nil
 	}
 
-	m := make(map[string]interface{})
-	if err := json.Unmarshal(b, &m); err != nil {
-		return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err)
-	}
-
-	t, err := streams.ToType(context.Background(), m)
-	if err != nil {
-		return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err)
-	}
-
-	switch t.GetTypeName() {
-	case string(gtsmodel.ActivityStreamsPerson):
-		p, ok := t.(vocab.ActivityStreamsPerson)
-		if !ok {
-			return nil, errors.New("error resolving type as activitystreams person")
-		}
-		return p, nil
-	case string(gtsmodel.ActivityStreamsApplication):
-		p, ok := t.(vocab.ActivityStreamsApplication)
-		if !ok {
-			return nil, errors.New("error resolving type as activitystreams application")
-		}
-		return p, nil
-	case string(gtsmodel.ActivityStreamsService):
-		p, ok := t.(vocab.ActivityStreamsService)
-		if !ok {
-			return nil, errors.New("error resolving type as activitystreams service")
-		}
-		return p, nil
-	}
-
-	return nil, fmt.Errorf("type name %s not supported", t.GetTypeName())
-}
-
-func (f *federator) DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) {
-	transport, err := f.GetTransportForUser(username)
-	if err != nil {
-		return nil, fmt.Errorf("transport err: %s", err)
-	}
-
-	b, err := transport.Dereference(context.Background(), remoteStatusID)
-	if err != nil {
-		return nil, fmt.Errorf("error deferencing %s: %s", remoteStatusID.String(), err)
-	}
-
-	m := make(map[string]interface{})
-	if err := json.Unmarshal(b, &m); err != nil {
-		return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err)
-	}
-
-	t, err := streams.ToType(context.Background(), m)
-	if err != nil {
-		return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err)
-	}
-
-	// Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile
-	switch t.GetTypeName() {
-	case gtsmodel.ActivityStreamsArticle:
-		p, ok := t.(vocab.ActivityStreamsArticle)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsArticle")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsDocument:
-		p, ok := t.(vocab.ActivityStreamsDocument)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsDocument")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsImage:
-		p, ok := t.(vocab.ActivityStreamsImage)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsImage")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsVideo:
-		p, ok := t.(vocab.ActivityStreamsVideo)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsVideo")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsNote:
-		p, ok := t.(vocab.ActivityStreamsNote)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsNote")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsPage:
-		p, ok := t.(vocab.ActivityStreamsPage)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsPage")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsEvent:
-		p, ok := t.(vocab.ActivityStreamsEvent)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsEvent")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsPlace:
-		p, ok := t.(vocab.ActivityStreamsPlace)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsPlace")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsProfile:
-		p, ok := t.(vocab.ActivityStreamsProfile)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsProfile")
-		}
-		return p, nil
-	}
-
-	return nil, fmt.Errorf("type name %s not supported", t.GetTypeName())
-}
-
-func (f *federator) GetTransportForUser(username string) (transport.Transport, error) {
-	// We need an account to use to create a transport for dereferecing the signature.
-	// If a username has been given, we can fetch the account with that username and use it.
-	// Otherwise, we can take the instance account and use those credentials to make the request.
-	ourAccount := &gtsmodel.Account{}
-	var u string
-	if username == "" {
-		u = f.config.Host
-	} else {
-		u = username
-	}
-	if err := f.db.GetLocalAccountByUsername(u, ourAccount); err != nil {
-		return nil, fmt.Errorf("error getting account %s from db: %s", username, err)
-	}
-
-	transport, err := f.transportController.NewTransport(ourAccount.PublicKeyURI, ourAccount.PrivateKey)
-	if err != nil {
-		return nil, fmt.Errorf("error creating transport for user %s: %s", username, err)
-	}
-	return transport, nil
+	// there's an actual error
+	return false, err
 }

internal/federation/commonbehavior.go

@@ -20,15 +20,10 @@ package federation
 
 import (
 	"context"
-	"fmt"
 	"net/http"
-	"net/url"
 
-	"github.com/go-fed/activity/pub"
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
 /*
@@ -101,53 +96,3 @@ func (f *federator) GetOutbox(ctx context.Context, r *http.Request) (vocab.Activ
 	// the CLIENT API, not through the federation API, so we just do nothing here.
 	return streams.NewActivityStreamsOrderedCollectionPage(), nil
 }
-
-// NewTransport returns a new Transport on behalf of a specific actor.
-//
-// The actorBoxIRI will be either the inbox or outbox of an actor who is
-// attempting to do the dereferencing or delivery. Any authentication
-// scheme applied on the request must be based on this actor. The
-// request must contain some sort of credential of the user, such as a
-// HTTP Signature.
-//
-// The gofedAgent passed in should be used by the Transport
-// implementation in the User-Agent, as well as the application-specific
-// user agent string. The gofedAgent will indicate this library's use as
-// well as the library's version number.
-//
-// Any server-wide rate-limiting that needs to occur should happen in a
-// Transport implementation. This factory function allows this to be
-// created, so peer servers are not DOS'd.
-//
-// Any retry logic should also be handled by the Transport
-// implementation.
-//
-// Note that the library will not maintain a long-lived pointer to the
-// returned Transport so that any private credentials are able to be
-// garbage collected.
-func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofedAgent string) (pub.Transport, error) {
-
-	var username string
-	var err error
-
-	if util.IsInboxPath(actorBoxIRI) {
-		username, err = util.ParseInboxPath(actorBoxIRI)
-		if err != nil {
-			return nil, fmt.Errorf("couldn't parse path %s as an inbox: %s", actorBoxIRI.String(), err)
-		}
-	} else if util.IsOutboxPath(actorBoxIRI) {
-		username, err = util.ParseOutboxPath(actorBoxIRI)
-		if err != nil {
-			return nil, fmt.Errorf("couldn't parse path %s as an outbox: %s", actorBoxIRI.String(), err)
-		}
-	} else {
-		return nil, fmt.Errorf("id %s was neither an inbox path nor an outbox path", actorBoxIRI.String())
-	}
-
-	account := &gtsmodel.Account{}
-	if err := f.db.GetLocalAccountByUsername(username, account); err != nil {
-		return nil, fmt.Errorf("error getting account with username %s from the db: %s", username, err)
-	}
-
-	return f.transportController.NewTransport(account.PublicKeyURI, account.PrivateKey)
-}

internal/federation/dereference.go

@@ -0,0 +1,153 @@
+package federation
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) {
+	f.startHandshake(username, remoteAccountID)
+	defer f.stopHandshake(username, remoteAccountID)
+
+	transport, err := f.GetTransportForUser(username)
+	if err != nil {
+		return nil, fmt.Errorf("transport err: %s", err)
+	}
+
+	b, err := transport.Dereference(context.Background(), remoteAccountID)
+	if err != nil {
+		return nil, fmt.Errorf("error deferencing %s: %s", remoteAccountID.String(), err)
+	}
+
+	m := make(map[string]interface{})
+	if err := json.Unmarshal(b, &m); err != nil {
+		return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err)
+	}
+
+	t, err := streams.ToType(context.Background(), m)
+	if err != nil {
+		return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err)
+	}
+
+	switch t.GetTypeName() {
+	case string(gtsmodel.ActivityStreamsPerson):
+		p, ok := t.(vocab.ActivityStreamsPerson)
+		if !ok {
+			return nil, errors.New("error resolving type as activitystreams person")
+		}
+		return p, nil
+	case string(gtsmodel.ActivityStreamsApplication):
+		p, ok := t.(vocab.ActivityStreamsApplication)
+		if !ok {
+			return nil, errors.New("error resolving type as activitystreams application")
+		}
+		return p, nil
+	case string(gtsmodel.ActivityStreamsService):
+		p, ok := t.(vocab.ActivityStreamsService)
+		if !ok {
+			return nil, errors.New("error resolving type as activitystreams service")
+		}
+		return p, nil
+	}
+
+	return nil, fmt.Errorf("type name %s not supported", t.GetTypeName())
+}
+
+func (f *federator) DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) {
+	transport, err := f.GetTransportForUser(username)
+	if err != nil {
+		return nil, fmt.Errorf("transport err: %s", err)
+	}
+
+	b, err := transport.Dereference(context.Background(), remoteStatusID)
+	if err != nil {
+		return nil, fmt.Errorf("error deferencing %s: %s", remoteStatusID.String(), err)
+	}
+
+	m := make(map[string]interface{})
+	if err := json.Unmarshal(b, &m); err != nil {
+		return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err)
+	}
+
+	t, err := streams.ToType(context.Background(), m)
+	if err != nil {
+		return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err)
+	}
+
+	// Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile
+	switch t.GetTypeName() {
+	case gtsmodel.ActivityStreamsArticle:
+		p, ok := t.(vocab.ActivityStreamsArticle)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsArticle")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsDocument:
+		p, ok := t.(vocab.ActivityStreamsDocument)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsDocument")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsImage:
+		p, ok := t.(vocab.ActivityStreamsImage)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsImage")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsVideo:
+		p, ok := t.(vocab.ActivityStreamsVideo)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsVideo")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsNote:
+		p, ok := t.(vocab.ActivityStreamsNote)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsNote")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsPage:
+		p, ok := t.(vocab.ActivityStreamsPage)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsPage")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsEvent:
+		p, ok := t.(vocab.ActivityStreamsEvent)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsEvent")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsPlace:
+		p, ok := t.(vocab.ActivityStreamsPlace)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsPlace")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsProfile:
+		p, ok := t.(vocab.ActivityStreamsProfile)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsProfile")
+		}
+		return p, nil
+	}
+
+	return nil, fmt.Errorf("type name %s not supported", t.GetTypeName())
+}
+
+func (f *federator) DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
+	transport, err := f.GetTransportForUser(username)
+	if err != nil {
+		return nil, fmt.Errorf("transport err: %s", err)
+	}
+
+	return transport.DereferenceInstance(context.Background(), remoteInstanceURI)
+}

internal/federation/federatingprotocol.go

@@ -125,6 +125,29 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
 		return ctx, false, fmt.Errorf("not authenticated: %s", err)
 	}
 
+	// authentication has passed, so add an instance entry for this instance if it hasn't been done already
+	i := &gtsmodel.Instance{}
+	if err := f.db.GetWhere([]db.Where{{Key: "domain", Value: publicKeyOwnerURI.Host, CaseInsensitive: true}}, i); err != nil {
+		if _, ok := err.(db.ErrNoEntries); !ok {
+			// there's been an actual error
+			return ctx, false, fmt.Errorf("error getting requesting account with public key id %s: %s", publicKeyOwnerURI.String(), err)
+		}
+
+		// we don't have an entry for this instance yet so dereference it
+		i, err = f.DereferenceRemoteInstance(username, &url.URL{
+			Scheme: publicKeyOwnerURI.Scheme,
+			Host:   publicKeyOwnerURI.Host,
+		})
+		if err != nil {
+			return nil, false, fmt.Errorf("could not dereference new remote instance %s during AuthenticatePostInbox: %s", publicKeyOwnerURI.Host, err)
+		}
+
+		// and put it in the db
+		if err := f.db.Put(i); err != nil {
+			return nil, false, fmt.Errorf("error inserting newly dereferenced instance %s: %s", publicKeyOwnerURI.Host, err)
+		}
+	}
+
 	requestingAccount := &gtsmodel.Account{}
 	if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: publicKeyOwnerURI.String()}}, requestingAccount); err != nil {
 		// there's been a proper error so return it

internal/federation/federator.go

@@ -21,12 +21,14 @@ package federation
 import (
 	"net/http"
 	"net/url"
+	"sync"
 
 	"github.com/go-fed/activity/pub"
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
@@ -49,11 +51,16 @@ type Federator interface {
 	// DereferenceRemoteStatus can be used to get the representation of a remote status, based on its ID (which is a URI).
 	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
 	DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error)
+	// DereferenceRemoteInstance takes the URL of a remote instance, and a username (optional) to spin up a transport with. It then
+	// does its damnedest to get some kind of information back about the instance, trying /api/v1/instance, then /.well-known/nodeinfo
+	DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
 	// GetTransportForUser returns a new transport initialized with the key credentials belonging to the given username.
 	// This can be used for making signed http requests.
 	//
 	// If username is an empty string, our instance user's credentials will be used instead.
 	GetTransportForUser(username string) (transport.Transport, error)
+	// Handshaking returns true if the given username is currently in the process of dereferencing the remoteAccountID.
+	Handshaking(username string, remoteAccountID *url.URL) bool
 	pub.CommonBehavior
 	pub.FederatingProtocol
 }
@@ -67,6 +74,8 @@ type federator struct {
 	transportController transport.Controller
 	actor               pub.FederatingActor
 	log                 *logrus.Logger
+	handshakes          map[string][]*url.URL
+	handshakeSync       *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
 }
 
 // NewFederator returns a new federator
@@ -81,6 +90,7 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr
 		typeConverter:       typeConverter,
 		transportController: transportController,
 		log:                 log,
+		handshakeSync:       &sync.Mutex{},
 	}
 	actor := newFederatingActor(f, f, federatingDB, clock)
 	f.actor = actor

internal/federation/handshake.go

@@ -0,0 +1,80 @@
+package federation
+
+import "net/url"
+
+func (f *federator) Handshaking(username string, remoteAccountID *url.URL) bool {
+	f.handshakeSync.Lock()
+	defer f.handshakeSync.Unlock()
+
+	if f.handshakes == nil {
+		// handshakes isn't even initialized yet so we can't be handshaking with anyone
+		return false
+	}
+
+	remoteIDs, ok := f.handshakes[username]
+	if !ok {
+		// user isn't handshaking with anyone, bail
+		return false
+	}
+
+	for _, id := range remoteIDs {
+		if id.String() == remoteAccountID.String() {
+			// we are currently handshaking with the remote account, yep
+			return true
+		}
+	}
+
+	// didn't find it which means we're not handshaking
+	return false
+}
+
+func (f *federator) startHandshake(username string, remoteAccountID *url.URL) {
+	f.handshakeSync.Lock()
+	defer f.handshakeSync.Unlock()
+
+	// lazily initialize handshakes
+	if f.handshakes == nil {
+		f.handshakes = make(map[string][]*url.URL)
+	}
+
+	remoteIDs, ok := f.handshakes[username]
+	if !ok {
+		// there was nothing in there yet, so just add this entry and return
+		f.handshakes[username] = []*url.URL{remoteAccountID}
+		return
+	}
+
+	// add the remote ID to the slice
+	remoteIDs = append(remoteIDs, remoteAccountID)
+	f.handshakes[username] = remoteIDs
+}
+
+func (f *federator) stopHandshake(username string, remoteAccountID *url.URL) {
+	f.handshakeSync.Lock()
+	defer f.handshakeSync.Unlock()
+
+	if f.handshakes == nil {
+		return
+	}
+
+	remoteIDs, ok := f.handshakes[username]
+	if !ok {
+		// there was nothing in there yet anyway so just bail
+		return
+	}
+
+	newRemoteIDs := []*url.URL{}
+	for _, id := range remoteIDs {
+		if id.String() != remoteAccountID.String() {
+			newRemoteIDs = append(newRemoteIDs, id)
+		}
+	}
+
+	if len(newRemoteIDs) == 0 {
+		// there are no handshakes so just remove this user entry from the map and save a few bytes
+		delete(f.handshakes, username)
+	} else {
+		// there are still other handshakes ongoing
+		f.handshakes[username] = newRemoteIDs
+	}
+}

internal/federation/transport.go

@@ -0,0 +1,84 @@
+package federation
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"github.com/go-fed/activity/pub"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// NewTransport returns a new Transport on behalf of a specific actor.
+//
+// The actorBoxIRI will be either the inbox or outbox of an actor who is
+// attempting to do the dereferencing or delivery. Any authentication
+// scheme applied on the request must be based on this actor. The
+// request must contain some sort of credential of the user, such as a
+// HTTP Signature.
+//
+// The gofedAgent passed in should be used by the Transport
+// implementation in the User-Agent, as well as the application-specific
+// user agent string. The gofedAgent will indicate this library's use as
+// well as the library's version number.
+//
+// Any server-wide rate-limiting that needs to occur should happen in a
+// Transport implementation. This factory function allows this to be
+// created, so peer servers are not DOS'd.
+//
+// Any retry logic should also be handled by the Transport
+// implementation.
+//
+// Note that the library will not maintain a long-lived pointer to the
+// returned Transport so that any private credentials are able to be
+// garbage collected.
+func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofedAgent string) (pub.Transport, error) {
+
+	var username string
+	var err error
+
+	if util.IsInboxPath(actorBoxIRI) {
+		username, err = util.ParseInboxPath(actorBoxIRI)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't parse path %s as an inbox: %s", actorBoxIRI.String(), err)
+		}
+	} else if util.IsOutboxPath(actorBoxIRI) {
+		username, err = util.ParseOutboxPath(actorBoxIRI)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't parse path %s as an outbox: %s", actorBoxIRI.String(), err)
+		}
+	} else {
+		return nil, fmt.Errorf("id %s was neither an inbox path nor an outbox path", actorBoxIRI.String())
+	}
+
+	account := &gtsmodel.Account{}
+	if err := f.db.GetLocalAccountByUsername(username, account); err != nil {
+		return nil, fmt.Errorf("error getting account with username %s from the db: %s", username, err)
+	}
+
+	return f.transportController.NewTransport(account.PublicKeyURI, account.PrivateKey)
+}
+
+func (f *federator) GetTransportForUser(username string) (transport.Transport, error) {
+	// We need an account to use to create a transport for dereferecing something.
+	// If a username has been given, we can fetch the account with that username and use it.
+	// Otherwise, we can take the instance account and use those credentials to make the request.
+	ourAccount := &gtsmodel.Account{}
+	var u string
+	if username == "" {
+		u = f.config.Host
+	} else {
+		u = username
+	}
+	if err := f.db.GetLocalAccountByUsername(u, ourAccount); err != nil {
+		return nil, fmt.Errorf("error getting account %s from db: %s", username, err)
+	}
+
+	transport, err := f.transportController.NewTransport(ourAccount.PublicKeyURI, ourAccount.PrivateKey)
+	if err != nil {
+		return nil, fmt.Errorf("error creating transport for user %s: %s", username, err)
+	}
+	return transport, nil
+}

internal/gtsmodel/instance.go

@@ -28,6 +28,8 @@ type Instance struct {
 	Terms string
 	// Contact email address for this instance
 	ContactEmail string
+	// Username of the contact account for this instance
+	ContactAccountUsername string
 	// Contact account ID in the database for this instance
 	ContactAccountID string `pg:"type:CHAR(26)"`
 	// Reputation score of this instance

internal/processing/federation.go

@@ -26,7 +26,6 @@ import (
 
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
-	"github.com/sirupsen/logrus"
 	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
@@ -35,23 +34,16 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
-// authenticateAndDereferenceFediRequest authenticates the HTTP signature of an incoming federation request, using the given
+// dereferenceFediRequest authenticates the HTTP signature of an incoming federation request, using the given
 // username to perform the validation. It will *also* dereference the originator of the request and return it as a gtsmodel account
 // for further processing. NOTE that this function will have the side effect of putting the dereferenced account into the database,
 // and passing it into the processor through a channel for further asynchronous processing.
-func (p *processor) authenticateAndDereferenceFediRequest(username string, r *http.Request) (*gtsmodel.Account, error) {
-
-	// first authenticate
-	requestingAccountURI, err := p.federator.AuthenticateFederatedRequest(username, r)
-	if err != nil {
-		return nil, fmt.Errorf("couldn't authenticate request for username %s: %s", username, err)
-	}
-
+func (p *processor) dereferenceFediRequest(username string, requestingAccountURI *url.URL) (*gtsmodel.Account, error) {
 	// OK now we can do the dereferencing part
 	// we might already have an entry for this account so check that first
 	requestingAccount := &gtsmodel.Account{}
 
-	err = p.db.GetWhere([]db.Where{{Key: "uri", Value: requestingAccountURI.String()}}, requestingAccount)
+	err := p.db.GetWhere([]db.Where{{Key: "uri", Value: requestingAccountURI.String()}}, requestingAccount)
 	if err == nil {
 		// we do have it yay, return it
 		return requestingAccount, nil
@@ -98,12 +90,6 @@ func (p *processor) authenticateAndDereferenceFediRequest(username string, r *ht
 }
 
 func (p *processor) GetFediUser(requestedUsername string, request *http.Request) (interface{}, gtserror.WithCode) {
-	l := p.log.WithFields(logrus.Fields{
-		"func":              "GetFediUser",
-		"requestedUsername": requestedUsername,
-		"requestURL":        request.URL.String(),
-	})
-
 	// get the account the request is referring to
 	requestedAccount := &gtsmodel.Account{}
 	if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil {
@@ -113,28 +99,35 @@ func (p *processor) GetFediUser(requestedUsername string, request *http.Request)
 	var requestedPerson vocab.ActivityStreamsPerson
 	var err error
 	if util.IsPublicKeyPath(request.URL) {
-		l.Debug("serving from public key path")
 		// if it's a public key path, we don't need to authenticate but we'll only serve the bare minimum user profile needed for the public key
 		requestedPerson, err = p.tc.AccountToASMinimal(requestedAccount)
 		if err != nil {
 			return nil, gtserror.NewErrorInternalError(err)
 		}
 	} else if util.IsUserPath(request.URL) {
-		l.Debug("serving from user path")
 		// if it's a user path, we want to fully authenticate the request before we serve any data, and then we can serve a more complete profile
-		requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+		requestingAccountURI, err := p.federator.AuthenticateFederatedRequest(requestedUsername, request)
 		if err != nil {
 			return nil, gtserror.NewErrorNotAuthorized(err)
 		}
 
-		blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID)
-		if err != nil {
-			return nil, gtserror.NewErrorInternalError(err)
+		// if we're already handshaking/dereferencing a remote account, we can skip the dereferencing part
+		if !p.federator.Handshaking(requestedUsername, requestingAccountURI) {
+			requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
+			if err != nil {
+				return nil, gtserror.NewErrorNotAuthorized(err)
+			}
+
+			blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID)
+			if err != nil {
+				return nil, gtserror.NewErrorInternalError(err)
+			}
+
+			if blocked {
+				return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID))
+			}
 		}
 
-		if blocked {
-			return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID))
-		}
 		requestedPerson, err = p.tc.AccountToAS(requestedAccount)
 		if err != nil {
 			return nil, gtserror.NewErrorInternalError(err)
@@ -159,7 +152,12 @@ func (p *processor) GetFediFollowers(requestedUsername string, request *http.Req
 	}
 
 	// authenticate the request
-	requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+	requestingAccountURI, err := p.federator.AuthenticateFederatedRequest(requestedUsername, request)
+	if err != nil {
+		return nil, gtserror.NewErrorNotAuthorized(err)
+	}
+
+	requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
 	if err != nil {
 		return nil, gtserror.NewErrorNotAuthorized(err)
 	}
@@ -199,7 +197,12 @@ func (p *processor) GetFediFollowing(requestedUsername string, request *http.Req
 	}
 
 	// authenticate the request
-	requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+	requestingAccountURI, err := p.federator.AuthenticateFederatedRequest(requestedUsername, request)
+	if err != nil {
+		return nil, gtserror.NewErrorNotAuthorized(err)
+	}
+
+	requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
 	if err != nil {
 		return nil, gtserror.NewErrorNotAuthorized(err)
 	}
@@ -239,7 +242,12 @@ func (p *processor) GetFediStatus(requestedUsername string, requestedStatusID st
 	}
 
 	// authenticate the request
-	requestingAccount, err := p.authenticateAndDereferenceFediRequest(requestedUsername, request)
+	requestingAccountURI, err := p.federator.AuthenticateFederatedRequest(requestedUsername, request)
+	if err != nil {
+		return nil, gtserror.NewErrorNotAuthorized(err)
+	}
+
+	requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
 	if err != nil {
 		return nil, gtserror.NewErrorNotAuthorized(err)
 	}

internal/transport/deliver.go

@@ -0,0 +1,16 @@
+package transport
+
+import (
+	"context"
+	"net/url"
+)
+
+func (t *transport) BatchDeliver(c context.Context, b []byte, recipients []*url.URL) error {
+	return t.sigTransport.BatchDeliver(c, b, recipients)
+}
+
+func (t *transport) Deliver(c context.Context, b []byte, to *url.URL) error {
+	l := t.log.WithField("func", "Deliver")
+	l.Debugf("performing POST to %s", to.String())
+	return t.sigTransport.Deliver(c, b, to)
+}

internal/transport/dereference.go

@@ -0,0 +1,12 @@
+package transport
+
+import (
+	"context"
+	"net/url"
+)
+
+func (t *transport) Dereference(c context.Context, iri *url.URL) ([]byte, error) {
+	l := t.log.WithField("func", "Dereference")
+	l.Debugf("performing GET to %s", iri.String())
+	return t.sigTransport.Dereference(c, iri)
+}

internal/transport/derefinstance.go

@@ -0,0 +1,326 @@
+package transport
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+
+	apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+func (t *transport) DereferenceInstance(c context.Context, iri *url.URL) (*gtsmodel.Instance, error) {
+	l := t.log.WithField("func", "DereferenceInstance")
+
+	var i *gtsmodel.Instance
+	var err error
+
+	// First try to dereference using /api/v1/instance.
+	// This will provide the most complete picture of an instance, and avoid unnecessary api calls.
+	//
+	// This will only work with Mastodon-api compatible instances: Mastodon, some Pleroma instances, GoToSocial.
+	l.Debugf("trying to dereference instance %s by /api/v1/instance", iri.Host)
+	i, err = dereferenceByAPIV1Instance(c, t, iri)
+	if err == nil {
+		l.Debugf("successfully dereferenced instance using /api/v1/instance")
+		return i, nil
+	}
+	l.Debugf("couldn't dereference instance using /api/v1/instance: %s", err)
+
+	// If that doesn't work, try to dereference using /.well-known/nodeinfo.
+	// This will involve two API calls and return less info overall, but should be more widely compatible.
+	l.Debugf("trying to dereference instance %s by /.well-known/nodeinfo", iri.Host)
+	i, err = dereferenceByNodeInfo(c, t, iri)
+	if err == nil {
+		l.Debugf("successfully dereferenced instance using /.well-known/nodeinfo")
+		return i, nil
+	}
+	l.Debugf("couldn't dereference instance using /.well-known/nodeinfo: %s", err)
+
+	// we couldn't dereference the instance using any of the known methods, so just return a minimal representation
+	l.Debugf("returning minimal representation of instance %s", iri.Host)
+	id, err := id.NewRandomULID()
+	if err != nil {
+		return nil, fmt.Errorf("error creating new id for instance %s: %s", iri.Host, err)
+	}
+
+	return &gtsmodel.Instance{
+		ID:     id,
+		Domain: iri.Host,
+		URI:    iri.String(),
+	}, nil
+}
+
+func dereferenceByAPIV1Instance(c context.Context, t *transport, iri *url.URL) (*gtsmodel.Instance, error) {
+	l := t.log.WithField("func", "dereferenceByAPIV1Instance")
+
+	cleanIRI := &url.URL{
+		Scheme: iri.Scheme,
+		Host:   iri.Host,
+		Path:   "api/v1/instance",
+	}
+
+	l.Debugf("performing GET to %s", cleanIRI.String())
+	req, err := http.NewRequest("GET", cleanIRI.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(c)
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
+	req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
+	req.Header.Set("Host", cleanIRI.Host)
+	t.getSignerMu.Lock()
+	err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
+	t.getSignerMu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	resp, err := t.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("GET request to %s failed (%d): %s", cleanIRI.String(), resp.StatusCode, resp.Status)
+	}
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(b) == 0 {
+		return nil, errors.New("response bytes was len 0")
+	}
+
+	// try to parse the returned bytes directly into an Instance model
+	apiResp := &apimodel.Instance{}
+	if err := json.Unmarshal(b, apiResp); err != nil {
+		return nil, err
+	}
+
+	var contactUsername string
+	if apiResp.ContactAccount != nil {
+		contactUsername = apiResp.ContactAccount.Username
+	}
+
+	ulid, err := id.NewRandomULID()
+	if err != nil {
+		return nil, err
+	}
+
+	i := &gtsmodel.Instance{
+		ID:                     ulid,
+		Domain:                 iri.Host,
+		Title:                  apiResp.Title,
+		URI:                    fmt.Sprintf("%s://%s", iri.Scheme, iri.Host),
+		ShortDescription:       apiResp.ShortDescription,
+		Description:            apiResp.Description,
+		ContactEmail:           apiResp.Email,
+		ContactAccountUsername: contactUsername,
+		Version:                apiResp.Version,
+	}
+
+	return i, nil
+}
+
+func dereferenceByNodeInfo(c context.Context, t *transport, iri *url.URL) (*gtsmodel.Instance, error) {
+	niIRI, err := callNodeInfoWellKnown(c, t, iri)
+	if err != nil {
+		return nil, fmt.Errorf("dereferenceByNodeInfo: error during initial call to well-known nodeinfo: %s", err)
+	}
+
+	ni, err := callNodeInfo(c, t, niIRI)
+	if err != nil {
+		return nil, fmt.Errorf("dereferenceByNodeInfo: error doing second call to nodeinfo uri %s: %s", niIRI.String(), err)
+	}
+
+	// we got a response of some kind! take what we can from it...
+	id, err := id.NewRandomULID()
+	if err != nil {
+		return nil, fmt.Errorf("dereferenceByNodeInfo: error creating new id for instance %s: %s", iri.Host, err)
+	}
+
+	// this is the bare minimum instance we'll return, and we'll add more stuff to it if we can
+	i := &gtsmodel.Instance{
+		ID:     id,
+		Domain: iri.Host,
+		URI:    iri.String(),
+	}
+
+	var title string
+	if i, present := ni.Metadata["nodeName"]; present {
+		// it's present, check it's a string
+		if v, ok := i.(string); ok {
+			// it is a string!
+			title = v
+		}
+	}
+	i.Title = title
+
+	var shortDescription string
+	if i, present := ni.Metadata["nodeDescription"]; present {
+		// it's present, check it's a string
+		if v, ok := i.(string); ok {
+			// it is a string!
+			shortDescription = v
+		}
+	}
+	i.ShortDescription = shortDescription
+
+	var contactEmail string
+	var contactAccountUsername string
+	if i, present := ni.Metadata["maintainer"]; present {
+		// it's present, check it's a map
+		if v, ok := i.(map[string]string); ok {
+			// see if there's an email in the map
+			if email, present := v["email"]; present {
+				if err := util.ValidateEmail(email); err == nil {
+					// valid email address
+					contactEmail = email
+				}
+			}
+			// see if there's a 'name' in the map
+			if name, present := v["name"]; present {
+				// name could be just a username, or could be a mention string eg @whatever@aaaa.com
+				username, _, err := util.ExtractMentionParts(name)
+				if err == nil {
+					// it was a mention string
+					contactAccountUsername = username
+				} else {
+					// not a mention string
+					contactAccountUsername = name
+				}
+			}
+		}
+	}
+	i.ContactEmail = contactEmail
+	i.ContactAccountUsername = contactAccountUsername
+
+	var software string
+	if ni.Software.Name != "" {
+		software = ni.Software.Name
+	}
+	if ni.Software.Version != "" {
+		software = software + " " + ni.Software.Version
+	}
+	i.Version = software
+
+	return i, nil
+}
+
+func callNodeInfoWellKnown(c context.Context, t *transport, iri *url.URL) (*url.URL, error) {
+	l := t.log.WithField("func", "callNodeInfoWellKnown")
+
+	cleanIRI := &url.URL{
+		Scheme: iri.Scheme,
+		Host:   iri.Host,
+		Path:   ".well-known/nodeinfo",
+	}
+
+	l.Debugf("performing GET to %s", cleanIRI.String())
+	req, err := http.NewRequest("GET", cleanIRI.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(c)
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
+	req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
+	req.Header.Set("Host", cleanIRI.Host)
+	t.getSignerMu.Lock()
+	err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
+	t.getSignerMu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	resp, err := t.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("callNodeInfoWellKnown: GET request to %s failed (%d): %s", cleanIRI.String(), resp.StatusCode, resp.Status)
+	}
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(b) == 0 {
+		return nil, errors.New("callNodeInfoWellKnown: response bytes was len 0")
+	}
+
+	wellKnownResp := &apimodel.WellKnownResponse{}
+	if err := json.Unmarshal(b, wellKnownResp); err != nil {
+		return nil, fmt.Errorf("callNodeInfoWellKnown: could not unmarshal server response as WellKnownResponse: %s", err)
+	}
+
+	// look through the links for the first one that matches the nodeinfo schema, this is what we need
+	var nodeinfoHref *url.URL
+	for _, l := range wellKnownResp.Links {
+		if l.Href == "" || !strings.HasPrefix(l.Rel, "http://nodeinfo.diaspora.software/ns/schema/2") {
+			continue
+		}
+		nodeinfoHref, err = url.Parse(l.Href)
+		if err != nil {
+			return nil, fmt.Errorf("callNodeInfoWellKnown: couldn't parse url %s: %s", l.Href, err)
+		}
+	}
+	if nodeinfoHref == nil {
+		return nil, errors.New("callNodeInfoWellKnown: could not find nodeinfo rel in well known response")
+	}
+
+	return nodeinfoHref, nil
+}
+
+func callNodeInfo(c context.Context, t *transport, iri *url.URL) (*apimodel.Nodeinfo, error) {
+	l := t.log.WithField("func", "callNodeInfo")
+
+	l.Debugf("performing GET to %s", iri.String())
+	req, err := http.NewRequest("GET", iri.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(c)
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
+	req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
+	req.Header.Set("Host", iri.Host)
+	t.getSignerMu.Lock()
+	err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
+	t.getSignerMu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	resp, err := t.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("callNodeInfo: GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
+	}
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(b) == 0 {
+		return nil, errors.New("callNodeInfo: response bytes was len 0")
+	}
+
+	niResp := &apimodel.Nodeinfo{}
+	if err := json.Unmarshal(b, niResp); err != nil {
+		return nil, fmt.Errorf("callNodeInfo: could not unmarshal server response as Nodeinfo: %s", err)
+	}
+
+	return niResp, nil
+}

internal/transport/derefmedia.go

@@ -0,0 +1,42 @@
+package transport
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+)
+
+func (t *transport) DereferenceMedia(c context.Context, iri *url.URL, expectedContentType string) ([]byte, error) {
+	l := t.log.WithField("func", "DereferenceMedia")
+	l.Debugf("performing GET to %s", iri.String())
+	req, err := http.NewRequest("GET", iri.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(c)
+	if expectedContentType == "" {
+		req.Header.Add("Accept", "*/*")
+	} else {
+		req.Header.Add("Accept", expectedContentType)
+	}
+	req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
+	req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
+	req.Header.Set("Host", iri.Host)
+	t.getSignerMu.Lock()
+	err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
+	t.getSignerMu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	resp, err := t.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
+	}
+	return ioutil.ReadAll(resp.Body)
+}

internal/transport/finger.go

@@ -0,0 +1,48 @@
+package transport
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+)
+
+func (t *transport) Finger(c context.Context, targetUsername string, targetDomain string) ([]byte, error) {
+	l := t.log.WithField("func", "Finger")
+	urlString := fmt.Sprintf("https://%s/.well-known/webfinger?resource=acct:%s@%s", targetDomain, targetUsername, targetDomain)
+	l.Debugf("performing GET to %s", urlString)
+
+	iri, err := url.Parse(urlString)
+	if err != nil {
+		return nil, fmt.Errorf("Finger: error parsing url %s: %s", urlString, err)
+	}
+
+	l.Debugf("performing GET to %s", iri.String())
+
+	req, err := http.NewRequest("GET", iri.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(c)
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Accept", "application/jrd+json")
+	req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
+	req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
+	req.Header.Set("Host", iri.Host)
+	t.getSignerMu.Lock()
+	err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
+	t.getSignerMu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	resp, err := t.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
+	}
+	return ioutil.ReadAll(resp.Body)
+}

internal/transport/transport.go

@@ -3,22 +3,23 @@ package transport
 import (
 	"context"
 	"crypto"
-	"fmt"
-	"io/ioutil"
-	"net/http"
 	"net/url"
 	"sync"
 
 	"github.com/go-fed/activity/pub"
 	"github.com/go-fed/httpsig"
 	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
 // Transport wraps the pub.Transport interface with some additional
 // functionality for fetching remote media.
 type Transport interface {
 	pub.Transport
+	// DereferenceMedia fetches the bytes of the given media attachment IRI, with the expectedContentType.
 	DereferenceMedia(c context.Context, iri *url.URL, expectedContentType string) ([]byte, error)
+	// DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo.
+	DereferenceInstance(c context.Context, iri *url.URL) (*gtsmodel.Instance, error)
 	// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
 	Finger(c context.Context, targetUsername string, targetDomains string) ([]byte, error)
 }
@@ -36,91 +37,3 @@ type transport struct {
 	getSignerMu  *sync.Mutex
 	log          *logrus.Logger
 }
-
-func (t *transport) BatchDeliver(c context.Context, b []byte, recipients []*url.URL) error {
-	return t.sigTransport.BatchDeliver(c, b, recipients)
-}
-
-func (t *transport) Deliver(c context.Context, b []byte, to *url.URL) error {
-	l := t.log.WithField("func", "Deliver")
-	l.Debugf("performing POST to %s", to.String())
-	return t.sigTransport.Deliver(c, b, to)
-}
-
-func (t *transport) Dereference(c context.Context, iri *url.URL) ([]byte, error) {
-	l := t.log.WithField("func", "Dereference")
-	l.Debugf("performing GET to %s", iri.String())
-	return t.sigTransport.Dereference(c, iri)
-}
-
-func (t *transport) DereferenceMedia(c context.Context, iri *url.URL, expectedContentType string) ([]byte, error) {
-	l := t.log.WithField("func", "DereferenceMedia")
-	l.Debugf("performing GET to %s", iri.String())
-	req, err := http.NewRequest("GET", iri.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-	req = req.WithContext(c)
-	if expectedContentType == "" {
-		req.Header.Add("Accept", "*/*")
-	} else {
-		req.Header.Add("Accept", expectedContentType)
-	}
-	req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
-	req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
-	req.Header.Set("Host", iri.Host)
-	t.getSignerMu.Lock()
-	err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
-	t.getSignerMu.Unlock()
-	if err != nil {
-		return nil, err
-	}
-	resp, err := t.client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
-	}
-	return ioutil.ReadAll(resp.Body)
-}
-
-func (t *transport) Finger(c context.Context, targetUsername string, targetDomain string) ([]byte, error) {
-	l := t.log.WithField("func", "Finger")
-	urlString := fmt.Sprintf("https://%s/.well-known/webfinger?resource=acct:%s@%s", targetDomain, targetUsername, targetDomain)
-	l.Debugf("performing GET to %s", urlString)
-
-	iri, err := url.Parse(urlString)
-	if err != nil {
-		return nil, fmt.Errorf("Finger: error parsing url %s: %s", urlString, err)
-	}
-
-	l.Debugf("performing GET to %s", iri.String())
-
-	req, err := http.NewRequest("GET", iri.String(), nil)
-	if err != nil {
-		return nil, err
-	}
-	req = req.WithContext(c)
-	req.Header.Add("Accept", "application/json")
-	req.Header.Add("Accept", "application/jrd+json")
-	req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT")
-	req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent))
-	req.Header.Set("Host", iri.Host)
-	t.getSignerMu.Lock()
-	err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil)
-	t.getSignerMu.Unlock()
-	if err != nil {
-		return nil, err
-	}
-	resp, err := t.client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status)
-	}
-	return ioutil.ReadAll(resp.Body)
-}

internal/util/regexes.go

@@ -61,7 +61,7 @@ var (
 	userPathRegex = regexp.MustCompile(userPathRegexString)
 
 	userPublicKeyPathRegexString = fmt.Sprintf(`^?/%s/(%s)/%s`, UsersPath, usernameRegexString, PublicKeyPath)
-	userPublicKeyPathRegex = regexp.MustCompile(userPublicKeyPathRegexString)
+	userPublicKeyPathRegex       = regexp.MustCompile(userPublicKeyPathRegexString)
 
 	inboxPathRegexString = fmt.Sprintf(`^/?%s/(%s)/%s$`, UsersPath, usernameRegexString, InboxPath)
 	// inboxPathRegex parses a path that validates and captures the username part from eg /users/example_username/inbox