[performance] cache more database calls, reduce required database cal…

…ls overall (#3290)

* improvements to caching for lists and relationship to accounts / follows

* fix nil panic in AddToList()

* ensure list related caches are correctly invalidated

* ensure returned ID lists are ordered correctly

* bump go-structr to v0.8.9 (returns early if zero uncached keys to be loaded)

* remove zero checks in uncached key load functions (go-structr now handles this)

* fix issues after rebase on upstream/main

* update the expected return order of CSV exports (since list entries are now down by entry creation date)

* rename some funcs, allow deleting list entries for multiple follow IDs at a time, fix up more tests

* use returning statements on delete to get cache invalidation info

* fixes to recent database delete changes

* fix broken list entries delete sql

* remove unused db function

* update remainder of delete functions to behave in similar way, some other small tweaks

* fix delete user sql, allow returning on err no entries

* uncomment + fix list database tests

* update remaining list tests

* update envparsing test

* add comments to each specific key being invalidated

* add more cache invalidation explanatory comments

* whoops; actually delete poll votes from database in the DeletePollByID() func

* remove added but-commented-out field

* improved comment regarding paging being disabled

* make cache invalidation comments match what's actually happening

* fix up delete query comments to match what is happening

* rename function to read a bit better

* don't use ErrNoEntries on delete when not needed (it's only needed for a RETURNING call)

* update function name in test

* move list exclusivity check to AFTER eligibility check. use log.Panic() instead of panic()

* use the poll_id column in poll_votes for selecting votes in poll ID

* fix function name

go.mod

@@ -23,7 +23,7 @@ require (
 	codeberg.org/gruf/go-runners v1.6.2
 	codeberg.org/gruf/go-sched v1.2.3
 	codeberg.org/gruf/go-storage v0.1.2
-	codeberg.org/gruf/go-structr v0.8.8
+	codeberg.org/gruf/go-structr v0.8.9
 	codeberg.org/superseriousbusiness/exif-terminator v0.9.0
 	github.com/DmitriyVTitov/size v1.5.0
 	github.com/KimMachineGun/automemlimit v0.6.1

go.sum

@@ -80,8 +80,8 @@ codeberg.org/gruf/go-sched v1.2.3 h1:H5ViDxxzOBR3uIyGBCf0eH8b1L8wMybOXcdtUUTXZHk
 codeberg.org/gruf/go-sched v1.2.3/go.mod h1:vT9uB6KWFIIwnG9vcPY2a0alYNoqdL1mSzRM8I+PK7A=
 codeberg.org/gruf/go-storage v0.1.2 h1:dIOVOKq1CJpRmuhbB8Zok3mmo8V6VV/nX5GLIm6hywA=
 codeberg.org/gruf/go-storage v0.1.2/go.mod h1:LRDpFHqRJi0f+35c3ltBH2e/pGfwY5dGlNlgCJ/R1DA=
-codeberg.org/gruf/go-structr v0.8.8 h1:lRPpyTmLKvQCkkQiSUbOAh6jtL2wncEO8DwksMqQXM8=
-codeberg.org/gruf/go-structr v0.8.8/go.mod h1:zkoXVrAnKosh8VFAsbP/Hhs8FmLBjbVVy5w/Ngm8ApM=
+codeberg.org/gruf/go-structr v0.8.9 h1:OyiSspWYCeJOm356fFPd+bDRumPrard2VAUXAPqZiJ0=
+codeberg.org/gruf/go-structr v0.8.9/go.mod h1:zkoXVrAnKosh8VFAsbP/Hhs8FmLBjbVVy5w/Ngm8ApM=
 codeberg.org/superseriousbusiness/exif-terminator v0.9.0 h1:/EfyGI6HIrbkhFwgXGSjZ9o1kr/+k8v4mKdfXTH02Go=
 codeberg.org/superseriousbusiness/exif-terminator v0.9.0/go.mod h1:gCWKduudUWFzsnixoMzu0FYVdxHWG+AbXnZ50DqxsUE=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=

internal/api/client/exports/exports_test.go

@@ -188,8 +188,8 @@ admin@localhost:8080
 			token:       suite.testTokens["local_account_1"],
 			user:        suite.testUsers["local_account_1"],
 			account:     suite.testAccounts["local_account_1"],
-			expect: `Cool Ass Posters From This Instance,admin@localhost:8080
-Cool Ass Posters From This Instance,1happyturtle@localhost:8080
+			expect: `Cool Ass Posters From This Instance,1happyturtle@localhost:8080
+Cool Ass Posters From This Instance,admin@localhost:8080
 `,
 		},
 		// Export Mutes.

internal/api/client/lists/listaccounts.go

@@ -25,6 +25,7 @@ import (
 	apiutil "github.com/superseriousbusiness/gotosocial/internal/api/util"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
+	"github.com/superseriousbusiness/gotosocial/internal/paging"
 )
 
 // ListAccountsGETHandler swagger:operation GET /api/v1/lists/{id}/accounts listAccounts
@@ -129,42 +130,27 @@ func (m *Module) ListAccountsGETHandler(c *gin.Context) {
 
 	targetListID := c.Param(IDKey)
 	if targetListID == "" {
-		err := errors.New("no list id specified")
-		apiutil.ErrorHandler(c, gtserror.NewErrorBadRequest(err, err.Error()), m.processor.InstanceGetV1)
-		return
-	}
-
-	limit, errWithCode := apiutil.ParseLimit(c.Query(apiutil.LimitKey), 40, 80, 0)
-	if errWithCode != nil {
+		const text = "no list id specified"
+		errWithCode := gtserror.NewErrorBadRequest(errors.New(text), text)
 		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
 		return
 	}
 
-	var (
-		ctx = c.Request.Context()
+	page, errWithCode := paging.ParseIDPage(c,
+		1,  // min limit
+		80, // max limit
+		0,  // default = paging disabled
 	)
-
-	if limit == 0 {
-		// Return all accounts in the list without pagination.
-		accounts, errWithCode := m.processor.List().GetAllListAccounts(ctx, authed.Account, targetListID)
-		if errWithCode != nil {
-			apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
-			return
-		}
-
-		c.JSON(http.StatusOK, accounts)
+	if errWithCode != nil {
+		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)
 		return
 	}
 
-	// Return subset of accounts in the list with pagination.
 	resp, errWithCode := m.processor.List().GetListAccounts(
-		ctx,
+		c.Request.Context(),
 		authed.Account,
 		targetListID,
-		c.Query(MaxIDKey),
-		c.Query(SinceIDKey),
-		c.Query(MinIDKey),
-		limit,
+		page,
 	)
 	if errWithCode != nil {
 		apiutil.ErrorHandler(c, errWithCode, m.processor.InstanceGetV1)

internal/api/client/lists/listaccounts_test.go

@@ -19,7 +19,7 @@ package lists_test
 
 import (
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
@@ -97,7 +97,7 @@ func (suite *ListAccountsTestSuite) getListAccounts(
 	result := recorder.Result()
 	defer result.Body.Close()
 
-	b, err := ioutil.ReadAll(result.Body)
+	b, err := io.ReadAll(result.Body)
 	if err != nil {
 		return nil, "", err
 	}
@@ -151,8 +151,7 @@ func (suite *ListAccountsTestSuite) TestGetListAccountsPaginatedDefaultLimit() {
 
 	suite.Len(accounts, 2)
 	suite.Equal(
-		`<http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=40&max_id=01H0G89MWVQE0M58VD2HQYMQWH>; rel="next", `+
-			`<http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=40&min_id=01H0G8FFM1AGQDRNGBGGX8CYJQ>; rel="prev"`,
+		"<http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=40&max_id=01F8MH5NBDF2MV7CTC4Q5128HF>; rel=\"next\", <http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=40&min_id=01F8MH17FWEB39HZJ76B6VXSKF>; rel=\"prev\"",
 		link,
 	)
 }
@@ -184,8 +183,7 @@ func (suite *ListAccountsTestSuite) TestGetListAccountsPaginatedNextPage() {
 
 	suite.Len(accounts, 1)
 	suite.Equal(
-		`<http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=1&max_id=01H0G8FFM1AGQDRNGBGGX8CYJQ>; rel="next", `+
-			`<http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=1&min_id=01H0G8FFM1AGQDRNGBGGX8CYJQ>; rel="prev"`,
+		"<http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=1&max_id=01F8MH17FWEB39HZJ76B6VXSKF>; rel=\"next\", <http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=1&min_id=01F8MH17FWEB39HZJ76B6VXSKF>; rel=\"prev\"",
 		link,
 	)
 
@@ -206,8 +204,7 @@ func (suite *ListAccountsTestSuite) TestGetListAccountsPaginatedNextPage() {
 
 	suite.Len(accounts, 1)
 	suite.Equal(
-		`<http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=1&max_id=01H0G89MWVQE0M58VD2HQYMQWH>; rel="next", `+
-			`<http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=1&min_id=01H0G89MWVQE0M58VD2HQYMQWH>; rel="prev"`,
+		"<http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=1&max_id=01F8MH17FWEB39HZJ76B6VXSKF>; rel=\"next\", <http://localhost:8080/api/v1/lists/01H0G8E4Q2J3FE3JDWJVWEDCD1/accounts?limit=1&min_id=01F8MH17FWEB39HZJ76B6VXSKF>; rel=\"prev\"",
 		link,
 	)
 }

internal/api/client/lists/listaccountsadd_test.go

@@ -98,14 +98,17 @@ func (suite *ListAccountsAddTestSuite) TestPostListAccountNotFollowed() {
 
 	resp, err := suite.postListAccounts(http.StatusNotFound, listID, accountIDs)
 	suite.NoError(err)
-	suite.Equal(`{"error":"Not Found: you do not follow account 01F8MH5ZK5VRH73AKHQM6Y9VNX"}`, string(resp))
+	suite.Equal(`{"error":"Not Found: account 01F8MH5ZK5VRH73AKHQM6Y9VNX not currently followed"}`, string(resp))
 }
 
 func (suite *ListAccountsAddTestSuite) TestPostListAccountOK() {
+	entry := suite.testListEntries["local_account_1_list_1_entry_1"]
+
 	// Remove turtle from the list.
 	if err := suite.db.DeleteListEntry(
 		context.Background(),
-		suite.testListEntries["local_account_1_list_1_entry_1"].ID,
+		entry.ListID,
+		entry.FollowID,
 	); err != nil {
 		suite.FailNow(err.Error())
 	}

internal/cache/cache.go

@@ -62,7 +62,6 @@ func (c *Caches) Init() {
 	log.Infof(nil, "init: %p", c)
 
 	c.initAccount()
-	c.initAccountIDsFollowingTag()
 	c.initAccountNote()
 	c.initAccountSettings()
 	c.initAccountStats()
@@ -84,11 +83,13 @@ func (c *Caches) Init() {
 	c.initFollowIDs()
 	c.initFollowRequest()
 	c.initFollowRequestIDs()
+	c.initFollowingTagIDs()
 	c.initInReplyToIDs()
 	c.initInstance()
 	c.initInteractionRequest()
 	c.initList()
-	c.initListEntry()
+	c.initListIDs()
+	c.initListedIDs()
 	c.initMarker()
 	c.initMedia()
 	c.initMention()
@@ -105,7 +106,6 @@ func (c *Caches) Init() {
 	c.initStatusFave()
 	c.initStatusFaveIDs()
 	c.initTag()
-	c.initTagIDsFollowedByAccount()
 	c.initThreadMute()
 	c.initToken()
 	c.initTombstone()
@@ -148,7 +148,6 @@ func (c *Caches) Stop() {
 // significant overhead to all cache writes.
 func (c *Caches) Sweep(threshold float64) {
 	c.DB.Account.Trim(threshold)
-	c.DB.AccountIDsFollowingTag.Trim(threshold)
 	c.DB.AccountNote.Trim(threshold)
 	c.DB.AccountSettings.Trim(threshold)
 	c.DB.AccountStats.Trim(threshold)
@@ -168,11 +167,13 @@ func (c *Caches) Sweep(threshold float64) {
 	c.DB.FollowIDs.Trim(threshold)
 	c.DB.FollowRequest.Trim(threshold)
 	c.DB.FollowRequestIDs.Trim(threshold)
+	c.DB.FollowingTagIDs.Trim(threshold)
 	c.DB.InReplyToIDs.Trim(threshold)
 	c.DB.Instance.Trim(threshold)
 	c.DB.InteractionRequest.Trim(threshold)
 	c.DB.List.Trim(threshold)
-	c.DB.ListEntry.Trim(threshold)
+	c.DB.ListIDs.Trim(threshold)
+	c.DB.ListedIDs.Trim(threshold)
 	c.DB.Marker.Trim(threshold)
 	c.DB.Media.Trim(threshold)
 	c.DB.Mention.Trim(threshold)
@@ -189,7 +190,6 @@ func (c *Caches) Sweep(threshold float64) {
 	c.DB.StatusFave.Trim(threshold)
 	c.DB.StatusFaveIDs.Trim(threshold)
 	c.DB.Tag.Trim(threshold)
-	c.DB.TagIDsFollowedByAccount.Trim(threshold)
 	c.DB.ThreadMute.Trim(threshold)
 	c.DB.Token.Trim(threshold)
 	c.DB.Tombstone.Trim(threshold)