Drop reference_sha column (matrix-org#3083)

Companion PR to matrix-org/gomatrixserverlib#383

.gitignore

@@ -74,4 +74,7 @@ complement/
 docs/_site
 
 media_store/
-build
+build
+
+# golang workspaces
+go.work*

build/gobind-pinecone/monolith_test.go

@@ -22,7 +22,10 @@ import (
 )
 
 func TestMonolithStarts(t *testing.T) {
-	monolith := DendriteMonolith{}
+	monolith := DendriteMonolith{
+		StorageDirectory: t.TempDir(),
+		CacheDirectory:   t.TempDir(),
+	}
 	monolith.Start()
 	monolith.PublicKey()
 	monolith.Stop()
@@ -60,7 +63,10 @@ func TestMonolithSetRelayServers(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		monolith := DendriteMonolith{}
+		monolith := DendriteMonolith{
+			StorageDirectory: t.TempDir(),
+			CacheDirectory:   t.TempDir(),
+		}
 		monolith.Start()
 
 		inputRelays := tc.relays

clientapi/routing/createroom.go

@@ -470,7 +470,7 @@ func createRoom(
 			}
 		}
 		if i > 0 {
-			builder.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()}
+			builder.PrevEvents = []string{builtEvents[i-1].EventID()}
 		}
 		var ev gomatrixserverlib.PDU
 		if err = builder.AddAuthEvents(&authEvents); err != nil {

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/matrix-org/dugong v0.0.0-20210921133753-66e6b1c67e2e
 	github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91
 	github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530
-	github.com/matrix-org/gomatrixserverlib v0.0.0-20230523164045-3fddabebb511
+	github.com/matrix-org/gomatrixserverlib v0.0.0-20230524095531-95ba6c68efb6
 	github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a
 	github.com/matrix-org/util v0.0.0-20221111132719-399730281e66
 	github.com/mattn/go-sqlite3 v1.14.16

go.sum

@@ -323,8 +323,8 @@ github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91 h1:s7fexw
 github.com/matrix-org/go-sqlite3-js v0.0.0-20220419092513-28aa791a1c91/go.mod h1:e+cg2q7C7yE5QnAXgzo512tgFh1RbQLC0+jozuegKgo=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530 h1:kHKxCOLcHH8r4Fzarl4+Y3K5hjothkVW5z7T1dUM11U=
 github.com/matrix-org/gomatrix v0.0.0-20220926102614-ceba4d9f7530/go.mod h1:/gBX06Kw0exX1HrwmoBibFA98yBk/jxKpGVeyQbff+s=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230523164045-3fddabebb511 h1:om6z/WEVZMxZfgtiyfp5r5ubAObGMyRrnlVD07gIRY4=
-github.com/matrix-org/gomatrixserverlib v0.0.0-20230523164045-3fddabebb511/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230524095531-95ba6c68efb6 h1:FQpdh/KGCCQJytz4GAdG6pbx3DJ1HNzdKFc/BCZ0hP0=
+github.com/matrix-org/gomatrixserverlib v0.0.0-20230524095531-95ba6c68efb6/go.mod h1:H9V9N3Uqn1bBJqYJNGK1noqtgJTaCEhtTdcH/mp50uU=
 github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a h1:awrPDf9LEFySxTLKYBMCiObelNx/cBuv/wzllvCCH3A=
 github.com/matrix-org/pinecone v0.11.1-0.20230210171230-8c3b24f2649a/go.mod h1:HchJX9oKMXaT2xYFs0Ha/6Zs06mxLU8k6F1ODnrGkeQ=
 github.com/matrix-org/util v0.0.0-20221111132719-399730281e66 h1:6z4KxomXSIGWqhHcfzExgkH3Z3UkIXry4ibJS4Aqz2Y=

internal/eventutil/events.go

@@ -129,18 +129,12 @@ func addPrevEventsToEvent(
 		return ErrRoomNoExists{}
 	}
 
-	verImpl, err := gomatrixserverlib.GetRoomVersion(queryRes.RoomVersion)
-	if err != nil {
-		return fmt.Errorf("GetRoomVersion: %w", err)
-	}
-	eventFormat := verImpl.EventFormat()
-
 	builder.Depth = queryRes.Depth
 
 	authEvents := gomatrixserverlib.NewAuthEvents(nil)
 
 	for i := range queryRes.StateEvents {
-		err = authEvents.AddEvent(queryRes.StateEvents[i].PDU)
+		err := authEvents.AddEvent(queryRes.StateEvents[i].PDU)
 		if err != nil {
 			return fmt.Errorf("authEvents.AddEvent: %w", err)
 		}
@@ -151,22 +145,7 @@ func addPrevEventsToEvent(
 		return fmt.Errorf("eventsNeeded.AuthEventReferences: %w", err)
 	}
 
-	truncAuth, truncPrev := truncateAuthAndPrevEvents(refs, queryRes.LatestEvents)
-	switch eventFormat {
-	case gomatrixserverlib.EventFormatV1:
-		builder.AuthEvents = truncAuth
-		builder.PrevEvents = truncPrev
-	case gomatrixserverlib.EventFormatV2:
-		v2AuthRefs, v2PrevRefs := []string{}, []string{}
-		for _, ref := range truncAuth {
-			v2AuthRefs = append(v2AuthRefs, ref.EventID)
-		}
-		for _, ref := range truncPrev {
-			v2PrevRefs = append(v2PrevRefs, ref.EventID)
-		}
-		builder.AuthEvents = v2AuthRefs
-		builder.PrevEvents = v2PrevRefs
-	}
+	builder.AuthEvents, builder.PrevEvents = truncateAuthAndPrevEvents(refs, queryRes.LatestEvents)
 
 	return nil
 }
@@ -176,8 +155,8 @@ func addPrevEventsToEvent(
 // NOTSPEC: The limits here feel a bit arbitrary but they are currently
 // here because of https://github.com/matrix-org/matrix-doc/issues/2307
 // and because Synapse will just drop events that don't comply.
-func truncateAuthAndPrevEvents(auth, prev []gomatrixserverlib.EventReference) (
-	truncAuth, truncPrev []gomatrixserverlib.EventReference,
+func truncateAuthAndPrevEvents(auth, prev []string) (
+	truncAuth, truncPrev []string,
 ) {
 	truncAuth, truncPrev = auth, prev
 	if len(truncAuth) > 10 {

roomserver/api/query.go

@@ -49,7 +49,7 @@ type QueryLatestEventsAndStateResponse struct {
 	RoomVersion gomatrixserverlib.RoomVersion `json:"room_version"`
 	// The latest events in the room.
 	// These are used to set the prev_events when sending an event.
-	LatestEvents []gomatrixserverlib.EventReference `json:"latest_events"`
+	LatestEvents []string `json:"latest_events"`
 	// The state events requested.
 	// This list will be in an arbitrary order.
 	// These are used to set the auth_events when sending an event.

roomserver/internal/input/input_events.go

@@ -883,9 +883,7 @@ func (r *Inputer) kickGuests(ctx context.Context, event gomatrixserverlib.PDU, r
 			Origin:       senderDomain,
 			SendAsServer: string(senderDomain),
 		})
-		prevEvents = []gomatrixserverlib.EventReference{
-			event.EventReference(),
-		}
+		prevEvents = []string{event.EventID()}
 	}
 
 	inputReq := &api.InputRoomEventsRequest{

roomserver/internal/input/input_latest_events.go

@@ -154,8 +154,8 @@ func (u *latestEventsUpdater) doUpdateLatestEvents() error {
 	extremitiesChanged, err := u.calculateLatest(
 		u.oldLatest, u.event,
 		types.StateAtEventAndReference{
-			EventReference: u.event.EventReference(),
-			StateAtEvent:   u.stateAtEvent,
+			EventID:      u.event.EventID(),
+			StateAtEvent: u.stateAtEvent,
 		},
 	)
 	if err != nil {
@@ -349,7 +349,7 @@ func (u *latestEventsUpdater) calculateLatest(
 	// If the "new" event is already referenced by an existing event
 	// then do nothing - it's not a candidate to be a new extremity if
 	// it has been referenced.
-	if referenced, err := u.updater.IsReferenced(newEvent.EventReference()); err != nil {
+	if referenced, err := u.updater.IsReferenced(newEvent.EventID()); err != nil {
 		return false, fmt.Errorf("u.updater.IsReferenced(new): %w", err)
 	} else if referenced {
 		u.latest = oldLatest
@@ -360,7 +360,7 @@ func (u *latestEventsUpdater) calculateLatest(
 	// have entries in the previous events table. If they do then we
 	// will no longer include them as forward extremities.
 	for k, l := range existingRefs {
-		referenced, err := u.updater.IsReferenced(l.EventReference)
+		referenced, err := u.updater.IsReferenced(l.EventID)
 		if err != nil {
 			return false, fmt.Errorf("u.updater.IsReferenced: %w", err)
 		} else if referenced {

roomserver/internal/input/input_missing.go

@@ -520,9 +520,9 @@ func (t *missingStateReq) getMissingEvents(ctx context.Context, e gomatrixserver
 		return nil, false, false, fmt.Errorf("t.DB.LatestEventIDs: %w", err)
 	}
 	latestEvents := make([]string, len(latest))
-	for i, ev := range latest {
-		latestEvents[i] = ev.EventID
-		t.hadEvent(ev.EventID)
+	for i := range latest {
+		latestEvents[i] = latest[i]
+		t.hadEvent(latest[i])
 	}
 
 	var missingResp *fclient.RespMissingEvents

roomserver/internal/perform/perform_admin.go

@@ -131,9 +131,7 @@ func (r *Admin) PerformAdminEvacuateRoom(
 			SendAsServer: string(senderDomain),
 		})
 		affected = append(affected, stateKey)
-		prevEvents = []gomatrixserverlib.EventReference{
-			event.EventReference(),
-		}
+		prevEvents = []string{event.EventID()}
 	}
 
 	inputReq := &api.InputRoomEventsRequest{
@@ -253,9 +251,9 @@ func (r *Admin) PerformAdminDownloadState(
 
 	for _, fwdExtremity := range fwdExtremities {
 		var state gomatrixserverlib.StateResponse
-		state, err = r.Inputer.FSAPI.LookupState(ctx, r.Inputer.ServerName, serverName, roomID, fwdExtremity.EventID, roomInfo.RoomVersion)
+		state, err = r.Inputer.FSAPI.LookupState(ctx, r.Inputer.ServerName, serverName, roomID, fwdExtremity, roomInfo.RoomVersion)
 		if err != nil {
-			return fmt.Errorf("r.Inputer.FSAPI.LookupState (%q): %s", fwdExtremity.EventID, err)
+			return fmt.Errorf("r.Inputer.FSAPI.LookupState (%q): %s", fwdExtremity, err)
 		}
 		for _, authEvent := range state.GetAuthEvents().UntrustedEvents(roomInfo.RoomVersion) {
 			if err = gomatrixserverlib.VerifyEventSignatures(ctx, authEvent, r.Inputer.KeyRing); err != nil {

roomserver/internal/perform/perform_inbound_peek.go

@@ -64,7 +64,7 @@ func (r *InboundPeeker) PerformInboundPeek(
 	if err != nil {
 		return err
 	}
-	latestEvents, err := r.DB.EventsFromIDs(ctx, info, []string{latestEventRefs[0].EventID})
+	latestEvents, err := r.DB.EventsFromIDs(ctx, info, []string{latestEventRefs[0]})
 	if err != nil {
 		return err
 	}

roomserver/internal/perform/perform_upgrade.go

@@ -471,7 +471,7 @@ func (r *Upgrader) sendInitialEvents(ctx context.Context, evTime time.Time, user
 			return fmt.Errorf("failed to set content of new %q event: %w", proto.Type, err)
 		}
 		if i > 0 {
-			proto.PrevEvents = []gomatrixserverlib.EventReference{builtEvents[i-1].EventReference()}
+			proto.PrevEvents = []string{builtEvents[i-1].EventID()}
 		}
 
 		var verImpl gomatrixserverlib.IRoomVersion

roomserver/internal/query/query_test.go

@@ -43,13 +43,10 @@ func createEventDB() *getEventDB {
 
 // Adds a fake event to the storage with given auth events.
 func (db *getEventDB) addFakeEvent(eventID string, authIDs []string) error {
-	authEvents := []gomatrixserverlib.EventReference{}
+	authEvents := make([]any, 0, len(authIDs))
 	for _, authID := range authIDs {
-		authEvents = append(authEvents, gomatrixserverlib.EventReference{
-			EventID: authID,
-		})
+		authEvents = append(authEvents, []any{authID, struct{}{}})
 	}
-
 	builder := map[string]interface{}{
 		"event_id":    eventID,
 		"auth_events": authEvents,

roomserver/storage/interface.go

@@ -102,7 +102,7 @@ type Database interface {
 	// Look up event references for the latest events in the room and the current state snapshot.
 	// Returns the latest events, the current state and the maximum depth of the latest events plus 1.
 	// Returns an error if there was a problem talking to the database.
-	LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error)
+	LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]string, types.StateSnapshotNID, int64, error)
 	// Look up the active invites targeting a user in a room and return the
 	// numeric state key IDs for the user IDs who sent them along with the event IDs for the invites.
 	// Returns an error if there was a problem talking to the database.
@@ -206,7 +206,7 @@ type RoomDatabase interface {
 	BulkSelectSnapshotsFromEventIDs(ctx context.Context, eventIDs []string) (map[types.StateSnapshotNID][]string, error)
 	StateEntriesForTuples(ctx context.Context, stateBlockNIDs []types.StateBlockNID, stateKeyTuples []types.StateKeyTuple) ([]types.StateEntryList, error)
 	AddState(ctx context.Context, roomNID types.RoomNID, stateBlockNIDs []types.StateBlockNID, state []types.StateEntry) (types.StateSnapshotNID, error)
-	LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]gomatrixserverlib.EventReference, types.StateSnapshotNID, int64, error)
+	LatestEventIDs(ctx context.Context, roomNID types.RoomNID) ([]string, types.StateSnapshotNID, int64, error)
 	GetOrCreateRoomInfo(ctx context.Context, event gomatrixserverlib.PDU) (*types.RoomInfo, error)
 	GetOrCreateEventTypeNID(ctx context.Context, eventType string) (eventTypeNID types.EventTypeNID, err error)
 	GetOrCreateEventStateKeyNID(ctx context.Context, eventStateKey *string) (types.EventStateKeyNID, error)

roomserver/storage/postgres/deltas/20230516154000_drop_reference_sha.go

@@ -0,0 +1,54 @@
+// Copyright 2023 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deltas
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+)
+
+func UpDropEventReferenceSHAEvents(ctx context.Context, tx *sql.Tx) error {
+	var count int
+	err := tx.QueryRowContext(ctx, `SELECT count(*) FROM roomserver_events GROUP BY event_id HAVING count(event_id) > 1`).
+		Scan(&count)
+	if err != nil && err != sql.ErrNoRows {
+		return fmt.Errorf("failed to query duplicate event ids")
+	}
+	if count > 0 {
+		return fmt.Errorf("unable to drop column, as there are duplicate event ids")
+	}
+	_, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_events DROP COLUMN IF EXISTS reference_sha256;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}
+
+func UpDropEventReferenceSHAPrevEvents(ctx context.Context, tx *sql.Tx) error {
+	_, err := tx.ExecContext(ctx, "ALTER TABLE roomserver_previous_events DROP CONSTRAINT roomserver_previous_event_id_unique;")
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	_, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_previous_events DROP COLUMN IF EXISTS previous_reference_sha256;`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	_, err = tx.ExecContext(ctx, `ALTER TABLE roomserver_previous_events ADD CONSTRAINT roomserver_previous_event_id_unique UNIQUE (previous_event_id);`)
+	if err != nil {
+		return fmt.Errorf("failed to execute upgrade: %w", err)
+	}
+	return nil
+}