Support multiple versions of rooms

src/d2m/actions/announce-thread.js

@@ -1,6 +1,6 @@
 // @ts-check
 
-const assert = require("assert")
+const assert = require("assert").strict
 
 const passthrough = require("../../passthrough")
 const {discord, sync, db, select} = passthrough

src/d2m/actions/create-room.js

@@ -228,7 +228,10 @@ async function createRoom(channel, guild, spaceID, kstate, privacyLevel) {
 			...spaceCreationContent
 		})
 
-		db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent)
+		db.transaction(() => {
+			db.prepare("INSERT INTO channel_room (channel_id, room_id, name, nick, thread_parent) VALUES (?, ?, ?, NULL, ?)").run(channel.id, roomID, channel.name, threadParent)
+			db.prepare("INSERT INTO historical_channel_room (channel_id, room_id) VALUES (?, ?)").run(channel.id, roomID)
+		})()
 
 		return roomID
 	})
@@ -400,7 +403,7 @@ async function _syncRoom(channelID, shouldActuallySync) {
 	}
 	const roomDiff = ks.diffKState(roomKState, channelKState)
 	const roomApply = ks.applyKStateDiffToRoom(roomID, roomDiff)
-	db.prepare("UPDATE channel_room SET name = ? WHERE room_id = ?").run(channel.name, roomID)
+	db.prepare("UPDATE channel_room SET name = ? WHERE channel_id = ?").run(channel.name, channel.id)
 
 	// sync room as space member
 	const spaceApply = _syncSpaceMember(channel, spaceID, roomID, guild.id)
@@ -464,7 +467,7 @@ async function unbridgeDeletedChannel(channel, guildID) {
 
 	// delete room from database
 	db.prepare("DELETE FROM member_cache WHERE room_id = ?").run(roomID)
-	db.prepare("DELETE FROM channel_room WHERE room_id = ? AND channel_id = ?").run(roomID, channel.id) // cascades to most other tables, like messages
+	db.prepare("DELETE FROM channel_room WHERE room_id = ? AND channel_id = ?").run(roomID, channel.id) // cascades to most other tables, like messages and historical rooms
 
 	if (!botInRoom) return
 

src/d2m/actions/delete-message.js

@@ -14,11 +14,13 @@ async function deleteMessage(data) {
 	const row = select("channel_room", ["room_id", "speedbump_checked", "thread_parent"], {channel_id: data.channel_id}).get()
 	if (!row) return
 
-	const eventsToRedact = select("event_message", "event_id", {message_id: data.id}).pluck().all()
-	db.prepare("DELETE FROM message_channel WHERE message_id = ?").run(data.id)
-	for (const eventID of eventsToRedact) {
+	// Assume we can redact from tombstoned rooms.
+	const eventsToRedact = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index")
+		.select("event_id", "room_id").where({message_id: data.id}).all()
+	db.prepare("DELETE FROM message_room WHERE message_id = ?").run(data.id)
+	for (const {event_id, room_id} of eventsToRedact) {
 		// Unfortunately, we can't specify a sender to do the redaction as, unless we find out that info via the audit logs
-		await api.redactEvent(row.room_id, eventID)
+		await api.redactEvent(room_id, event_id)
 	}
 
 	await speedbump.updateCache(row.thread_parent || data.channel_id, row.speedbump_checked)
@@ -28,15 +30,17 @@ async function deleteMessage(data) {
  * @param {import("discord-api-types/v10").GatewayMessageDeleteBulkDispatchData} data
  */
 async function deleteMessageBulk(data) {
-	const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
-	if (!roomID) return
+	const row = select("channel_room", "room_id", {channel_id: data.channel_id}).get()
+	if (!row) return
 
 	const sids = JSON.stringify(data.ids)
-	const eventsToRedact = from("event_message").pluck("event_id").and("WHERE message_id IN (SELECT value FROM json_each(?))").all(sids)
-	db.prepare("DELETE FROM message_channel WHERE message_id IN (SELECT value FROM json_each(?))").run(sids)
-	for (const eventID of eventsToRedact) {
+	// Assume we can redact from tombstoned rooms.
+	const eventsToRedact = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index")
+		.select("event_id", "room_id").and("WHERE message_id IN (SELECT value FROM json_each(?))").all(sids)
+	db.prepare("DELETE FROM message_room WHERE message_id IN (SELECT value FROM json_each(?))").run(sids)
+	for (const {event_id, room_id} of eventsToRedact) {
 		// Awaiting will make it go slower, but since this could be a long-running operation either way, we want to leave rate limit capacity for other operations
-		await api.redactEvent(roomID, eventID)
+		await api.redactEvent(room_id, event_id)
 	}
 }
 

src/d2m/actions/edit-message.js

@@ -3,7 +3,7 @@
 const assert = require("assert").strict
 
 const passthrough = require("../../passthrough")
-const {sync, db, select} = passthrough
+const {sync, db, select, from} = passthrough
 /** @type {import("../converters/edit-to-changes")} */
 const editToChanges = sync.require("../converters/edit-to-changes")
 /** @type {import("./register-pk-user")} */
@@ -19,6 +19,12 @@ const mreq = sync.require("../../matrix/mreq")
  * @param {{speedbump_id: string, speedbump_webhook_id: string} | null} row data about the webhook which is proxying messages in this channel
  */
 async function editMessage(message, guild, row) {
+	const historicalRoomOfMessage = from("message_room").join("historical_channel_room", "historical_room_index").where({message_id: message.id}).select("room_id").get()
+	const currentRoom = from("channel_room").join("historical_channel_room", "room_id").where({channel_id: message.channel_id}).select("room_id", "historical_room_index").get()
+	assert(currentRoom)
+
+	if (historicalRoomOfMessage && historicalRoomOfMessage.room_id !== currentRoom.room_id) return // tombstoned rooms should not have new events (including edits) sent to them
+
 	let {roomID, eventsToRedact, eventsToReplace, eventsToSend, senderMxid, promotions} = await editToChanges.editToChanges(message, guild, api)
 
 	if (row && row.speedbump_webhook_id === message.webhook_id) {
@@ -61,7 +67,7 @@ async function editMessage(message, guild, row) {
 
 	// 4. Send all the things.
 	if (eventsToSend.length) {
-		db.prepare("INSERT OR IGNORE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id)
+		db.prepare("INSERT OR IGNORE INTO message_room (message_id, historical_room_index) VALUES (?, ?)").run(message.id, currentRoom.historical_room_index)
 	}
 	for (const content of eventsToSend) {
 		const eventType = content.$type

src/d2m/actions/register-user.js

@@ -4,10 +4,9 @@ const assert = require("assert").strict
 const {reg} = require("../../matrix/read-registration")
 const DiscordTypes = require("discord-api-types/v10")
 const Ty = require("../../types")
-const mixin = require("@cloudrac3r/mixin-deep")
 
 const passthrough = require("../../passthrough")
-const {discord, sync, db, select} = passthrough
+const {discord, sync, db, from, select} = passthrough
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
 /** @type {import("../../matrix/file")} */
@@ -222,7 +221,8 @@ async function syncUser(user, member, channel, guild, roomID) {
  * @param {string} roomID
  */
 async function syncAllUsersInRoom(roomID) {
-	const mxids = select("sim_member", "mxid", {room_id: roomID}).pluck().all()
+	const users = from("sim_member").join("sim", "mxid")
+		.where({room_id: roomID}).and("and user_id not like '%-%' and user_id not like '%\\_%' escape '\\'").pluck("user_id").all()
 
 	const channelID = select("channel_room", "channel_id", {room_id: roomID}).pluck().get()
 	assert.ok(typeof channelID === "string")
@@ -234,10 +234,7 @@ async function syncAllUsersInRoom(roomID) {
 	/** @ts-ignore @type {DiscordTypes.APIGuild} */
 	const guild = discord.guilds.get(guildID)
 
-	for (const mxid of mxids) {
-		const userID = select("sim", "user_id", {mxid}).pluck().get()
-		assert.ok(typeof userID === "string")
-
+	for (const userID of users) {
 		/** @ts-ignore @type {Required<DiscordTypes.APIGuildMember>} */
 		const member = await discord.snow.guild.getGuildMember(guildID, userID)
 		/** @ts-ignore @type {Required<DiscordTypes.APIUser>} user */

src/d2m/actions/remove-reaction.js

@@ -4,7 +4,7 @@ const Ty = require("../../types")
 const DiscordTypes = require("discord-api-types/v10")
 
 const passthrough = require("../../passthrough")
-const {discord, sync, db, select} = passthrough
+const {discord, sync, db, from, select} = passthrough
 /** @type {import("../../matrix/api")} */
 const api = sync.require("../../matrix/api")
 /** @type {import("../converters/emoji-to-key")} */
@@ -18,12 +18,15 @@ const converter = sync.require("../converters/remove-reaction")
  * @param {DiscordTypes.GatewayMessageReactionRemoveDispatchData | DiscordTypes.GatewayMessageReactionRemoveEmojiDispatchData | DiscordTypes.GatewayMessageReactionRemoveAllDispatchData} data
  */
 async function removeSomeReactions(data) {
-	const roomID = select("channel_room", "room_id", {channel_id: data.channel_id}).pluck().get()
-	if (!roomID) return
-	const eventIDForMessage = select("event_message", "event_id", {message_id: data.message_id, reaction_part: 0}).pluck().get()
-	if (!eventIDForMessage) return
+	const row = select("channel_room", "room_id", {channel_id: data.channel_id}).get()
+	if (!row) return
 
-	const reactions = await api.getFullRelations(roomID, eventIDForMessage, "m.annotation")
+	const eventReactedTo = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index")
+		.where({message_id: data.message_id, reaction_part: 0}).select("event_id", "room_id").get()
+	if (!eventReactedTo) return
+
+	// Due to server restrictions, all relations (i.e. reactions) have to be in the same room as the original event.
+	const reactions = await api.getFullRelations(eventReactedTo.room_id, eventReactedTo.event_id, "m.annotation")
 
 	// Run the proper strategy and any strategy-specific database changes
 	const removals = await
@@ -33,7 +36,7 @@ async function removeSomeReactions(data) {
 
 	// Redact the events and delete individual stored events in the database
 	for (const removal of removals) {
-		await api.redactEvent(roomID, removal.eventID, removal.mxid)
+		await api.redactEvent(eventReactedTo.room_id, removal.eventID, removal.mxid)
 		if (removal.hash) db.prepare("DELETE FROM reaction WHERE hashed_event_id = ?").run(removal.hash)
 	}
 }

src/d2m/actions/send-message.js

@@ -28,6 +28,8 @@ const dUtils = sync.require("../../discord/utils")
  */
 async function sendMessage(message, channel, guild, row) {
 	const roomID = await createRoom.ensureRoom(message.channel_id)
+	const historicalRoomIndex = select("historical_channel_room", "historical_room_index", {room_id: roomID}).pluck().get()
+	assert(historicalRoomIndex)
 
 	let senderMxid = null
 	if (dUtils.isWebhookMessage(message)) {
@@ -52,7 +54,7 @@ async function sendMessage(message, channel, guild, row) {
 	const events = await messageToEvent.messageToEvent(message, guild, {}, {api, snow: discord.snow})
 	const eventIDs = []
 	if (events.length) {
-		db.prepare("INSERT OR IGNORE INTO message_channel (message_id, channel_id) VALUES (?, ?)").run(message.id, message.channel_id)
+		db.prepare("INSERT OR IGNORE INTO message_room (message_id, historical_room_index) VALUES (?, ?)").run(message.id, historicalRoomIndex)
 		if (senderMxid) api.sendTyping(roomID, false, senderMxid).catch(() => {})
 	}
 	for (const event of events) {

src/d2m/converters/message-to-event.js

@@ -265,7 +265,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 				- So make sure we don't do anything in this case.
 	*/
 	const mentions = {}
-	/** @type {{event_id: string, room_id: string, source: number}?} */
+	/** @type {{event_id: string, room_id: string, source: number, channel_id: string}?} */
 	let repliedToEventRow = null
 	let repliedToUnknownEvent = false
 	let repliedToEventSenderMxid = null
@@ -280,9 +280,9 @@ async function messageToEvent(message, guild, options = {}, di) {
 	// Mentions scenarios 1 and 2, part A. i.e. translate relevant message.mentions to m.mentions
 	// (Still need to do scenarios 1 and 2 part B, and scenario 3.)
 	if (message.type === DiscordTypes.MessageType.Reply && message.message_reference?.message_id) {
-		const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").select("event_id", "room_id", "source").and("WHERE message_id = ? AND part = 0").get(message.message_reference.message_id)
+		const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index").select("event_id", "room_id", "reference_channel_id", "source").and("WHERE message_id = ? AND part = 0").get(message.message_reference.message_id)
 		if (row) {
-			repliedToEventRow = row
+			repliedToEventRow = Object.assign(row, {channel_id: row.reference_channel_id})
 		} else if (message.referenced_message) {
 			repliedToUnknownEvent = true
 		}
@@ -294,7 +294,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 			assert(message.embeds[0].description)
 			const match = message.embeds[0].description.match(/\/channels\/[0-9]*\/[0-9]*\/([0-9]{2,})/)
 			if (match) {
-				const row = from("event_message").join("message_channel", "message_id").join("channel_room", "channel_id").select("event_id", "room_id", "source").and("WHERE message_id = ? AND part = 0").get(match[1])
+				const row = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index").select("event_id", "room_id", "reference_channel_id", "source").and("WHERE message_id = ? AND part = 0").get(match[1])
 				if (row) {
 					/*
 						we generate a partial referenced_message based on what PK provided. we don't need everything, since this will only be used for further message-to-event converting.
@@ -313,7 +313,7 @@ async function messageToEvent(message, guild, options = {}, di) {
 						}
 					}
 					message.embeds.shift()
-					repliedToEventRow = row
+					repliedToEventRow = Object.assign(row, {channel_id: row.reference_channel_id})
 				}
 			}
 		}
@@ -485,8 +485,12 @@ async function messageToEvent(message, guild, options = {}, di) {
 			const {body: repliedToBody, html: repliedToHtml} = await transformContent(repliedToContent)
 			if (repliedToEventRow) {
 				// Generate a reply pointing to the Matrix event we found
-				html = `<mx-reply><blockquote><a href="https://matrix.to/#/${repliedToEventRow.room_id}/${repliedToEventRow.event_id}">In reply to</a> ${repliedToUserHtml}`
-					+ `<br>${repliedToHtml}</blockquote></mx-reply>`
+				const latestRoomID = select("channel_room", "room_id", {channel_id: repliedToEventRow.channel_id}).pluck().get()  // native replies don't work across room upgrades, so make sure the old and new message are in the same room
+				html =
+					(latestRoomID === repliedToEventRow.room_id ? "<mx-reply>" : "")
+					+ `<blockquote><a href="https://matrix.to/#/${repliedToEventRow.room_id}/${repliedToEventRow.event_id}">In reply to</a> ${repliedToUserHtml}`
+					+ `<br>${repliedToHtml}</blockquote>`
+					+ (latestRoomID === repliedToEventRow.room_id ? "</mx-reply>" : "")
 					+ html
 				body = (`${repliedToDisplayName}: ` // scenario 1 part B for mentions
 					+ repliedToBody).split("\n").map(line => "> " + line).join("\n")
@@ -544,21 +548,23 @@ async function messageToEvent(message, guild, options = {}, di) {
 	// Forwarded content appears first
 	if (message.message_reference?.type === DiscordTypes.MessageReferenceType.Forward && message.message_snapshots?.length) {
 		// Forwarded notice
-		const eventID = select("event_message", "event_id", {message_id: message.message_reference.message_id}).pluck().get()
+		const event = from("event_message").join("message_room", "message_id").join("historical_channel_room", "historical_room_index")
+			.select("event_id", "room_id").where({message_id: message.message_reference.message_id}).get()
 		const room = select("channel_room", ["room_id", "name", "nick"], {channel_id: message.message_reference.channel_id}).get()
 		const forwardedNotice = new mxUtils.MatrixStringBuilder()
 		if (room) {
 			const roomName = room && (room.nick || room.name)
-			const via = await getViaServersMemo(room.room_id)
-			if (eventID) {
+			if (event) {
+				const via = await getViaServersMemo(event.room_id)
 				forwardedNotice.addLine(
 					`[🔀 Forwarded from #${roomName}]`,
-					tag`🔀 <em>Forwarded from <a href="https://matrix.to/#/${room.room_id}/${eventID}?${via}">${roomName}</a></em>`
+					tag`🔀 <em>Forwarded from ${roomName} <a href="https://matrix.to/#/${room.room_id}/${event.event_id}?${via}">[jump to event]</a></em>`
 				)
 			} else {
+				const via = await getViaServersMemo(room.room_id)
 				forwardedNotice.addLine(
 					`[🔀 Forwarded from #${roomName}]`,
-					tag`🔀 <em>Forwarded from <a href="https://matrix.to/#/${room.room_id}?${via}">${roomName}</a></em>`
+					tag`🔀 <em>Forwarded from ${roomName} <a href="https://matrix.to/#/${room.room_id}?${via}">[jump to room]</a></em>`
 				)
 			}
 		} else {

src/d2m/converters/message-to-event.test.js

@@ -1152,7 +1152,7 @@ test("message2event: constructed forwarded message", async t => {
 			body: "[🔀 Forwarded from #wonderland]"
 				+ "\n» What's cooking, good looking? :hipposcope:",
 			format: "org.matrix.custom.html",
-			formatted_body: `🔀 <em>Forwarded from <a href="https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$tBIT8mO7XTTCgIINyiAIy6M2MSoPAdJenRl_RLyYuaE?via=cadence.moe&amp;via=matrix.org">wonderland</a></em>`
+			formatted_body: `🔀 <em>Forwarded from wonderland <a href="https://matrix.to/#/!qzDBLKlildpzrrOnFZ:cadence.moe/$tBIT8mO7XTTCgIINyiAIy6M2MSoPAdJenRl_RLyYuaE?via=cadence.moe&amp;via=matrix.org">[jump to event]</a></em>`
 				+ `<br><blockquote>What's cooking, good looking? <img data-mx-emoticon height="32" src="mxc://cadence.moe/WbYqNlACRuicynBfdnPYtmvc" title=":hipposcope:" alt=":hipposcope:"></blockquote>`,
 			"m.mentions": {},
 			msgtype: "m.notice",
@@ -1210,7 +1210,7 @@ test("message2event: constructed forwarded text", async t => {
 			body: "[🔀 Forwarded from #amanda-spam]"
 				+ "\n» What's cooking, good looking?",
 			format: "org.matrix.custom.html",
-			formatted_body: `🔀 <em>Forwarded from <a href="https://matrix.to/#/!CzvdIdUQXgUjDVKxeU:cadence.moe?via=cadence.moe&amp;via=matrix.org">amanda-spam</a></em>`
+			formatted_body: `🔀 <em>Forwarded from amanda-spam <a href="https://matrix.to/#/!CzvdIdUQXgUjDVKxeU:cadence.moe?via=cadence.moe&amp;via=matrix.org">[jump to room]</a></em>`
 				+ `<br><blockquote>What's cooking, good looking?</blockquote>`,
 			"m.mentions": {},
 			msgtype: "m.notice",

src/d2m/converters/user-to-mxid.js

@@ -99,6 +99,10 @@ function webhookAuthorToFakeUserID(author) {
 	return `webhook_${downcased}`
 }
 
+function isWebhookUserID(userID) {
+	return userID.match(/^webhook_[a-z90-9._=/-]+$/)
+}
+
 /**
  * @param {Ty.WebhookAuthor} author
  * @returns {string}
@@ -132,3 +136,4 @@ function webhookAuthorToSimName(author) {
 module.exports.userToSimName = userToSimName
 module.exports.webhookAuthorToFakeUserID = webhookAuthorToFakeUserID
 module.exports.webhookAuthorToSimName = webhookAuthorToSimName
+module.exports.isWebhookUserID = isWebhookUserID

src/d2m/discord-packets.js

@@ -6,10 +6,6 @@ const DiscordTypes = require("discord-api-types/v10")
 const passthrough = require("../passthrough")
 const {sync, db} = passthrough
 
-function populateGuildID(guildID, channelID) {
-	db.prepare("UPDATE channel_room SET guild_id = ? WHERE channel_id = ?").run(guildID, channelID)
-}
-
 const utils = {
 	/**
 	 * @param {import("./discord-client")} client
@@ -41,14 +37,12 @@ const utils = {
 				channel.guild_id = message.d.id
 				arr.push(channel.id)
 				client.channels.set(channel.id, channel)
-				populateGuildID(message.d.id, channel.id)
 			}
 			for (const thread of message.d.threads || []) {
 				// @ts-ignore
 				thread.guild_id = message.d.id
 				arr.push(thread.id)
 				client.channels.set(thread.id, thread)
-				populateGuildID(message.d.id, thread.id)
 			}
 
 			if (listen === "full") {
@@ -112,7 +106,6 @@ const utils = {
 		} else if (message.t === "THREAD_CREATE") {
 			client.channels.set(message.d.id, message.d)
 			if (message.d["guild_id"]) {
-				populateGuildID(message.d["guild_id"], message.d.id)
 				const channels = client.guildChannelMap.get(message.d["guild_id"])
 				if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
 			}
@@ -140,7 +133,6 @@ const utils = {
 		} else if (message.t === "CHANNEL_CREATE") {
 			client.channels.set(message.d.id, message.d)
 			if (message.d["guild_id"]) { // obj[prop] notation can be used to access a property without typescript complaining that it doesn't exist on all values something can have
-				populateGuildID(message.d["guild_id"], message.d.id)
 				const channels = client.guildChannelMap.get(message.d["guild_id"])
 				if (channels && !channels.includes(message.d.id)) channels.push(message.d.id)
 			}

src/d2m/event-dispatcher.js

@@ -67,7 +67,7 @@ module.exports = {
 	async checkMissedMessages(client, guild) {
 		if (guild.unavailable) return
 		const bridgedChannels = select("channel_room", "channel_id").pluck().all()
-		const preparedExists = db.prepare("SELECT channel_id FROM message_channel WHERE channel_id = ? LIMIT 1")
+		const preparedExists = from("message_room").join("historical_channel_room", "historical_room_index").pluck("message_id").and("WHERE reference_channel_id = ? LIMIT 1").prepare()
 		const preparedGet = select("event_message", "event_id", {}, "WHERE message_id = ?").pluck()
 		/** @type {(DiscordTypes.APIChannel & {type: DiscordTypes.GuildChannelType})[]} */
 		let channels = []