Render pug blocks without a separate template

.vscode/settings.json

@@ -1,5 +1,11 @@
 {
 	"editor.insertSpaces": false,
 	"editor.detectIndentation": false,
-	"editor.tabSize": 3
+	"editor.tabSize": 3,
+	"[jade]": {
+		"editor.stickyScroll.enabled": true,
+		"editor.stickyScroll.maxLineCount": 6,
+		"editor.tabSize": 2,
+		"editor.insertSpaces": true
+	}
 }

package-lock.json

@@ -1148,6 +1148,7 @@
       "version": "13.0.3",
       "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-13.0.3.tgz",
       "integrity": "sha512-RbOBxmLBG8uvFUc15X9+9SFemKcQ0WBuISBVkpuiaUB2qblC8UWlHEjdWVoZ8AdhSwmoEgsiXKfopX0CQxaACQ==",
+      "hasInstallScript": true,
       "license": "MIT",
       "dependencies": {
         "node-addon-api": "^8.0.0"

scripts/start-server.js

@@ -13,6 +13,10 @@ const sync = new HeatSync()
 
 Object.assign(passthrough, {sync, db})
 
+const orm = sync.require("../src/db/orm")
+passthrough.from = orm.from
+passthrough.select = orm.select
+
 const DiscordClient = require("../src/d2m/discord-client")
 
 const discord = new DiscordClient(reg.ooye.discord_token, "half")
@@ -21,10 +25,6 @@ passthrough.discord = discord
 const {as} = require("../src/matrix/appservice")
 passthrough.as = as
 
-const orm = sync.require("../src/db/orm")
-passthrough.from = orm.from
-passthrough.select = orm.select
-
 ;(async () => {
 	await migrate.migrate(db)
 	await discord.cloud.connect()

src/js/errors.js

@@ -0,0 +1,53 @@
+/**
+ * This function is adapted from Evan Kaufman's fantastic work.
+ * The original function and my adapted function are both MIT licensed.
+ * @url https://github.com/EvanK/npm-loggable-error/
+ * @param {number} [depth]
+ * @returns {string}
+*/
+function stringifyErrorStack(err, depth = 0) {
+	let collapsed = " ".repeat(depth);
+	if (!(err instanceof Error)) {
+		return collapsed + err
+	}
+
+	// add full stack trace if one exists, otherwise convert to string
+	let stackLines = String(err?.stack ?? err).replace(/^/gm, " ".repeat(depth)).trim().split("\n")
+	let cloudstormLine = stackLines.findIndex(l => l.includes("/node_modules/cloudstorm/"))
+	if (cloudstormLine !== -1) {
+		stackLines = stackLines.slice(0, cloudstormLine - 2)
+	}
+	collapsed += stackLines.join("\n")
+
+	const props = Object.getOwnPropertyNames(err).filter(p => !["message", "stack"].includes(p))
+
+	// only break into object notation if we have additional props to dump
+	if (props.length) {
+		const dedent = " ".repeat(depth);
+		const indent = " ".repeat(depth + 2);
+
+		collapsed += " {\n";
+
+		// loop and print each (indented) prop name
+		for (let property of props) {
+			collapsed += `${indent}[${property}]: `;
+
+			// if another error object, stringify it too
+			if (err[property] instanceof Error) {
+				collapsed += stringifyErrorStack(err[property], depth + 2).trimStart();
+			}
+			// otherwise stringify as JSON
+			else {
+				collapsed += JSON.stringify(err[property]);
+			}
+
+			collapsed += "\n";
+		}
+
+		collapsed += `${dedent}}\n`;
+	}
+
+	return collapsed;
+}
+
+module.exports.stringifyErrorStack = stringifyErrorStack

src/js/errors.test.js

@@ -0,0 +1,23 @@
+// @ts-check
+
+const {test} = require("supertape")
+const {stringifyErrorStack} = require("./errors")
+
+test("stringify error stack: works", t => {
+	function a() {
+		const e = new Error("message", {cause: new Error("inner")})
+		// @ts-ignore
+		e.prop = 2.1
+		throw e
+	}
+	try {
+		a()
+		t.fail("shouldn't get here")
+	} catch (e) {
+		const str = stringifyErrorStack(e)
+		t.match(str, /^Error: message$/m)
+		t.match(str, /^    at a \(.*errors\.test\.js/m)
+		t.match(str, /^  \[cause\]: Error: inner$/m)
+		t.match(str, /^  \[prop\]: 2.1$/m)
+	}
+})

src/m2d/event-dispatcher.js

@@ -38,62 +38,12 @@ const retrigger = sync.require("../d2m/actions/retrigger")
 const homeserverStatus = sync.require("../matrix/homeserver-status")
 /** @type {import("./actions/typing")} */
 const typing = sync.require("./actions/typing")
+/** @type {import("../js/errors")} */
+const errors = sync.require("../js/errors")
 const {reg} = require("../matrix/read-registration")
 
 let lastReportedEvent = 0
 
-/**
- * This function is adapted from Evan Kaufman's fantastic work.
- * The original function and my adapted function are both MIT licensed.
- * @url https://github.com/EvanK/npm-loggable-error/
- * @param {number} [depth]
- * @returns {string}
-*/
-function stringifyErrorStack(err, depth = 0) {
-	let collapsed = " ".repeat(depth);
-	if (!(err instanceof Error)) {
-		return collapsed + err
-	}
-
-	// add full stack trace if one exists, otherwise convert to string
-	let stackLines = String(err?.stack ?? err).replace(/^/gm, " ".repeat(depth)).trim().split("\n")
-	let cloudstormLine = stackLines.findIndex(l => l.includes("/node_modules/cloudstorm/"))
-	if (cloudstormLine !== -1) {
-		stackLines = stackLines.slice(0, cloudstormLine - 2)
-	}
-	collapsed += stackLines.join("\n")
-
-	const props = Object.getOwnPropertyNames(err).filter(p => !["message", "stack"].includes(p))
-
-	// only break into object notation if we have additional props to dump
-	if (props.length) {
-		const dedent = " ".repeat(depth);
-		const indent = " ".repeat(depth + 2);
-
-		collapsed += " {\n";
-
-		// loop and print each (indented) prop name
-		for (let property of props) {
-			collapsed += `${indent}[${property}]: `;
-
-			// if another error object, stringify it too
-			if (err[property] instanceof Error) {
-				collapsed += stringifyErrorStack(err[property], depth + 2).trimStart();
-			}
-			// otherwise stringify as JSON
-			else {
-				collapsed += JSON.stringify(err[property]);
-			}
-
-			collapsed += "\n";
-		}
-
-		collapsed += `${dedent}}\n`;
-	}
-
-	return collapsed;
-}
-
 function printError(type, source, e, payload) {
 	console.error(`Error while processing a ${type} ${source} event:`)
 	console.error(e)
@@ -145,7 +95,7 @@ async function sendError(roomID, source, type, e, payload) {
 		builder.addLine(errorIntroLine)
 
 		// Where
-		const stack = cleanErrorStack(stringifyErrorStack(e))
+		const stack = cleanErrorStack(errors.stringifyErrorStack(e))
 		builder.addLine(`Error trace:\n${stack}`, tag`<details><summary>Error trace</summary><pre>${stack}</pre></details>`)
 
 		// How
@@ -543,7 +493,6 @@ async event => {
 	}
 }))
 
-module.exports.stringifyErrorStack = stringifyErrorStack
 module.exports.cleanErrorStack = cleanErrorStack
 module.exports.sendError = sendError
 module.exports.printError = printError

src/m2d/event-dispatcher.test.js

@@ -1,26 +1,7 @@
 // @ts-check
 
 const {test} = require("supertape")
-const {stringifyErrorStack, cleanErrorStack} = require("./event-dispatcher")
+const {cleanErrorStack} = require("./event-dispatcher")
-
-test("stringify error stack: works", t => {
-	function a() {
-		const e = new Error("message", {cause: new Error("inner")})
-		// @ts-ignore
-		e.prop = 2.1
-		throw e
-	}
-	try {
-		a()
-		t.fail("shouldn't get here")
-	} catch (e) {
-		const str = stringifyErrorStack(e)
-		t.match(str, /^Error: message$/m)
-		t.match(str, /^    at a \(.*event-dispatcher\.test\.js/m)
-		t.match(str, /^  \[cause\]: Error: inner$/m)
-		t.match(str, /^  \[prop\]: 2.1$/m)
-	}
-})
 
 test("clean error stack: removes webhook token", t => {
 	t.notMatch(

src/web/pug-sync.js

@@ -2,20 +2,24 @@
 
 const assert = require("assert/strict")
 const fs = require("fs")
+const util = require("util")
 const {join} = require("path")
 const getRelativePath = require("get-relative-path")
 const h3 = require("h3")
-const {defineEventHandler, defaultContentType, setResponseStatus, getQuery} = h3
+const {defineEventHandler, defaultContentType, setResponseStatus, getQuery, getRequestURL} = h3
 const {compileFile} = require("@cloudrac3r/pug")
+const walkAST = require("pug-walk")
 const pretty = process.argv.join(" ").includes("test")
 
 const {sync} = require("../passthrough")
 /** @type {import("./auth")} */
 const auth = sync.require("./auth")
+/** @type {import("../js/errors")} */
+const errors = sync.require("../js/errors")
 
 // Pug
 
-let globals = {}
+let globals = sync.remember(() => ({}))
 
 /** @type {Map<string, (event: import("h3").H3Event, locals: Record<string, any>) => Promise<string>>} */
 const pugCache = new Map()
@@ -30,21 +34,60 @@ function addGlobals(obj) {
  * @param {Record<string, any>} locals
  */
 function render(event, filename, locals) {
+	return renderFragments(event, filename, null, locals)
+}
+
+/**
+ * @param {import("h3").H3Event} event
+ * @param {string} filename
+ * @param {string[] | null} fragments
+ * @param {Record<string, any>} locals
+ */
+function renderFragments(event, filename, fragments, locals) {
 	const path = join(__dirname, "pug", filename)
-	return renderPath(event, path, locals)
+	return renderPath(event, path, fragments, locals)
+}
+
+/**
+ * @param {string[] | null} fragments
+ * @param {string[]} requiresOut
+ */
+function fragmentSelectorPlugins(fragments, requiresOut) {
+	if (!fragments) return []
+	return [{
+		parse: true,
+		postParse(ast, options) {
+			const found = []
+			walkAST(ast, (node, replace) => {
+				if (node.type === "NamedBlock") {
+					const [name, requires] = node.name.split("requires").map(x => x.trim())
+					// console.log(name, "|", requires)
+					if (fragments?.includes(name)) {
+						if (requires) requiresOut.push(...requires.split(",").map(x => x.trim()))
+						found.push(node)
+					}
+				}
+			})
+			// console.log(util.inspect(found, false, 9, true))
+			return {type: "Block", nodes: found, line: 0, filename: found[0].filename}
+		}
+	}]
 }
 
 /**
  * @param {import("h3").H3Event} event
  * @param {string} path
+ * @param {string[] | null} fragments
  * @param {Record<string, any>} locals
  */
-function renderPath(event, path, locals) {
+function renderPath(event, path, fragments, locals) {
+	const cacheKey = path + (fragments ? fragments.map(f => `#${f}`).join("") : "")
+
 	function compile() {
 		try {
-			const template = compileFile(path, {pretty})
+			const requiredVars = []
-			pugCache.set(path, async (event, locals) => {
+			const template = compileFile(path, {pretty, plugins: fragmentSelectorPlugins(fragments, requiredVars)})
-				defaultContentType(event, "text/html; charset=utf-8")
+			pugCache.set(cacheKey, async (event, locals) => {
 				const session = await auth.useSession(event)
 				const managed = await auth.getManagedGuilds(event)
 				const rel = (to, paramsObject) => {
@@ -55,16 +98,36 @@ function renderPath(event, path, locals) {
 					}
 					return result
 				}
-				return template(Object.assign({},
+				const templateVars = Object.assign({},
 					getQuery(event), // Query parameters can be easily accessed on the top level but don't allow them to overwrite anything
 					globals, // Globals
 					locals, // Explicit locals overwrite globals in case we need to DI something
 					{session, event, rel, managed} // These are assigned last so they overwrite everything else. It would be catastrophically bad if they can't be trusted.
-				))
+				)
+				// Verify all fragment block required variables are present
+				for (const key of requiredVars) {
+					if (templateVars[key] === undefined) {
+						setResponseStatus(event, 500, "Internal Template Variables Error")
+						defaultContentType(event, "text/plain")
+						return `requested fragments ${JSON.stringify(fragments)}\nrequire variables ${JSON.stringify(requiredVars)}\nbut "${key}" is ${templateVars[key]} (present: ${key in templateVars})`
+					}
+				}
+				try {
+					const content = template(templateVars)
+					defaultContentType(event, "text/html; charset=utf-8")
+					return content
+				} catch (e) {
+					console.error(e)
+					setResponseStatus(event, 500, "Internal Template Rendering Error")
+					defaultContentType(event, "text/plain")
+					e.url = getRequestURL(event).toString()
+					return errors.stringifyErrorStack(e)
+				}
 			})
 		/* c8 ignore start */
 		} catch (e) {
-			pugCache.set(path, async (event) => {
+			pugCache.set(cacheKey, async (event) => {
+				console.error(e)
 				setResponseStatus(event, 500, "Internal Template Error")
 				defaultContentType(event, "text/plain")
 				return e.toString()
@@ -73,14 +136,16 @@ function renderPath(event, path, locals) {
 		/* c8 ignore stop */
 	}
 
-	if (!pugCache.has(path)) {
+	if (!pugCache.has(cacheKey)) {
 		compile()
-		fs.watch(path, {persistent: false}, compile)
+		const ac = new AbortController()
-		fs.watch(join(__dirname, "pug", "includes"), {persistent: false}, compile)
+		const {signal} = ac
-		fs.watch(join(__dirname, "pug", "fragments"), {persistent: false}, compile)
+		fs.watch(path, {persistent: false, signal}, compile)
+		fs.watch(join(__dirname, "pug", "includes"), {persistent: false, signal}, compile)
+		sync.events.once(__filename, () => ac.abort())
 	}
 
-	const cb = pugCache.get(path)
+	const cb = pugCache.get(cacheKey)
 	assert(cb)
 	return cb(event, locals)
 }
@@ -98,5 +163,6 @@ function createRoute(router, url, filename) {
 
 module.exports.addGlobals = addGlobals
 module.exports.render = render
+module.exports.renderFragments = renderFragments
 module.exports.renderPath = renderPath
 module.exports.createRoute = createRoute

src/web/pug/fragments/default-roles-list.pug

@@ -1,5 +0,0 @@
-//- locals: guild, guild_id
-
-include ../includes/default-roles-list.pug
-+default-roles-list(guild, guild_id)
-+add-roles-menu(guild, guild_id)

src/web/pug/guild.pug

@@ -1,5 +1,4 @@
 extends includes/template.pug
-include includes/default-roles-list.pug
 
 mixin badge-readonly
   .s-badge.s-badge__xs.s-badge__icon.s-badge__muted
@@ -113,15 +112,33 @@ block body
         .d-flex.fw-wrap.g4
           .s-tag.s-tag__md.fs-body1.s-tag__required @everyone
 
-          +default-roles-list(guild, guild_id)
+          block default-roles-list requires guild, guild_id
+            #default-roles-list(style="display: contents")
+              each roleID in select("role_default", "role_id", {guild_id}).pluck().all()
+                - let r = guild.roles.find(r => r.id === roleID)
+                if r
+                  .s-tag.s-tag__md.fs-body1= r.name
+                    span(id=`role-loading-${roleID}`)
+                    button(name="remove_role" value=roleID hx-post="api/default-roles" hx-trigger="click consume" hx-indicator=`#role-loading-${roleID}`).s-tag--dismiss
+                      != icons.Icons.IconClearSm
 
           button(type="button" popovertarget="role-add").s-btn__dropdown.s-tag.s-tag__md.fs-body1.p0
             .s-tag--dismiss.m1
               != icons.Icons.IconPlusSm
 
-          #role-add.s-popover(popover style="display: revert").ws2.px0.py4.bs-lg.overflow-visible
+          #role-add.s-popover(popover style="display: revert").ws2.px0.py4.bs-lg.overflow-visible.box-border.lh-sm
             .s-popover--arrow.s-popover--arrow__tc
-            +add-roles-menu(guild, guild_id)
+
+            block add-roles-menu requires guild, guild_id
+              ul.s-menu(role="menu" hx-swap-oob="true").overflow-y-auto.overflow-x-hidden#add-roles-menu
+                li.s-menu--title.d-flex(role="separator") Select role
+                  span#add-role-loading
+                each r in guild.roles.sort((a, b) => b.position - a.position)
+                  if r.id !== guild_id && !r.managed
+                    - let selected = !!select("role_default", "role_id", {guild_id, role_id: r.id}).get()
+                    li(role="menuitem")
+                      button(name="toggle_role" value=r.id class={"is-selected": selected}).s-block-link.s-block-link__left= r.name
+
         p.fc-light.mb0.mt8 Matrix users will start with these roles. If your main channels are gated by a role, use this to let Matrix users skip the gate.
 
     h3.mt32.fs-category Features

src/web/pug/includes/default-roles-list.pug

@@ -1,19 +0,0 @@
-mixin default-roles-list(guild, guild_id)
-  #default-roles-list(style="display: contents")
-    each roleID in select("role_default", "role_id", {guild_id}).pluck().all()
-      - let r = guild.roles.find(r => r.id === roleID)
-      if r
-        .s-tag.s-tag__md.fs-body1= r.name
-          span(id=`role-loading-${roleID}`)
-          button(name="remove_role" value=roleID hx-post="api/default-roles" hx-trigger="click consume" hx-indicator=`#role-loading-${roleID}`).s-tag--dismiss
-            != icons.Icons.IconClearSm
-
-mixin add-roles-menu(guild, guild_id)
-  ul.s-menu(role="menu" hx-swap-oob="true").overflow-y-auto.overflow-x-hidden#add-roles-menu
-    li.s-menu--title.d-flex(role="separator") Select role
-      span#add-role-loading
-    each r in guild.roles.sort((a, b) => b.position - a.position)
-      if r.id !== guild_id && !r.managed
-        - let selected = !!select("role_default", "role_id", {guild_id, role_id: r.id}).get()
-        li(role="menuitem")
-          button(name="toggle_role" value=r.id class={"is-selected": selected}).s-block-link.s-block-link__left= r.name

src/web/pug/invite.pug

@@ -17,10 +17,10 @@ block body
       .fl-grow1
         h2.fs-headline1 Invite a Matrix user
 
-        form.d-flex.gy16.fd-column(method="post" action=rel("/api/invite") hx-post=rel("/api/invite") hx-indicator="#invite-button" hx-select="#ok" hx-target="#form-container")
+        form.d-flex.gy16.wmx3.fd-column(method="post" action=rel("/api/invite") hx-post=rel("/api/invite") hx-indicator="#invite-button" hx-select="#ok" hx-target="#form-container")
           .d-flex.gy4.fd-column
             label.s-label(for="mxid") Matrix ID
-            input.fl-grow1.s-input.wmx3#mxid(name="mxid" required placeholder="@user:example.org")
+            input.fl-grow1.s-input#mxid(name="mxid" required placeholder="@user:example.org")
           .d-flex.gy4.fd-column
             label.s-label(for="permissions") Permissions
             .s-select

src/web/routes/guild-settings.js

@@ -135,7 +135,7 @@ as.router.post("/api/default-roles", defineEventHandler(async event => {
 	await createSpace.syncSpaceFully(guildID) // this is inefficient but OK to call infrequently on user request
 
 	if (getRequestHeader(event, "HX-Request")) {
-		return pugSync.render(event, "fragments/default-roles-list.pug", {guild, guild_id: guildID})
+		return pugSync.renderFragments(event, "guild.pug", ["default-roles-list", "add-roles-menu"], {guild, guild_id: guildID})
 	} else {
 		return sendRedirect(event, `/guild?guild_id=${guildID}`, 302)
 	}

src/web/server.js

@@ -98,7 +98,7 @@ function tryStatic(event, fallthrough) {
 		getContents: id => {
 			if (id.match(/\.pug$/)) {
 				const path = join(publicDir, id)
-				return pugSync.renderPath(event, path, {})
+				return pugSync.renderPath(event, path, null, {})
 			} else {
 				return fs.createReadStream(join(publicDir, id))
 			}

test/test.js

@@ -130,6 +130,7 @@ file._actuallyUploadDiscordFileToMxc = function(url, res) { throw new Error(`Not
 
 	require("./addbot.test")
 	require("../src/db/orm.test")
+	require("../src/js/errors.test")
 	require("../src/web/server.test")
 	require("../src/discord/utils.test")
 	require("../src/matrix/kstate.test")