Introduction
This document is the source of truth for the Arcology web server — a Ktor-based HTTP server that serves published org-roam content as HTML. It ties together the publishing layer (route table, domain map) and the HTML renderer to serve pages over HTTP.
ArcologyServer
Main Routing Module
GET /— localhost: sitemap index listing all published paths. non-localhost: returnSITE/indexpublished pageGET /{path...}— published page (looks up route, renders or serves from cache)GET /health— health check (database, cache dir, org dir; 503 on degradation, see HealthCheck)GET /metrics— Prometheus text exposition of the ArcologyMetrics registry (blocked publicly at the nginx layer)GET /sitemap— SigmaJS force-directed graph of all published pages (Sitemap Graph and Tag Index)GET /sitemap.json— graphology-shaped nodes/edges JSON for the graph, strongETag→ 304 viaConditionalHeadersGET /tags,GET /tags/{tag}— tag index with HTMX partials (full page, or the list fragment when htmx sendsHX-Request)GET /arcology/node/{nodeId}— internal preview (renders any node by ID, even unpublished)GET /attachment/{hash}-{size}.{ext}— crushed attachment bytes from the attachment cache; content-addressed, strongETag= the hash,Cache-Control: immutable,If-None-Match→ 304 viaConditionalHeadersGET /arcology/attachment/{name}/html— HTMX fragment: an<img>pointing at the named (large) variant, swapped in client-side by =hx-swap="outerHTML"=. Separate from the bytes route so both are curl-debuggable without header sniffing.GET /404— not-found page for unpublished =links (accepts?node=<id>=)
We use Pebble Templates for the Arcology Web Templates
import io.ktor.server.pebble.PebbleContent
import io.ktor.server.pebble.Pebble
import io.pebbletemplates.pebble.loader.ClasspathLoader
import io.ktor.server.application.Application
import io.ktor.server.application.ApplicationCall
import io.ktor.server.application.call
import io.ktor.server.application.install
import io.ktor.server.plugins.conditionalheaders.ConditionalHeaders
import io.ktor.server.metrics.micrometer.MicrometerMetrics
import io.ktor.server.routing.get
import io.ktor.server.routing.routing
import io.ktor.server.response.respond
import io.ktor.server.response.respondOutputStream
import io.ktor.server.response.respondText
import io.ktor.server.engine.embeddedServer
import io.micrometer.core.instrument.distribution.DistributionStatisticConfig
import io.micrometer.prometheusmetrics.PrometheusConfig
import io.micrometer.prometheusmetrics.PrometheusMeterRegistry
import kotlin.time.TimeSource
import kotlin.time.toJavaDuration
import kotlin.time.measureTimedValue
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Jsonprivate val prometheusRegistry: PrometheusMeterRegistry = metrics?.registry ?: PrometheusMeterRegistry(PrometheusConfig.DEFAULT)
private val healthJson = Json { encodeDefaults = true }
fun Application.module() {
install(Pebble) {
loader(ClasspathLoader().apply { prefix = "templates/" })
}
install(ConditionalHeaders)
install(MicrometerMetrics) {
registry = prometheusRegistry
// The plugin binds the JVM binders (its default list) and registers
// its registry-wide MeterFilter here — both must precede any meter
// registration, which holds because MetricsConfiguration.registry()
// is bare and every Arcology meter is per-request or lazy. The
// distribution config gives Ktor's request timer server-side
// histogram buckets (percentilesHistogram) so quantiles aggregate
// in PromQL; client-side percentiles are a known aggregation footgun.
distributionStatisticConfig = DistributionStatisticConfig.builder()
.percentilesHistogram(true)
.build()
}
routing {
get("/metrics") {
val start = TimeSource.Monotonic.markNow()
call.respondText(prometheusRegistry.scrape(), ContentType.Text.Plain)
logRequest(call, start, "GET", "/metrics", call.request.headers["Host"], HttpStatusCode.OK)
}
get("/") {
val start = TimeSource.Monotonic.markNow()
val host = call.request.headers["Host"]
val localhost = isLocalhost(host)
if (!localhost) {
// Production domain: serve SITE/index if it exists, fall back to sitemap.
val site = resolveSiteFromHost(host)
if (site != null) {
val model = servePath("index", site)
if (model != null) {
model.etag?.let { call.response.headers.append(HttpHeaders.ETag, it) }
call.respond(PebbleContent(model.template, modelToMap(model)))
logRequest(call, start, "GET", "/", host, HttpStatusCode.OK, "site=$site index")
return@get
}
}
}
// Localhost or no SITE/index route: serve the sitemap.
try {
val model = buildSitemapModel(localhost)
call.respond(PebbleContent("sitemap.peb", model))
logRequest(call, start, "GET", "/", host, HttpStatusCode.OK, "sitemap")
} catch (e: Exception) {
logError("/", host, "sitemap error", e)
call.respondText("500 Internal Server Error: ${e.message}", ContentType.Text.Plain, HttpStatusCode.InternalServerError)
logRequest(call, start, "GET", "/", host, HttpStatusCode.InternalServerError)
}
}
get("/health") {
val start = TimeSource.Monotonic.markNow()
val host = call.request.headers["Host"]
val report = HealthCheck.check(
repository = repository,
cacheDir = htmlCache.cacheDir,
orgDir = orgDir
)
val body = healthJson.encodeToString(report)
if (report.status == "ok") {
call.respondText(body, ContentType.Application.Json, HttpStatusCode.OK)
logRequest(call, start, "GET", "/health", host, HttpStatusCode.OK)
} else {
call.respondText(body, ContentType.Application.Json, HttpStatusCode.ServiceUnavailable)
logRequest(call, start, "GET", "/health", host, HttpStatusCode.ServiceUnavailable, "status=${report.status}")
}
}
get("/sitemap") {
val start = TimeSource.Monotonic.markNow()
val host = call.request.headers["Host"]
val localhost = isLocalhost(host)
try {
val model = mapOf(
"site" to siteModelFor(resolveSiteFromHost(host)),
"localhost" to localhost
)
call.respond(PebbleContent("graph.peb", model))
logRequest(call, start, "GET", "/sitemap", host, HttpStatusCode.OK, if (localhost) "localhost" else "graph")
} catch (e: Exception) {
logError("/sitemap", host, "render error", e)
call.respondText("500 Internal Server Error: ${e.message ?: "unknown"}", ContentType.Text.Plain, HttpStatusCode.InternalServerError)
logRequest(call, start, "GET", "/sitemap", host, HttpStatusCode.InternalServerError)
}
}
get("/sitemap.json") {
val start = TimeSource.Monotonic.markNow()
val host = call.request.headers["Host"]
val localhost = isLocalhost(host)
try {
val (body, etag) = sitemapGraph.json(localhost)
call.response.headers.append(HttpHeaders.ETag, etag)
call.respondText(body, ContentType.Application.Json)
logRequest(call, start, "GET", "/sitemap.json", host, HttpStatusCode.OK, if (localhost) "localhost" else "graph")
} catch (e: Exception) {
logError("/sitemap.json", host, "graph build error", e)
call.respondText("500 Internal Server Error: ${e.message ?: "unknown"}", ContentType.Text.Plain, HttpStatusCode.InternalServerError)
logRequest(call, start, "GET", "/sitemap.json", host, HttpStatusCode.InternalServerError)
}
}
get("/tags") {
val start = TimeSource.Monotonic.markNow()
val host = call.request.headers["Host"]
val localhost = isLocalhost(host)
try {
val tags = sitemapTags.allTags(localhost).map { (tag, count) ->
mapOf("tag" to tag, "count" to count)
}
val model = mapOf(
"site" to siteModelFor(resolveSiteFromHost(host)),
"tags" to tags,
"localhost" to localhost
)
call.respond(PebbleContent("tags.peb", model))
logRequest(call, start, "GET", "/tags", host, HttpStatusCode.OK, if (localhost) "localhost" else "tags")
} catch (e: Exception) {
logError("/tags", host, "render error", e)
call.respondText("500 Internal Server Error: ${e.message ?: "unknown"}", ContentType.Text.Plain, HttpStatusCode.InternalServerError)
logRequest(call, start, "GET", "/tags", host, HttpStatusCode.InternalServerError)
}
}
get("/tags/{tag}") {
val start = TimeSource.Monotonic.markNow()
val tag = call.parameters["tag"] ?: ""
val host = call.request.headers["Host"]
val localhost = isLocalhost(host)
val isHtmx = call.request.headers["HX-Request"] == "true"
try {
val pages = sitemapTags.tagPages(tag, localhost).map { page ->
mapOf("title" to page.title, "href" to page.href, "weight" to page.weight)
}
val model = mapOf(
"site" to siteModelFor(resolveSiteFromHost(host)),
"tag" to tag,
"pages" to pages,
"localhost" to localhost
)
// htmx requests get just the list fragment to swap in place of
// the empty <ul>; everything else gets the full page.
val template = if (isHtmx) "tag-list.peb" else "tag.peb"
call.respond(PebbleContent(template, model))
logRequest(call, start, "GET", "/tags/$tag", host, HttpStatusCode.OK, if (isHtmx) "htmx-partial" else "tag")
} catch (e: Exception) {
logError("/tags/$tag", host, "render error", e)
call.respondText("500 Internal Server Error: ${e.message ?: "unknown"}", ContentType.Text.Plain, HttpStatusCode.InternalServerError)
logRequest(call, start, "GET", "/tags/$tag", host, HttpStatusCode.InternalServerError)
}
}
get("/sites.css") {
val start = TimeSource.Monotonic.markNow()
call.respondText(sitesCssMemo, ContentType.Text.CSS)
logRequest(call, start, "GET", "/sites.css", call.request.headers["Host"], HttpStatusCode.OK)
}
get("/opml.xml") {
val start = TimeSource.Monotonic.markNow()
// OPML subscription list scoped to the requesting site, matching
// how <link rel="alternate"> autodiscovery is scoped (localhost
// sees every site's feeds, since it browses the whole arcology).
val host = call.request.headers["Host"]
val localhost = isLocalhost(host)
val site = if (localhost) null else resolveSiteFromHost(host)
val opml = opmlFor(site, localhost, feedRepository, domainMap)
if (opml == null) {
call.respondText("404 Not Found: no site for host '$host'", ContentType.Text.Plain, HttpStatusCode.NotFound)
logRequest(call, start, "GET", "/opml.xml", host, HttpStatusCode.NotFound)
} else {
call.respondText(opml, ContentType("application", "xml"))
logRequest(call, start, "GET", "/opml.xml", host, HttpStatusCode.OK, if (localhost) "localhost" else "site=$site")
}
}
get("/static/{path...}") {
val start = TimeSource.Monotonic.markNow()
val resourcePath = "static/" + (call.parameters.getAll("path")?.joinToString("/") ?: "")
val resource = this::class.java.classLoader.getResourceAsStream(resourcePath)
if (resource != null) {
val contentType = when {
resourcePath.endsWith(".css") -> ContentType.Text.CSS
resourcePath.endsWith(".woff2") -> ContentType("font", "woff2")
resourcePath.endsWith(".woff") -> ContentType("font", "woff")
resourcePath.endsWith(".ttf") -> ContentType("font", "ttf")
resourcePath.endsWith(".js") -> ContentType.Text.JavaScript
resourcePath.endsWith(".png") -> ContentType.Image.PNG
resourcePath.endsWith(".jpg") || resourcePath.endsWith(".jpeg") -> ContentType.Image.JPEG
resourcePath.endsWith(".svg") -> ContentType.Image.SVG
else -> ContentType.Text.Plain
}
call.respondOutputStream(contentType) { resource.copyTo(this) }
} else {
call.respond(HttpStatusCode.NotFound)
}
}
get("/attachment/{name}") {
val start = TimeSource.Monotonic.markNow()
val name = call.parameters["name"] ?: ""
val host = call.request.headers["Host"]
val file = attachmentFileFor(name)
if (file == null) {
call.respondText("404 Not Found: no attachment '$name'", ContentType.Text.Plain, HttpStatusCode.NotFound)
logRequest(call, start, "GET", "/attachment/$name", host, HttpStatusCode.NotFound)
return@get
}
// Content-addressed: the hash in the name IS the strong ETag, and the
// bytes can never change for a given name. ConditionalHeaders turns
// If-None-Match into a 304.
call.response.headers.append(HttpHeaders.ETag, "\"${name.substringBefore('-')}\"")
call.response.headers.append(HttpHeaders.CacheControl, "public, max-age=31536000, immutable")
call.respondOutputStream(contentTypeForAttachmentName(name)) { file.inputStream().copyTo(this) }
logRequest(call, start, "GET", "/attachment/$name", host, HttpStatusCode.OK)
}
get("/arcology/attachment/{name}/html") {
val start = TimeSource.Monotonic.markNow()
val name = call.parameters["name"] ?: ""
val host = call.request.headers["Host"]
if (attachmentFileFor(name) == null) {
call.respondText("404 Not Found: no attachment '$name'", ContentType.Text.Plain, HttpStatusCode.NotFound)
logRequest(call, start, "GET", "/arcology/attachment/$name/html", host, HttpStatusCode.NotFound)
return@get
}
// HX fragment: an <img> pointing at the large variant, swapped in
// client-side by htmx (hx-swap="outerHTML"). Separate from the bytes
// route so both stay curl-debuggable without header sniffing.
call.respondText(
"""<img src="/attachment/${htmlEscape(name)}" alt="" loading="lazy"/>""",
ContentType.Text.Html
)
logRequest(call, start, "GET", "/arcology/attachment/$name/html", host, HttpStatusCode.OK)
}
get("/404") {
val start = TimeSource.Monotonic.markNow()
val node = call.request.queryParameters["node"]
val model = mapOf("nodeId" to (node ?: ""))
call.respond(PebbleContent("404.peb", model))
logRequest(call, start, "GET", "/404", call.request.headers["Host"], HttpStatusCode.NotFound, "node=${node ?: "-"}")
}
get("/arcology/node/{nodeId}") {
val start = TimeSource.Monotonic.markNow()
val nodeId = call.parameters["nodeId"] ?: ""
val host = call.request.headers["Host"]
try {
val model = serveNodePreview(nodeId)
if (model != null) {
model.etag?.let { call.response.headers.append(HttpHeaders.ETag, it) }
call.respond(PebbleContent(model.template, modelToMap(model)))
logRequest(call, start, "GET", "/arcology/node/$nodeId", host, HttpStatusCode.OK)
} else {
call.respondText("404 Not Found: node '$nodeId' not found", ContentType.Text.Plain, HttpStatusCode.NotFound)
logRequest(call, start, "GET", "/arcology/node/$nodeId", host, HttpStatusCode.NotFound)
}
} catch (e: Exception) {
logError("/arcology/node/$nodeId", host, "render error", e)
call.respondText("500 Internal Server Error: ${e.message ?: "unknown"}", ContentType.Text.Plain, HttpStatusCode.InternalServerError)
logRequest(call, start, "GET", "/arcology/node/$nodeId", host, HttpStatusCode.InternalServerError)
}
}
get("/{path...}") {
val start = TimeSource.Monotonic.markNow()
val path = call.parameters.getAll("path")?.joinToString("/") ?: ""
val host = call.request.headers["Host"]
if (path == "health" || path == "arcology" || path == "404" || path == "sites.css" ||
path == "sitemap" || path == "sitemap.json" || path == "tags" || path == "opml.xml" || path == "metrics") {
return@get
}
val localhost = isLocalhost(host)
try {
// .xml paths are Atom feeds ([[file:feeds.org][FeedPublisher]]): the
// ARCOLOGY_FEED value's path is the literal URL. Localhost uses the
// full key (SITE/path.xml); production prefixes the host's site.
if (path.endsWith(".xml") && feedPublisher != null) {
val site = if (localhost) path.substringBefore("/") else resolveSiteFromHost(host)
val fullKey = if (localhost || site == null) path else "$site/$path"
val feedModel = feedPublisher?.serveFeed(fullKey, localhost)
if (feedModel != null) {
feedModel.etag?.let { call.response.headers.append(HttpHeaders.ETag, it) }
call.respondText(feedXmlFor(feedModel), ContentType.Text.Xml)
logRequest(call, start, "GET", "/$path", host, HttpStatusCode.OK, "feed key=$fullKey")
return@get
}
// Feed miss falls through to the page 404 below.
}
val model = if (localhost) {
servePathLocalhost(path)
} else {
val site = resolveSiteFromHost(host)
servePath(path, site)
}
if (model != null) {
model.etag?.let { call.response.headers.append(HttpHeaders.ETag, it) }
call.respond(PebbleContent(model.template, modelToMap(model)))
logRequest(call, start, "GET", "/$path", host, HttpStatusCode.OK, if (localhost) "localhost" else "site=${resolveSiteFromHost(host)}")
} else {
call.respondText("404 Not Found: no route for '$path'", ContentType.Text.Plain, HttpStatusCode.NotFound)
logRequest(call, start, "GET", "/$path", host, HttpStatusCode.NotFound, if (localhost) "localhost" else "site=${resolveSiteFromHost(host)}")
}
} catch (e: Exception) {
logError("/$path", host, "render error", e)
call.respondText("500 Internal Server Error: ${e.message ?: "unknown"}\n\nCheck the server console for details.", ContentType.Text.Plain, HttpStatusCode.InternalServerError)
logRequest(call, start, "GET", "/$path", host, HttpStatusCode.InternalServerError)
}
}
}
}
private fun isLocalhost(hostHeader: String?): Boolean {
if (hostHeader == null) return false
val host = hostHeader.substringBefore(":").trim().lowercase()
return host == "localhost" || host == "127.0.0.1" || host == "0.0.0.0"
}
fun start(port: Int = 8080) {
embeddedServer(CIO, port = port, host = "0.0.0.0", module = { module() }).start(wait = true)
}Sitemap Graph and Tag Index Builders
The /sitemap, /sitemap.json, and /tags routes above are thin handlers around two builders documented in sitemap.org: SitemapGraphBuilder (the SigmaJS graph payload, with its content-keyed in-memory cache and strong ETag) and TagIndexService (tag counting and weighted page lists). They are constructed lazy from the server's existing dependencies so the graph cache persists across requests, and the handlers call them as suspend functions directly — no runBlocking, unlike the page-serving paths, because these routes never need to block on file reads.
private val sitemapGraph: SitemapGraphBuilder by lazy {
SitemapGraphBuilder(repository, publishingRepository, domainMap).apply { metrics = this@ArcologyServer.metrics }
}
private val sitemapTags: TagIndexService by lazy {
TagIndexService(repository, publishingRepository, domainMap)
}Serving pages
The "fun" functions, servePath and servePathLocalhost take an ARCOLOGY_KEY and return a PageModel suitable to be rendered by the template engine. Both construct the OrgHtmlRenderer with the per-heading backlink/reference maps from collectSidebar so the renderer can emit inline backlink/reference sections under each heading — this is what powers the topic.peb template's clustered backlinks and the wide.peb template's inline references.
On a cache miss the CachedPage envelope is consumed exactly once and feeds both the cache write and the live PageModel. consumeHeadings() drains the renderer's accumulator (snapshot + clear), so a second consume would return an empty list — which is exactly the bug the first render of a page used to ship without its TOC until the cache warmed up. The miss branch therefore appends page.html page.sidenotes page.headings straight from the envelope instead of re-consuming the renderer.
import io.ktor.http.ContentType
import io.ktor.http.HttpStatusCode
import io.ktor.http.HttpHeaders
import io.ktor.server.cio.CIO
import kotlinx.coroutines.runBlocking private fun servePathLocalhost(fullKey: String): PageModel? {
val pathLabel = normalizePathLabel("/$fullKey")
val (result, duration) = measureTimedValue { servePathLocalhostBody(fullKey) }
metrics?.pageTimer("localhost", pathLabel)?.record(duration.toJavaDuration())
return result
}
private fun servePathLocalhostBody(fullKey: String): PageModel? {
val siteKey = fullKey.substringBefore("/")
val path = fullKey.substringAfter("/")
// Localhost sees ALL routes (incl. drafts/archived) so you can browse everything.
val entries = runBlocking { publishingRepository.getRoutesByPath(path) }
.filter { it.site == siteKey }
if (entries.isEmpty()) return null
val publishedIndex = runBlocking { buildRouteIndex(publishingRepository.getPublishedRoutes()) }
val resolver = linkResolverFor(site = siteKey, localhost = true, publishedIndex = publishedIndex)
val sidebar = collectSidebar(entries, entries.first().file, siteKey, publishedIndex)
val renderer = OrgHtmlRenderer(
resolver, publishedIndex.keys,
sidebar.backlinksByNode, sidebar.referencesByNode
)
// The cache key mixes in the sidebar digest: inline backlinks/refs depend on
// the whole link graph, so a stale cache entry must be invalidated when the
// graph changes even if this file's own hash is unchanged.
val sidebarDigest = sidebarDigestFor(sidebar)
val bodies = StringBuilder()
val sidenotes = StringBuilder()
val headingsList = mutableListOf<HeadingLink>()
for (entry in entries) {
val fileHash = fileHashFor(entry) + sidebarDigest
val cached = htmlCache.getPage(entry.nodeId, fileHash)
if (cached != null) {
metrics?.cacheHit("page")
bodies.append(cached.html)
sidenotes.append(cached.sidenotes)
headingsList.addAll(cached.headings)
} else {
metrics?.cacheMiss("page")
val html = metrics?.nodeRenderTimer?.let { timer ->
timer.recordCallable { renderNodeBody(entry, renderer) }
} ?: renderNodeBody(entry, renderer)
if (html != null) {
// One consume feeds both the cache write and the live page:
// consumeHeadings() drains the renderer's accumulator, so a
// second consume would return an empty list and the first
// render of a page would ship without its TOC.
val page = CachedPage(
html = html,
sidenotes = renderer.renderSidenotes(),
headings = renderer.consumeHeadings()
)
htmlCache.putPage(entry.nodeId, fileHash, page)
bodies.append(page.html)
sidenotes.append(page.sidenotes)
headingsList.addAll(page.headings)
}
}
// Reset footnote state for the next entry (no-op on cache hits).
renderer.consumeFootnotes()
}
val templateName = resolveTemplate(entries.first())
return PageModel(
site = siteModelFor(siteKey),
headTitle = fullKey,
pageTitle = entries.firstOrNull()?.title ?: fullKey,
htmlContent = bodies.toString(),
sidenotesHtml = sidenotes.toString(),
headings = headingsList,
backlinks = sidebar.backlinks,
tags = sidebar.tags,
references = sidebar.references,
keywords = sidebar.keywords,
feeds = feedLinksFor(siteKey, localhost = true),
allowCrawl = true,
template = templateName,
etag = weakEtagFor(entries, sidebar)
)
}
private fun servePath(path: String, site: String?): PageModel? {
val pathLabel = normalizePathLabel("/$path")
val (result, duration) = measureTimedValue { servePathBody(path, site) }
metrics?.pageTimer(site, pathLabel)?.record(duration.toJavaDuration())
return result
}
private fun servePathBody(path: String, site: String?): PageModel? {
// Production sees only published (non-draft, non-archived) routes.
val entries = runBlocking { publishingRepository.getPublishedRoutesByPath(path) }
.let { if (site != null) it.filter { e -> e.site == site } else it }
if (entries.isEmpty()) return null
val publishedIndex = runBlocking { buildRouteIndex(publishingRepository.getPublishedRoutes()) }
val resolver = linkResolverFor(site = site, localhost = false, publishedIndex = publishedIndex)
val sidebar = collectSidebar(entries, entries.first().file, site, publishedIndex)
val renderer = OrgHtmlRenderer(
resolver, publishedIndex.keys,
sidebar.backlinksByNode, sidebar.referencesByNode
)
// The cache key mixes in the sidebar digest: inline backlinks/refs depend on
// the whole link graph, so a stale cache entry must be invalidated when the
// graph changes even if this file's own hash is unchanged.
val sidebarDigest = sidebarDigestFor(sidebar)
val bodies = StringBuilder()
val sidenotes = StringBuilder()
val headingsList = mutableListOf<HeadingLink>()
for (entry in entries) {
val fileHash = fileHashFor(entry) + sidebarDigest
val cached = htmlCache.getPage(entry.nodeId, fileHash)
if (cached != null) {
metrics?.cacheHit("page")
bodies.append(cached.html)
sidenotes.append(cached.sidenotes)
headingsList.addAll(cached.headings)
} else {
metrics?.cacheMiss("page")
val html = metrics?.nodeRenderTimer?.let { timer ->
timer.recordCallable { renderNodeBody(entry, renderer) }
} ?: renderNodeBody(entry, renderer)
if (html != null) {
// One consume feeds both the cache write and the live page:
// consumeHeadings() drains the renderer's accumulator, so a
// second consume would return an empty list and the first
// render of a page would ship without its TOC.
val page = CachedPage(
html = html,
sidenotes = renderer.renderSidenotes(),
headings = renderer.consumeHeadings()
)
htmlCache.putPage(entry.nodeId, fileHash, page)
bodies.append(page.html)
sidenotes.append(page.sidenotes)
headingsList.addAll(page.headings)
}
}
// Reset footnote state for the next entry (no-op on cache hits).
renderer.consumeFootnotes()
}
val templateName = resolveTemplate(entries.first())
return PageModel(
site = siteModelFor(site),
headTitle = entries.firstOrNull()?.title ?: path,
pageTitle = entries.firstOrNull()?.title ?: path,
htmlContent = bodies.toString(),
sidenotesHtml = sidenotes.toString(),
headings = headingsList,
backlinks = sidebar.backlinks,
tags = sidebar.tags,
references = sidebar.references,
keywords = sidebar.keywords,
feeds = feedLinksFor(site, localhost = false),
allowCrawl = true,
template = templateName,
etag = weakEtagFor(entries, sidebar)
)
}
private fun serveNodePreview(nodeId: String): PageModel? {
val node = runBlocking { repository.getNodeById(nodeId) } ?: return null
val entry = RouteEntry(
nodeId = nodeId, site = "", path = "",
file = node.file, title = node.title,
timestamp = null, isDraft = false, isArchived = false
)
val publishedIndex = runBlocking { buildRouteIndex(publishingRepository.getPublishedRoutes()) }
val resolver = linkResolverFor(site = null, localhost = true, publishedIndex = publishedIndex)
val sidebar = runBlocking { sidebarService.collect(listOf(nodeId), node.file, currentSite = null, publishedIndex = publishedIndex) }
val renderer = OrgHtmlRenderer(
resolver, publishedIndex.keys,
sidebar.backlinksByNode, sidebar.referencesByNode
)
val body = renderNodeBody(entry, renderer)
val sidenotes = renderer.renderSidenotes()
val headings = renderer.consumeHeadings()
renderer.consumeFootnotes()
if (body == null) return null
val templateName = resolveTemplate(entry)
return PageModel(
site = siteModelFor(null),
headTitle = node.title ?: nodeId,
pageTitle = node.title ?: nodeId,
htmlContent = body,
sidenotesHtml = sidenotes,
headings = headings,
backlinks = sidebar.backlinks,
tags = sidebar.tags,
references = sidebar.references,
keywords = sidebar.keywords,
feeds = PageModel.NoFeeds,
allowCrawl = false,
template = templateName
)
}Rendering pages
The node-body pipeline is split into two top-level internal functions so that the FeedPublisher can reuse it: loadNodeParseResult does the file I/O (read + parse, one call per source file), renderParsedNode is the pure half (find the section by node ID, render with the given renderer). renderNodeBody composes the two with a RoamRepository lookup and is what the page-serving paths use. This lets a feed parse its file once and render N entry sections, while pages keep their existing call shape.
import computer.whatthefuck.arcology.parser.ParseResult
import java.io.File/**
* File I/O half of the node-body pipeline: read the node's source file and
* parse it. Returns null when the node doesn't exist or the file is missing.
* Callers rendering many nodes from one file (the FeedPublisher) should call
* this once per file and share the result.
*/
internal fun loadNodeParseResult(
orgDir: String,
parser: OrgFileParser,
repository: RoamRepository,
nodeId: String,
fileOverride: String? = null
): ParseResult.Success? {
val node = runBlocking { repository.getNodeById(nodeId) } ?: return null
val filePath = File(orgDir, fileOverride ?: node.file).absolutePath
val file = File(filePath)
if (!file.exists()) return null
val content = file.readText()
val parseResult = parser.parseFileContent(node.file, content, kotlin.time.Instant.fromEpochSeconds(0))
return parseResult as? ParseResult.Success
}
/**
* Pure half of the node-body pipeline: render a node from an already-parsed
* document. Heading nodes render just their section; the file-level node
* (its ID is the preface's, not a section's) renders the whole document —
* preface + all top-level sections with recursion into sub-headings.
* See [[id:20260721T133000.000001]]
*/
internal fun renderParsedNode(
parseResult: ParseResult.Success,
nodeId: String,
renderer: OrgHtmlRenderer
): String? {
val document = parseResult.document
val section = document?.let { findSectionByNodeId(it.content, nodeId) }
return if (section != null) {
renderer.renderSections(listOf(section))
} else if (document != null) {
renderer.renderDocument(document)
} else {
null
}
}
/**
* Finds a section by node ID at any depth in the document AST. Was a member
* of [ArcologyServer]; hoisted here so [[renderParsedNode]] and the
* [[file:feeds.org][FeedPublisher]] can share it.
*/
internal fun findSectionByNodeId(sections: List<xyz.lepisma.orgmode.OrgSection>, nodeId: String): xyz.lepisma.orgmode.OrgSection? {
for (section in sections) {
val headingId = section.heading.properties?.map?.get("ID")?.let { orgLine ->
orgLine.items.filterIsInstance<xyz.lepisma.orgmode.OrgInlineElem.Text>()
.joinToString("") { it.text }.trim()
}
if (headingId == nodeId) return section
// Recurse into nested sections in the body
val nested = section.body.filterIsInstance<xyz.lepisma.orgmode.OrgSection>()
if (nested.isNotEmpty()) {
findSectionByNodeId(nested, nodeId)?.let { return it }
}
}
return null
}private fun renderNodeBody(entry: RouteEntry, renderer: OrgHtmlRenderer): String? {
val parseResult = loadNodeParseResult(orgDir, parser, repository, entry.nodeId) ?: return null
return renderParsedNode(parseResult, entry.nodeId, renderer)
}private fun collectSidebar(entries: List<RouteEntry>, filePath: String, currentSite: String?, publishedIndex: Map<String, RouteEntry>): SidebarData {
// Query ALL nodes in the file, not just the route entry node IDs. Topic/wide
// templates render sub-headings whose backlinks and refs point to those
// sub-heading node IDs — they're not in `entries` (they carry no
// ARCOLOGY_KEY) but the renderer needs their backlink/ref data for the
// inline per-heading sections. The flat sidebar lists grow accordingly,
// which is the more correct behavior for multi-node files anyway.
val fileNodeIds = runBlocking { repository.getNodesByFile(filePath) }.map { it.id }
val nodeIds = (fileNodeIds + entries.map { it.nodeId }).distinct()
val firstNodeId = entries.first().nodeId
val fileHash = fileHashFor(entries.first())
val cached = htmlCache.getSidebar(firstNodeId, fileHash)
if (cached != null) {
// Re-hydrate the sidebar HTML is not worth it; we cache the rendered partial instead.
// For simplicity, recompute — the sidebar HTML cache is only a speed win when rendering
// the partial is expensive, and since we recompute here anyway, the cache is a no-op
// until we wire the partial rendering through Pebble. Left as a future optimization.
}
return runBlocking { sidebarService.collect(nodeIds, filePath, currentSite, publishedIndex) }
}Data Transformation and Builders
private fun modelToMap(model: PageModel): Map<String, Any> = mapOf(
"site" to model.site,
"headTitle" to model.headTitle,
"pageTitle" to model.pageTitle,
"htmlContent" to model.htmlContent,
"sidenotesHtml" to model.sidenotesHtml,
"headings" to model.headings,
"backlinks" to model.backlinks,
"tags" to model.tags,
"references" to model.references,
"keywords" to model.keywords,
"feeds" to model.feeds,
"allowCrawl" to model.allowCrawl,
// Sidebar panel visibility flags, derived from the resolved template rather
// than Pebble's `include ... with` forwarding (which does not reach the
// partial's conditions through Ktor's PebbleContent model). topic.peb
// suppresses both backlinks and references panels — those are inlined under
// each heading by the renderer — and wide.peb suppresses references only.
"wide" to (model.template == "wide.peb" || model.template == "topic.peb"),
"topic" to (model.template == "topic.peb"),
"refsInline" to (model.template == "wide.peb" || model.template == "topic.peb")
).filterValues { it != null } as Map<String, Any>
private fun resolveSiteFromHost(hostHeader: String?): String? {
if (hostHeader == null) return null
val host = hostHeader.substringBefore(":").trim().lowercase()
return domainMap.resolveSite(host)
}
private fun siteModelFor(site: String?): SiteModel {
val meta = site?.let { domainMap.meta(it) }
return SiteModel(
key = site ?: "",
title = meta?.title,
cssFile = meta?.cssFile,
hljsTheme = meta?.hljsTheme,
domain = site?.let { domainMap.resolve(it) }
)
}
/**
* Feed autodiscovery links for a site: every feed declared under SITE becomes
* a <link rel="alternate"> on the site's pages ([[file:templates.org]]
* renders the loop from PageModel.feeds). Silent no-op when the feed
* repository isn't wired (tests construct the server without it).
*/
private fun feedLinksFor(site: String?, localhost: Boolean): List<FeedLink> {
val feedRepo = feedRepository ?: return PageModel.NoFeeds
if (site == null) return PageModel.NoFeeds
return runBlocking {
feedRepo.getFeedsBySite(site)
.groupBy { it.routeKey }
.map { (routeKey, declarers) ->
val path = declarers.first().path
val domain = domainMap.resolve(site)
val url = if (localhost || domain == null) "/$routeKey" else "http://$domain/$path"
FeedLink(url = url, title = declarers.firstOrNull { it.title != null }?.title ?: routeKey)
}
}
}OPML Feed List
GET /opml.xml — an OPML 2.0 subscription list of the site's feeds, scoped like the =<link rel="alternate">= autodiscovery: one <outline> per feed declared under the requesting SITE (declarers coalesced), xmlUrl absolute when the site's domain is known, htmlUrl pointing at the feed's parent page. Localhost gets every site's feeds (it browses the whole arcology) with in-server relative URLs; an unknown site 404s. The builder is top-level so tests call it directly (same pattern as loadNodeParseResult).
/**
,* OPML subscription list for a site: one <outline> per feed declared under
,* SITE (coalesced the same way as the autodiscovery links), xmlUrl absolute
,* when the site's domain is known. null site with localhost=false → 404 at
,* the route. Localhost lists every site's feeds. Top-level so tests can call
,* it directly (same pattern as loadNodeParseResult).
,*/
internal fun opmlFor(
site: String?,
localhost: Boolean,
feedRepository: FeedRepository?,
domainMap: DomainMap
): String? {
val feedRepo = feedRepository ?: return null
val allDeclarers = runBlocking {
if (localhost) feedRepo.getAllFeeds() else feedRepo.getFeedsBySite(site!!)
}
if (allDeclarers.isEmpty()) return null
val perFeed = allDeclarers.groupBy { it.routeKey }
fun outlineAttrs(declarers: List<FeedModel>, routeKey: String): String {
val path = declarers.first().path
val feedSite = declarers.first().site
val domain = domainMap.resolve(feedSite)
val xmlUrl = if (localhost || domain == null) "/$routeKey" else "http://$domain/$path"
val title = declarers.firstOrNull { it.title != null }?.title ?: routeKey
val htmlUrl = if (localhost || domain == null) "/${feedSite}/${path.removeSuffix(".xml")}" else "http://$domain/${path.removeSuffix(".xml")}"
return " <outline type=\"rss\" text=\"${opmlEscape(title)}\" title=\"${opmlEscape(title)}\" xmlUrl=\"${opmlEscape(xmlUrl)}\" htmlUrl=\"${opmlEscape(htmlUrl)}\"/>"
}
val body = if (localhost) {
// Localhost: every site's feeds, grouped by site for readability.
perFeed.entries
.groupBy { it.value.first().site }
.toSortedMap()
.flatMap { (_, feeds) -> feeds.sortedBy { it.key }.map { outlineAttrs(it.value, it.key) } }
} else {
perFeed.entries.sortedBy { it.key }.map { outlineAttrs(it.value, it.key) }
}
val ownerTitle = site?.let { domainMap.meta(it)?.title ?: it } ?: "The Arcology"
val sb = StringBuilder()
sb.append("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n")
sb.append("<opml version=\"2.0\">\n")
sb.append(" <head>\n")
sb.append(" <title>${opmlEscape(ownerTitle)} feeds</title>\n")
sb.append(" </head>\n")
sb.append(" <body>\n")
for (line in body) sb.append(line).append('\n')
sb.append(" </body>\n")
sb.append("</opml>\n")
return sb.toString()
}
internal fun opmlEscape(text: String): String =
text.replace("&", "&").replace("<", "<").replace(">", ">").replace("\"", """)/** Render the FeedXmlModel into the feed.peb template's output as a string. */
private fun feedXmlFor(model: FeedXmlModel): String {
val scope = mutableMapOf<String, Any>(
"title" to model.title,
"pageUrl" to model.pageUrl,
"feedUrl" to model.feedUrl,
"author" to model.author,
"updatedAt" to model.updatedAt,
"feedEntries" to model.entries.map { entry ->
mapOf(
"nodeId" to entry.nodeId,
"title" to entry.title,
"url" to entry.url,
"published" to entry.published,
"updated" to entry.updated,
"html" to entry.html
)
}
)
val writer = java.io.StringWriter()
pebbleEngine.getTemplate("feed.peb").evaluate(writer, scope)
return writer.toString()
}
private fun fileHashFor(entry: RouteEntry): String {
val orgFile = File(orgDir, entry.file)
return if (orgFile.exists()) {
val bytes = orgFile.readBytes()
val digest = java.security.MessageDigest.getInstance("SHA-256").digest(bytes)
digest.joinToString("") { "%02x".format(it) }
} else {
entry.file.hashCode().toString()
}
}
/**
* Digest of the per-heading backlink/reference data collected for a page.
* Mixed into the HtmlCache key and the weak ETag because inline backlinks and
* references depend on the *whole* link graph, not just this file's bytes — a
* new link from another file, or a re-index of a linking file, must invalidate
* the cached body even though this file's own hash is unchanged.
*/
private fun sidebarDigestFor(sidebar: SidebarData): String {
val sb = StringBuilder()
for ((nodeId, backlinks) in sidebar.backlinksByNode) {
sb.append(nodeId).append(':')
for (bl in backlinks) sb.append(bl.nodeId).append(',').append(bl.url).append(';')
sb.append('\n')
}
for ((nodeId, refs) in sidebar.referencesByNode) {
sb.append(nodeId).append(':')
for (ref in refs) sb.append(ref).append(';')
sb.append('\n')
}
val digest = java.security.MessageDigest.getInstance("SHA-256").digest(sb.toString().toByteArray())
return digest.joinToString("") { "%02x".format(it) }
}
/**
* Weak ETag for a rendered page: SHA-256 over the concatenated file hashes of
* every entry on the page plus the sidebar digest (per-heading backlink/ref
* data). Content-identical pages share an ETag; ConditionalHeaders converts
* If-None-Match into a 304.
*/
private fun weakEtagFor(entries: List<RouteEntry>, sidebar: SidebarData? = null): String {
val joined = entries.joinToString(",") { fileHashFor(it) } +
(sidebar?.let { sidebarDigestFor(it) } ?: "")
val digest = java.security.MessageDigest.getInstance("SHA-256").digest(joined.toByteArray())
val hash = digest.joinToString("") { "%02x".format(it) }
return "W/\"$hash\""
}
/**
,* Valid attachment names are strictly {64 lowercase hex}-{size}.{ext}: the regex
,* doubles as path-traversal protection since no '/', '.', or '..' can appear
,* outside the extension.
,*/
private val ATTACHMENT_NAME_REGEX = Regex("^[a-f0-9]{64}-\\d+\\.[a-z0-9]+$")
private fun attachmentFileFor(name: String): File? {
if (!ATTACHMENT_NAME_REGEX.matches(name)) return null
val file = File(attachmentDir, name)
return if (file.exists() && file.isFile) file else null
}
private fun contentTypeForAttachmentName(name: String): ContentType {
return when (name.substringAfterLast('.')) {
"jpg", "jpeg" -> ContentType.Image.JPEG
"png" -> ContentType.Image.PNG
"gif" -> ContentType.Image.GIF
"webp" -> ContentType("image", "webp")
"avif" -> ContentType("image", "avif")
"svg" -> ContentType.Image.SVG
"pdf" -> ContentType.Application.Pdf
"mp4" -> ContentType("video", "mp4")
"webm" -> ContentType("video", "webm")
"txt", "org" -> ContentType.Text.Plain
else -> ContentType.Application.OctetStream
}
}
/**
* Build a node-id → [RouteEntry] index from a flat list of routes. First-wins on
* duplicate node IDs, matching the old [RouteIndexBuilder.build] semantics. Built
* per-request from a fresh [PublishingRepository] query so the server sees routes
* added after startup without a restart.
*/
private fun buildRouteIndex(routes: List<RouteEntry>): Map<String, RouteEntry> {
val index = mutableMapOf<String, RouteEntry>()
for (entry in routes) {
if (!index.containsKey(entry.nodeId)) {
index[entry.nodeId] = entry
}
}
return index
}
/** Test-only forwarder so [ArcologyServerRouteLookupTest] can exercise the private [servePath] without widening its visibility. */
internal fun servePathPublic(path: String, site: String?): PageModel? = servePath(path, site)
/** Test-only forwarder so [ArcologyServerRouteLookupTest] can exercise the private [servePathLocalhost] without widening its visibility. */
internal fun servePathLocalhostPublic(fullKey: String): PageModel? = servePathLocalhost(fullKey)
private fun linkResolverFor(site: String?, localhost: Boolean, publishedIndex: Map<String, RouteEntry>): LinkResolver {
return CrossDomainLinkResolver(
routeIndex = publishedIndex,
domainMap = domainMap,
repository = repository,
currentSite = site,
localhost = localhost,
publishingRepository = publishingRepository
)
}
private fun resolveTemplate(entry: RouteEntry): String {
val prop = runBlocking {
repository.getHeadingProperty(entry.nodeId, "ARCOLOGY_PAGE_TEMPLATE")
?: repository.getFileProperties(entry.file)["ARCOLOGY_PAGE_TEMPLATE"]
}
return when (prop?.trim()?.lowercase()) {
"wide" -> "wide.peb"
"topic" -> "topic.peb"
else -> "page.peb"
}
}
private fun buildSitemapModel(localhost: Boolean): Map<String, Any> {
val allRoutes = runBlocking {
if (localhost) publishingRepository.getAllRoutes() else publishingRepository.getPublishedRoutes()
}
val bySite: Map<String, List<RouteEntry>> = allRoutes.groupBy { it.site }
val sitesList = domainMap.allSites().sorted().map { siteKey ->
val meta = domainMap.meta(siteKey)
val routes = bySite[siteKey]?.map { it.path } ?: emptyList()
mapOf(
"key" to siteKey,
"title" to (meta?.title ?: siteKey),
"routes" to routes.map { mapOf("path" to it, "count" to 1) }
)
}
return mapOf(
"sites" to sitesList,
"localhost" to localhost
)
}
private val sitesCssMemo: String by lazy { buildSitesCss() }
private fun buildSitesCss(): String {
val sb = StringBuilder()
for (siteKey in domainMap.allSites()) {
val meta = domainMap.meta(siteKey) ?: continue
val color = meta.linkColor ?: continue
for (domain in meta.domains) {
sb.append("a[href*=\"//$domain\"] {\n")
sb.append(" border-radius: 0.25em;\n")
sb.append(" padding: 0.1em;\n")
sb.append(" background-color: ${color}66;\n")
sb.append("}\n")
sb.append("a[href*=\"//$domain\"]:hover {\n")
sb.append(" background-color: ${color}FF !important;\n")
sb.append("}\n")
}
}
sb.append("a[href*=\"/404\"] {\n")
sb.append(" text-decoration: underline wavy var(--alert);\n")
sb.append(" color: var(--alert) !important;\n")
sb.append("}\n")
return sb.toString()
}Utility Helper Functions
import java.time.Instant as JavaInstantThe instrumentation point: logRequest runs at every terminal response, so it carries the duration (start), classifies the user-agent and referer, increments arcology_requests_total, and emits one structured JSON access line (AccessLogEvent). logError emits a ~warn~-level JSON line via AccessLog.error.
private fun logRequest(
call: ApplicationCall,
start: TimeSource.Monotonic.ValueTimeMark,
method: String,
uri: String,
host: String?,
status: HttpStatusCode,
extra: String = ""
) {
val duration = start.elapsedNow()
val site = resolveSiteFromHost(host)
val path = normalizePathLabel(uri)
val ua = call.request.headers["User-Agent"]
val referer = call.request.headers["Referer"]
// TCP peer address; behind nginx this is the proxy — X-Forwarded-For
// carries the client chain, first hop = client.
val remoteHost = call.request.local.remoteHost
val forwardedFor = call.request.headers["X-Forwarded-For"]
metrics?.incRequest(site, path, status.value, UserAgentBucketer.classify(ua), RefererClassifier.classify(referer, host))
AccessLog.request(AccessLogEvent(
timestamp = JavaInstant.now().toString(),
method = method,
path = path,
host = host,
site = site,
status = status.value,
durationMs = duration.inWholeMilliseconds,
uaClass = UserAgentBucketer.classify(ua).name,
refererType = RefererClassifier.classify(referer, host).name,
remoteHost = remoteHost,
forwardedFor = forwardedFor,
userAgent = ua,
extra = extra
))
}
private fun logError(uri: String, host: String?, message: String, throwable: Throwable? = null) {
AccessLog.error(uri, host, message, throwable)
}/** Delegates to the top-level [[findSectionByNodeId]] (hoisted to NodeRendering.kt for the FeedPublisher). */
internal fun findSectionByNodeId(sections: List<xyz.lepisma.orgmode.OrgSection>, nodeId: String): xyz.lepisma.orgmode.OrgSection? =
computer.whatthefuck.arcology.publishing.findSectionByNodeId(sections, nodeId)
private fun htmlEscape(text: String): String {
return text.replace("&", "&").replace("<", "<").replace(">", ">")
}Tests: Nested Section Lookup
Tests that findSectionByNodeId correctly finds headings nested at arbitrary depths in the org document AST. This covers the bug where level-3 headings with ARCOLOGY_KEY properties were invisible because renderNode only scanned top-level sections.
package computer.whatthefuck.arcology.publishing
import computer.whatthefuck.arcology.parser.OrgFileParser
import io.pebbletemplates.pebble.PebbleEngine
import io.pebbletemplates.pebble.loader.ClasspathLoader
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertNull
class NestedSectionLookupTest {
private val parser = OrgFileParser()
private val nestedOrg = """
:PROPERTIES:
:ID: file-level-id
:END:
#+title: Places
,* Japan
:PROPERTIES:
:ID: japan-id
:END:
Some content about Japan.
,** Kyoto
:PROPERTIES:
:ID: kyoto-id
:END:
,*** Wazuka
:PROPERTIES:
:ID: wazuka-id
:ARCOLOGY_KEY: lionsrear/wazuka
:END:
Wazuka is a tea town.
,** Tokyo
:PROPERTIES:
:ID: tokyo-id
:END:
Big city.
""".trimIndent() @Test
fun `finds top-level section by node id`() {
val result = parser.parseFileContent("places.org", nestedOrg, kotlin.time.Instant.fromEpochSeconds(0))
val document = (result as computer.whatthefuck.arcology.parser.ParseResult.Success).document!!
val server = createMinimalServer(document)
val section = server.findSectionByNodeId(document.content, "japan-id")
assertNotNull(section)
assertEquals("Japan", section.heading.title.items.filterIsInstance<xyz.lepisma.orgmode.OrgInlineElem.Text>().joinToString("") { it.text }.trim())
} @Test
fun `finds level-2 nested section by node id`() {
val result = parser.parseFileContent("places.org", nestedOrg, kotlin.time.Instant.fromEpochSeconds(0))
val document = (result as computer.whatthefuck.arcology.parser.ParseResult.Success).document!!
val server = createMinimalServer(document)
val section = server.findSectionByNodeId(document.content, "kyoto-id")
assertNotNull(section)
} @Test
fun `finds level-3 deeply nested section by node id`() {
val result = parser.parseFileContent("places.org", nestedOrg, kotlin.time.Instant.fromEpochSeconds(0))
val document = (result as computer.whatthefuck.arcology.parser.ParseResult.Success).document!!
val server = createMinimalServer(document)
val section = server.findSectionByNodeId(document.content, "wazuka-id")
assertNotNull(section)
val title = section.heading.title.items.filterIsInstance<xyz.lepisma.orgmode.OrgInlineElem.Text>().joinToString("") { it.text }.trim()
assertEquals("Wazuka", title)
} @Test
fun `finds sibling at same nesting level`() {
val result = parser.parseFileContent("places.org", nestedOrg, kotlin.time.Instant.fromEpochSeconds(0))
val document = (result as computer.whatthefuck.arcology.parser.ParseResult.Success).document!!
val server = createMinimalServer(document)
val section = server.findSectionByNodeId(document.content, "tokyo-id")
assertNotNull(section)
} @Test
fun `returns null for non-existent node id`() {
val result = parser.parseFileContent("places.org", nestedOrg, kotlin.time.Instant.fromEpochSeconds(0))
val document = (result as computer.whatthefuck.arcology.parser.ParseResult.Success).document!!
val server = createMinimalServer(document)
val section = server.findSectionByNodeId(document.content, "does-not-exist")
assertNull(section)
} private fun createMinimalServer(document: xyz.lepisma.orgmode.OrgDocument): ArcologyServer {
val repo = object : computer.whatthefuck.arcology.database.RoamRepository {
override suspend fun getAllFiles() = emptyList<computer.whatthefuck.arcology.domain.OrgFile>()
override suspend fun getFileByPath(path: String) = null
override suspend fun insertFile(file: computer.whatthefuck.arcology.domain.OrgFile) {}
override suspend fun deleteFile(path: String) {}
override suspend fun getAllNodes() = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun getNodeById(id: String) = null
override suspend fun getNodesByFile(file: String) = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun getRecentNodes(limit: Long) = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun searchNodesByTitle(query: String) = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun searchNodesByFilePath(query: String) = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun insertNode(node: computer.whatthefuck.arcology.domain.OrgNode) {}
override suspend fun deleteNode(id: String) {}
override suspend fun insertNodeAncestor(nodeId: String, ancestorId: String) {}
override suspend fun deleteNodeAncestorsByFile(file: String) {}
override suspend fun getNodeAncestors(nodeId: String) = emptyList<String>()
override suspend fun getLinksFrom(nodeId: String) = emptyList<computer.whatthefuck.arcology.domain.OrgLink>()
override suspend fun getLinksTo(nodeId: String) = emptyList<computer.whatthefuck.arcology.domain.OrgLink>()
override suspend fun insertLink(link: computer.whatthefuck.arcology.domain.OrgLink) {}
override suspend fun deleteLinksByFile(file: String) {}
override suspend fun getTagsByNode(nodeId: String) = emptyList<String>()
override suspend fun getTagsByNodes(nodeIds: List<String>) = emptyMap<String, List<String>>()
override suspend fun getNodesByTag(tag: String) = emptyList<String>()
override suspend fun getAllTags() = emptyList<String>()
override suspend fun getTagsWithCount() = emptyList<Pair<String, Long>>()
override suspend fun insertTag(tag: computer.whatthefuck.arcology.domain.OrgTag) {}
override suspend fun deleteTagsByFile(file: String) {}
override suspend fun getRefsByNode(nodeId: String) = emptyList<Pair<String, String>>()
override suspend fun getRefsByNodes(nodeIds: List<String>) = emptyMap<String, List<String>>()
override suspend fun insertRef(ref: computer.whatthefuck.arcology.domain.OrgRef) {}
override suspend fun deleteRefsByFile(file: String) {}
override suspend fun getAliasesByNode(nodeId: String) = emptyList<String>()
override suspend fun getNodesByAlias(alias: String) = emptyList<String>()
override suspend fun insertAlias(alias: computer.whatthefuck.arcology.domain.OrgAlias) {}
override suspend fun deleteAliasesByFile(file: String) {}
override suspend fun getHeadingProperties(nodeId: String) = emptyMap<String, String?>()
override suspend fun getHeadingProperty(nodeId: String, key: String) = null
override suspend fun insertHeadingProperty(property: computer.whatthefuck.arcology.domain.NodeProperty) {}
override suspend fun deleteHeadingProperty(nodeId: String, key: String) {}
override suspend fun deleteHeadingPropertiesByFile(file: String) {}
override suspend fun getFileProperties(file: String) = emptyMap<String, String?>()
override suspend fun insertFileProperty(property: computer.whatthefuck.arcology.domain.FileProperty) {}
override suspend fun getNodesByPropertyKey(key: String) = emptyList<Pair<String, String?>>()
override suspend fun getNodesWithLocation() = emptyList<Pair<computer.whatthefuck.arcology.domain.OrgNode, computer.whatthefuck.arcology.domain.GeoCoordinate>>()
override suspend fun searchNodes(query: String) = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun searchNodesByTitles(query: String, limit: Long) = emptyList<String>()
override suspend fun searchNodesByContent(query: String, limit: Long) = emptyList<String>()
override suspend fun searchNodesByTitlesBM25(query: String, limit: Long) = emptyList<computer.whatthefuck.arcology.domain.FtsSearchResult>()
override suspend fun searchNodesByContentBM25(query: String, limit: Long) = emptyList<computer.whatthefuck.arcology.domain.FtsSearchResult>()
override suspend fun insertNodeToFts(node: computer.whatthefuck.arcology.domain.OrgNode, tags: List<String>, aliases: List<String>, content: String) {}
override suspend fun updateNodeInFts(node: computer.whatthefuck.arcology.domain.OrgNode, tags: List<String>, aliases: List<String>, content: String) {}
override suspend fun deleteNodeFromFts(nodeId: String) {}
override suspend fun getFtsContentCount() = 0L
override suspend fun getFailedFile(path: String) = null
override suspend fun getAllFailedFiles() = emptyList<computer.whatthefuck.arcology.domain.FailedFile>()
override suspend fun insertFailedFile(failedFile: computer.whatthefuck.arcology.domain.FailedFile) {}
override suspend fun updateFailedFile(path: String, errorMessage: String, fileHash: String?) {}
override suspend fun deleteFailedFile(path: String) {}
override suspend fun deleteAllFailedFiles() {}
override suspend fun insertFtsStaging(nodeId: String, title: String, tags: String, aliases: String, content: String) {}
override suspend fun getFtsStagingBatch(limit: Long, offset: Long) = emptyList<computer.whatthefuck.arcology.domain.FtsStagingEntry>()
override suspend fun getFtsStagingCount() = 0L
override suspend fun clearFtsStaging() {}
override suspend fun clearAllTitleFts() {}
override suspend fun clearAllContentFts() {}
override suspend fun bulkInsertTitleFts(entries: List<computer.whatthefuck.arcology.domain.FtsTitleEntry>) {}
override suspend fun bulkInsertContentFts(entries: List<computer.whatthefuck.arcology.domain.FtsContentEntry>) {}
override suspend fun <T> transaction(block: suspend () -> T): T = block()
override suspend fun getAttachmentsByNode(nodeId: String) = emptyList<computer.whatthefuck.arcology.domain.OrgAttachment>()
override suspend fun getAttachmentsByType(type: String) = emptyList<computer.whatthefuck.arcology.domain.OrgAttachment>()
override suspend fun insertAttachment(attachment: computer.whatthefuck.arcology.domain.OrgAttachment) {}
override suspend fun deleteAttachmentsByNode(nodeId: String) {}
override suspend fun getAllDiscoveryCache() = emptyList<computer.whatthefuck.arcology.domain.FileDiscoveryCache>()
override suspend fun getDiscoveryCacheByUri(uri: String) = null
override suspend fun getDiscoveryCacheModifiedSince(timestamp: Long) = emptyList<computer.whatthefuck.arcology.domain.FileDiscoveryCache>()
override suspend fun insertDiscoveryCache(entry: computer.whatthefuck.arcology.domain.FileDiscoveryCache) {}
override suspend fun updateDiscoveryCacheHash(uri: String, contentHash: String?, lastSeenAt: Long) {}
override suspend fun deleteDiscoveryCacheByUri(uri: String) {}
override suspend fun deleteStaleDiscoveryCache(beforeTimestamp: Long) {}
override suspend fun clearDiscoveryCache() {}
override suspend fun countDiscoveryCache() = 0L
}
return ArcologyServer(
repository = repo,
domainMap = DomainMap.fromMap(emptyMap()),
htmlCache = HtmlCache(java.io.File(System.getProperty("java.io.tmpdir"))),
orgDir = "/tmp",
parser = parser,
sidebarService = SidebarService(repo, DomainMap.fromMap(emptyMap())),
pebbleEngine = PebbleEngine.Builder()
.loader(ClasspathLoader().apply { prefix = "templates/" })
.autoEscaping(true)
.build(),
publishingRepository = FakePublishingRepository(emptyList())
)
}}Per-request link resolution
A CrossDomainLinkResolver is constructed per request with the current site (derived from the Host header) and localhost flag, then wrapped in an OrgHtmlRenderer. This ensures =...= links are rewritten to relative URLs for same-site, absolute, schema-less //<domain>/... URLs for cross-site, and =/404?node=<id>= for unpublished nodes. The resolver is cheap to construct; the routeIndex it consults is built per request from a fresh [PublishingRepository] query (see [buildRouteIndex]) so the server picks up routes added to the database after startup without a restart, and the domainMap is shared across requests since it only changes when domains.json changes.
In localhost mode (when the Host header is localhost or localhost:PORT), the URL path is the full ARCOLOGY_KEY (SITE/path). For example, localhost:8080/garden/index matches the route with key garden/index. This lets you browse all sites from a single server without virtual host configuration.
When the Host header matches a domain in the DomainMap, only the path portion (after the SITE prefix) is used for lookup — this is the production multi-domain mode.
Noweb Construction
import computer.whatthefuck.arcology.database.RoamRepository
import computer.whatthefuck.arcology.parser.OrgFileParser
import io.pebbletemplates.pebble.PebbleEnginePageModel and Sidebar Service
The PageModel is the view model passed to Pebble templates. SidebarService computes the sidebar data (backlinks, tags, references, keywords) for a set of node IDs by querying RoamRepository. Backlinks are node-granular (getLinksTo per node), deduplicated by source node, and only include sources that are published (present in the per-request publishedIndex passed in by the caller, which contains only non-draft, non-archived, =ARCOLOGY_KEY=-bearing nodes) and not tagged with any tag in the shared EXCLUDE_TAGS set (noexport NOEXPORT ARCHIVE). Unpublished sources are skipped entirely rather than rendered as /404 links. The EXCLUDE_TAGS constant is shared with ArcologyPublishingPlugin and OrgHtmlRenderer so the three call sites cannot drift. SidebarService itself holds no route state; the caller threads the current publishedIndex through [collectSidebar] on every request.
#/sites.css is served as a memoized string built from DomainMap site metadata. It changes only when domains.json changes (rare, requires rebuild), so the in-process memoize is sufficient — no disk cache needed.
Data classes
The BacklinkSnippet data class (source node ID, title, URL) is defined in OrgHtmlRenderer (commonMain) so the renderer can reference it without depending on jvmMain or RoamRepository. The SidebarService groups these by target node ID and passes the grouping to the renderer; the flat list (for the sidebar panel) is derived from the same grouped map.
package computer.whatthefuck.arcology.publishing
import computer.whatthefuck.arcology.database.RoamRepository
import kotlinx.coroutines.runBlocking
import kotlinx.serialization.Serializable
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Jsondata class SiteModel(
val key: String,
val title: String?,
val cssFile: String?,
val hljsTheme: String?,
val domain: String?
)
data class FeedLink(val url: String, val title: String)
data class KeywordEntry(val key: String, val value: String)
data class SidebarData(
val backlinks: List<BacklinkSnippet>,
val backlinksByNode: Map<String, List<BacklinkSnippet>>,
val tags: List<String>,
val references: List<String>,
val referencesByNode: Map<String, List<String>>,
val keywords: List<KeywordEntry>
)
data class PageModel(
val site: SiteModel,
val headTitle: String,
val pageTitle: String,
val htmlContent: String,
val sidenotesHtml: String,
val headings: List<HeadingLink>,
val backlinks: List<BacklinkSnippet>,
val tags: List<String>,
val references: List<String>,
val keywords: List<KeywordEntry>,
val feeds: List<FeedLink>,
val allowCrawl: Boolean,
val template: String = "page.peb",
val etag: String? = null
) {
companion object {
val NoFeeds: List<FeedLink> = emptyList()
}
}SidebarService
class SidebarService(
private val repository: RoamRepository,
private val domainMap: DomainMap
) {
private val excludeTags = EXCLUDE_TAGS
suspend fun collect(nodeIds: List<String>, filePath: String, currentSite: String?, publishedIndex: Map<String, RouteEntry>): SidebarData {
val backlinksByNode = mutableMapOf<String, MutableList<BacklinkSnippet>>()
val seenBacklinkSources = mutableSetOf<String>()
val sourceTagsByNode: Map<String, List<String>> = repository.getTagsByNodes(nodeIds)
val excludeNodeIds = nodeIds.filter { nid ->
(sourceTagsByNode[nid] ?: emptyList()).any { it in excludeTags }
}.toSet()
for (nodeId in nodeIds) {
if (nodeId in excludeNodeIds) continue
val linksTo = repository.getLinksTo(nodeId)
for (link in linksTo) {
val sourceId = link.fromNode
if (seenBacklinkSources.contains(sourceId)) continue
// Only include backlinks from published nodes — the publishedIndex
// contains only published (non-draft, non-archived, ARCOLOGY_KEY-bearing) nodes.
val entry = publishedIndex[sourceId] ?: continue
val sourceTags = repository.getTagsByNode(sourceId)
if (sourceTags.any { it in excludeTags }) continue
seenBacklinkSources.add(sourceId)
val title = repository.getNodeById(sourceId)?.title ?: sourceId
val url = if (currentSite != null && entry.site == currentSite) {
// Same-site: relative URL (lets the browser keep the current domain)
"/${entry.path}"
} else {
// Cross-site or unknown current site: absolute URL to the source's domain
val domain = domainMap.resolve(entry.site)
if (domain != null) "//$domain/${entry.path}" else "/${entry.path}"
}
backlinksByNode.getOrPut(nodeId) { mutableListOf() }
.add(BacklinkSnippet(nodeId = sourceId, title = title, url = url))
}
}
val tags = nodeIds.flatMap { repository.getTagsByNode(it) }.distinct().filterNot { it in excludeTags }
val referencesByNode: Map<String, List<String>> = repository.getRefsByNodes(nodeIds)
val references = referencesByNode.values.flatten().distinct()
val fileProps = repository.getFileProperties(filePath)
val keywords = fileProps.entries
.filter { it.key.startsWith("ARCOLOGY_") }
.map { KeywordEntry(it.key, it.value ?: "") }
return SidebarData(
backlinks = backlinksByNode.values.flatten(),
backlinksByNode = backlinksByNode,
tags = tags,
references = references,
referencesByNode = referencesByNode,
keywords = keywords
)
}
}HtmlCache
File-system cache for rendered HTML. Files are named {nodeId}.{hash}.html. On lookup, if the file exists and the hash matches the current content hash, the cached HTML is returned. Otherwise, the entry is stale and must be re-rendered.
The hash component is caller-supplied, and the server mixes in more than the org file's bytes: servePath and servePathLocalhost append a sidebar digest — a SHA-256 over the per-heading backlink/reference data — to the file hash before consulting the cache. Inline backlinks and references depend on the whole link graph, so a new == link from a different file, or a re-index of a linking file, must invalidate the cached body even though this file's own content is unchanged. Without the digest, a cache hit would serve HTML rendered before the new link existed, and the weak ETag (which mixes the same digest via weakEtagFor) would keep confirming the stale body as fresh.
Cache envelope: html + sidenotes + headings
A bare HTML body is not enough to reconstruct a full page on a cache hit: the inline TOC (and other consumeHeadings() consumers) and the sidenotes are produced by the render pass, and on a cache hit renderNodeBody never runs, so consumeHeadings() returns an empty list and renderSidenotes() returns empty. Cache entries are therefore a JSON envelope — {"html": …, "sidenotes": …, "headings": [HeadingLink…]} — so servePath can hydrate all three from the cache. getHtml / putHtml read and write the legacy bare-HTML shape for compatibility with entries written before the envelope existed: reads fall back to {"html": blob} when no envelope is parseable.
/**
* Cache entry envelope: render-pass artifacts that a bare body cannot carry —
* the TOC heading list and sidenotes are produced during the render pass, so
* they must be stored alongside the HTML or a cache hit loses them.
*/
@Serializable
data class CachedPage(
val html: String,
val sidenotes: String = "",
val headings: List<HeadingLink> = emptyList()
)
val cacheJson = Json { ignoreUnknownKeys = true }Preamble and core cache operations
package computer.whatthefuck.arcology.publishing
import java.io.File
import java.nio.file.Path
import java.security.MessageDigestclass HtmlCache(val cacheDir: File) {
init {
if (!cacheDir.exists()) {
cacheDir.mkdirs()
}
}
fun get(nodeId: String, contentHash: String): String? {
val file = cacheFile(nodeId, contentHash)
if (!file.exists()) {
invalidateOldVersions(nodeId, contentHash)
return null
}
return file.readText()
}
fun put(nodeId: String, contentHash: String, html: String) {
invalidateOldVersions(nodeId, contentHash)
val file = cacheFile(nodeId, contentHash)
file.writeText(html)
}
/**
* Fetch a cached page envelope. Falls back to the legacy bare-HTML shape —
* entries written by older builds store the HTML blob directly; those are
* wrapped as {"html": blob} with empty sidenotes and no headings, so the
* caller re-renders TOC/sidenotes or lives without them for one generation.
*/
fun getPage(nodeId: String, contentHash: String): CachedPage? = when (val raw = get(nodeId, contentHash)) {
null -> null
else -> try {
cacheJson.decodeFromString<CachedPage>(raw)
} catch (e: Exception) {
// Legacy bare-HTML entry: hydrate html only; it re-saves as an
// envelope on the caller's next put.
CachedPage(html = raw)
}
}
fun putPage(nodeId: String, contentHash: String, page: CachedPage) {
put(nodeId, contentHash, cacheJson.encodeToString(page))
}
fun invalidate(nodeId: String) {
val prefix = "$nodeId."
cacheDir.listFiles()?.forEach { file ->
if (file.name.startsWith(prefix) && file.name.endsWith(".html")) {
file.delete()
}
}
val sidebarPrefix = "sidebar-$nodeId-"
cacheDir.listFiles()?.forEach { file ->
if (file.name.startsWith(sidebarPrefix) && file.name.endsWith(".html")) {
file.delete()
}
}
}
fun getSidebar(nodeId: String, contentSha256: String): String? {
val file = sidebarFile(nodeId, contentSha256)
if (!file.exists()) return null
return file.readText()
}
fun putSidebar(nodeId: String, contentSha256: String, sidebarHtml: String) {
// Best-effort cleanup of stale sidebar entries for this node
val safeNodeId = nodeId.replace(Regex("[^a-zA-Z0-9._-]"), "_")
val stalePrefix = "sidebar-$safeNodeId-"
cacheDir.listFiles()?.forEach { file ->
if (file.name.startsWith(stalePrefix) && file.name.endsWith(".html") && file.name != sidebarFile(nodeId, contentSha256).name) {
file.delete()
}
}
sidebarFile(nodeId, contentSha256).writeText(sidebarHtml)
}
fun clear() {
cacheDir.listFiles()?.forEach { it.delete() }
}Private file-path helpers
private fun cacheFile(nodeId: String, contentHash: String): File {
val safeNodeId = nodeId.replace(Regex("[^a-zA-Z0-9._-]"), "_")
return File(cacheDir, "$safeNodeId.$contentHash.html")
}
private fun sidebarFile(nodeId: String, contentSha256: String): File {
val safeNodeId = nodeId.replace(Regex("[^a-zA-Z0-9._-]"), "_")
return File(cacheDir, "sidebar-$safeNodeId-$contentSha256.html")
}
private fun invalidateOldVersions(nodeId: String, currentHash: String) {
val prefix = nodeId.replace(Regex("[^a-zA-Z0-9._-]"), "_") + "."
cacheDir.listFiles()?.forEach { file ->
if (file.name.startsWith(prefix) && file.name.endsWith(".html") && !file.name.contains(".$currentHash.")) {
file.delete()
}
}
}
}Tests: HtmlCache
package computer.whatthefuck.arcology.publishing
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
import kotlin.test.assertNotNull
import java.io.File
import java.nio.file.Files
class HtmlCacheTest {
private fun createTempCache(): HtmlCache {
val tempDir = Files.createTempDirectory("arcology-cache-test").toFile()
return HtmlCache(tempDir)
} @Test
fun `put and get returns cached html`() {
val cache = createTempCache()
cache.put("node1", "hash1", "<p>cached</p>")
val result = cache.get("node1", "hash1")
assertEquals("<p>cached</p>", result)
} @Test
fun `get returns null for missing entry`() {
val cache = createTempCache()
val result = cache.get("node1", "hash1")
assertNull(result)
} @Test
fun `get returns null when hash differs`() {
val cache = createTempCache()
cache.put("node1", "hash1", "<p>old</p>")
val result = cache.get("node1", "hash2")
assertNull(result)
} @Test
fun `put with new hash replaces old version`() {
val cache = createTempCache()
cache.put("node1", "hash1", "<p>old</p>")
cache.put("node1", "hash2", "<p>new</p>")
val result = cache.get("node1", "hash2")
assertEquals("<p>new</p>", result)
val oldResult = cache.get("node1", "hash1")
assertNull(oldResult)
} @Test
fun `invalidate removes all versions for a node`() {
val cache = createTempCache()
cache.put("node1", "hash1", "<p>v1</p>")
cache.put("node1", "hash2", "<p>v2</p>")
cache.invalidate("node1")
assertNull(cache.get("node1", "hash1"))
assertNull(cache.get("node1", "hash2"))
} @Test
fun `clear removes all entries`() {
val cache = createTempCache()
cache.put("node1", "hash1", "<p>1</p>")
cache.put("node2", "hash2", "<p>2</p>")
cache.clear()
assertNull(cache.get("node1", "hash1"))
assertNull(cache.get("node2", "hash2"))
} @Test
fun `sanitizes node IDs with special characters`() {
val cache = createTempCache()
cache.put("node/with/slashes", "hash1", "<p>safe</p>")
val result = cache.get("node/with/slashes", "hash1")
assertNotNull(result)
assertEquals("<p>safe</p>", result)
} @Test
fun `putPage and getPage round-trip the full envelope`() {
val cache = createTempCache()
val page = CachedPage(
html = "<p>body</p>",
sidenotes = "<aside>sidenote</aside>",
headings = listOf(HeadingLink(2, "My Heading", "my-heading"))
)
cache.putPage("node1", "hash1", page)
val result = cache.getPage("node1", "hash1")
assertNotNull(result)
assertEquals("<p>body</p>", result.html)
assertEquals("<aside>sidenote</aside>", result.sidenotes)
assertEquals(1, result.headings.size)
assertEquals("my-heading", result.headings[0].anchor)
} @Test
fun `getPage falls back on legacy bare-HTML entry`() {
val cache = createTempCache()
// Legacy entries (older builds) store the raw HTML blob directly.
cache.put("node1", "hash1", "<p>old-style body</p>")
val result = cache.getPage("node1", "hash1")
assertNotNull(result)
assertEquals("<p>old-style body</p>", result.html)
assertEquals("", result.sidenotes)
assertEquals(0, result.headings.size)
} @Test
fun `getPage returns null for missing entry`() {
val cache = createTempCache()
assertNull(cache.getPage("missing", "hash"))
}}CrossDomainLinkResolver
Implements LinkResolver using a reverse route index (node ID → RouteEntry), the DomainMap, and the RoamRepository ancestor closure. Resolves =...= links to URLs appropriate for the site currently being served.
Resolution algorithm
Given a target node ID:
Direct hit: if the target has its own
ARCOLOGY_KEY(it's in the reverse route index), build a URL to its page — no anchor.Ancestor anchor: otherwise, walk the target's ancestors (via
getNodeAncestors, which is self-inclusive but self was already checked in step 1). The first ancestor that has a published route gets a URL to its page with#<targetId>appended as the anchor.404: if no ancestor is published, return
null. The renderer then emits =<a href="/404?node=<id>" class="unpublished">=.
URL building
The URL form depends on the serving context:
Localhost mode (browse-all-sites): always
/SITE/path— the fullARCOLOGY_KEY, since localhost routes by full key.Production, same site:
/path— a relative URL, so links stay on the same domain.Production, cross site:
//<domain>/path— an absolute URL viaDomainMap.resolve.Unknown host (=currentSite == null=): absolute URL using the entry's own domain — safest for off-host scrapers and feed readers.
Reverse route index
The reverse index is built once at server startup (in ServeCommand) by flattening RouteTable into Map<String, RouteEntry>. If multiple entries share a node ID (a heading published under multiple keys), the first one encountered wins; this is rare and the duplication is usually intentional cross-posting.
Attachment resolution
file: / attachment: link targets resolve against published_attachments (via the PublishingRepository, which is optional so test constructors can omit it): first an exact source_path match, then a basename match for ./relative, =~/=-absolute, and bare attachment:foo.jpg targets. Rows are grouped by source_hash; on basename collisions the lexicographically-first source path wins deterministically. The smallest size variant becomes the url (the initial <img> src), the largest — if different — provides the HTMX fragment URL for the swap.
package computer.whatthefuck.arcology.publishing
import computer.whatthefuck.arcology.database.RoamRepository
import kotlinx.coroutines.runBlockingclass CrossDomainLinkResolver(
private val routeIndex: Map<String, RouteEntry>,
private val domainMap: DomainMap,
private val repository: RoamRepository,
private val currentSite: String?,
private val localhost: Boolean,
private val publishingRepository: PublishingRepository? = null
) : LinkResolver {
override fun resolveNodeId(nodeId: String): String? {
val direct = routeIndex[nodeId]
if (direct != null) {
return buildUrl(direct, anchor = null)
}
val ancestors = runBlocking { repository.getNodeAncestors(nodeId) }
for (ancestorId in ancestors) {
if (ancestorId == nodeId) continue
val ancestorEntry = routeIndex[ancestorId]
if (ancestorEntry != null) {
return buildUrl(ancestorEntry, anchor = nodeId)
}
}
return null
}
override fun resolveFilePath(filePath: String): String? {
return null
}
override fun resolveAttachment(filePath: String): AttachmentRef? {
val repo = publishingRepository ?: return null
val normalized = filePath.replace('\\', '/')
.removePrefix("./")
.removePrefix("../")
val rows = runBlocking {
val exact = repo.getAttachmentsBySourcePath(normalized)
if (exact.isNotEmpty()) {
exact
} else {
repo.getAttachmentsByBasename(normalized.substringAfterLast('/'))
}
}
if (rows.isEmpty()) return null
val variants = rows
.groupBy { it.sourceHash }
.minByOrNull { (_, groupRows) -> groupRows.first().sourcePath }
?.value
?: return null
val sorted = variants.sortedBy { it.size }
val small = sorted.first()
val large = sorted.lastOrNull { it.size != small.size }
return AttachmentRef(
url = "/attachment/${small.urlName}",
largeHtmlUrl = large?.let { "/arcology/attachment/${it.urlName}/html" },
isImage = small.type in ATTACHMENT_IMAGE_EXTENSIONS
)
}
private fun buildUrl(entry: RouteEntry, anchor: String?): String {
val base = if (localhost) {
"/${entry.site}/${entry.path}"
} else if (currentSite == entry.site) {
"/${entry.path}"
} else {
val domain = domainMap.resolve(entry.site)
if (domain != null) {
"//$domain/${entry.path}"
} else {
"/${entry.path}"
}
}
return if (anchor != null) "$base#$anchor" else base
}
}object RouteIndexBuilder {
fun build(routeTable: RouteTable): Map<String, RouteEntry> {
val index = mutableMapOf<String, RouteEntry>()
for (entries in routeTable.routes.values) {
for (entry in entries) {
if (!index.containsKey(entry.nodeId)) {
index[entry.nodeId] = entry
}
}
}
return index
}
}NEXT change http back to https in the link resolver...
Tests: CrossDomainLinkResolver
Tests the resolution algorithm: direct hits, ancestor anchors, cross-domain URL building, localhost mode, and 404 fallback. Uses a fake RoamRepository that returns a canned ancestor map.
package computer.whatthefuck.arcology.publishing
import computer.whatthefuck.arcology.database.RoamRepository
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
import kotlin.test.assertNotNull
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class CrossDomainLinkResolverTest {
private val domainMap = DomainMap.fromMap(mapOf(
"lionsrear" to "thelionsrear.com",
"garden" to "arcology.garden"
))
private fun resolver(
routeIndex: Map<String, RouteEntry>,
ancestors: Map<String, List<String>> = emptyMap(),
currentSite: String? = "lionsrear",
localhost: Boolean = false,
attachments: List<PublishedAttachment> = emptyList()
): CrossDomainLinkResolver {
val repo = AncestorFakeRepository(ancestors)
return CrossDomainLinkResolver(
routeIndex, domainMap, repo, currentSite, localhost,
publishingRepository = FakeAttachmentLookup(attachments)
)
}
private fun attachment(
nodeId: String, sourcePath: String, hash: String, size: Int, type: String,
file: String = "test.org"
): PublishedAttachment {
return PublishedAttachment(
nodeId = nodeId, sourcePath = sourcePath, sourceHash = hash,
size = size, type = type, file = file
)
}
private fun entry(nodeId: String, site: String, path: String): RouteEntry {
return RouteEntry(
nodeId = nodeId, site = site, path = path, file = "test.org",
title = "Test", timestamp = null, isDraft = false, isArchived = false
)
} @Test
fun `direct hit on same site returns relative path`() {
val index = mapOf("nodeA" to entry("nodeA", "lionsrear", "index"))
val r = resolver(index)
assertEquals("/index", r.resolveNodeId("nodeA"))
} @Test
fun `direct hit on cross site returns absolute url`() {
val index = mapOf("nodeB" to entry("nodeB", "garden", "plants/fern"))
val r = resolver(index, currentSite = "lionsrear")
assertEquals("//arcology.garden/plants/fern", r.resolveNodeId("nodeB"))
} @Test
fun `direct hit in localhost mode returns full key`() {
val index = mapOf("nodeA" to entry("nodeA", "lionsrear", "index"))
val r = resolver(index, localhost = true)
assertEquals("/lionsrear/index", r.resolveNodeId("nodeA"))
} @Test
fun `subheading under published ancestor gets anchor on same site`() {
val index = mapOf("parent" to entry("parent", "lionsrear", "plants"))
val ancestors = mapOf("child" to listOf("child", "parent"))
val r = resolver(index, ancestors)
assertEquals("/plants#child", r.resolveNodeId("child"))
} @Test
fun `subheading under published ancestor on cross site gets absolute url with anchor`() {
val index = mapOf("parent" to entry("parent", "garden", "plants"))
val ancestors = mapOf("child" to listOf("child", "parent"))
val r = resolver(index, ancestors, currentSite = "lionsrear")
assertEquals("//arcology.garden/plants#child", r.resolveNodeId("child"))
} @Test
fun `subheading under localhost ancestor gets full key with anchor`() {
val index = mapOf("parent" to entry("parent", "lionsrear", "plants"))
val ancestors = mapOf("child" to listOf("child", "parent"))
val r = resolver(index, ancestors, localhost = true)
assertEquals("/lionsrear/plants#child", r.resolveNodeId("child"))
} @Test
fun `no published ancestor returns null`() {
val index = emptyMap<String, RouteEntry>()
val ancestors = mapOf("orphan" to listOf("orphan", "unpublished-ancestor"))
val r = resolver(index, ancestors)
assertNull(r.resolveNodeId("orphan"))
} @Test
fun `resolves node published via heading property ARCOLOGY_KEY`() {
val index = mapOf("wazuka-id" to entry("wazuka-id", "lionsrear", "wazuka"))
val r = resolver(index)
assertEquals("/wazuka", r.resolveNodeId("wazuka-id"))
} @Test
fun `unknown host falls back to absolute url via entry domain`() {
val index = mapOf("nodeB" to entry("nodeB", "garden", "plants/fern"))
val r = resolver(index, currentSite = null)
assertEquals("//arcology.garden/plants/fern", r.resolveNodeId("nodeB"))
} @Test
fun `resolveFilePath always returns null`() {
val r = resolver(emptyMap())
assertNull(r.resolveFilePath("anything"))
} @Test
fun `resolveAttachment matches exact source path`() {
val hash = "a".repeat(64)
val r = resolver(emptyMap(), attachments = listOf(
attachment("n1", "data/202607/xxx/photo.jpg", hash, 512, "jpg"),
attachment("n1", "data/202607/xxx/photo.jpg", hash, 2048, "jpg")
))
val ref = r.resolveAttachment("data/202607/xxx/photo.jpg")
assertNotNull(ref)
assertEquals("/attachment/$hash-512.jpg", ref.url)
assertEquals("/arcology/attachment/$hash-2048.jpg/html", ref.largeHtmlUrl)
assertTrue(ref.isImage)
} @Test
fun `resolveAttachment strips dot-slash prefix before exact match`() {
val hash = "b".repeat(64)
val r = resolver(emptyMap(), attachments = listOf(
attachment("n1", "data/xx/foo.png", hash, 0, "png")
))
val ref = r.resolveAttachment("./data/xx/foo.png")
assertNotNull(ref)
assertEquals("/attachment/$hash-0.png", ref.url)
assertNull(ref.largeHtmlUrl)
assertTrue(ref.isImage)
} @Test
fun `resolveAttachment falls back to basename match`() {
val hash = "c".repeat(64)
val r = resolver(emptyMap(), attachments = listOf(
attachment("n1", "data/202607/xxx/report.pdf", hash, 0, "pdf")
))
// attachment: links and ~/ absolute paths both resolve by basename
val ref = r.resolveAttachment("report.pdf")
assertNotNull(ref)
assertEquals("/attachment/$hash-0.pdf", ref.url)
assertFalse(ref.isImage)
val homeRef = r.resolveAttachment("~/org/data/202607/xxx/report.pdf")
assertNotNull(homeRef)
assertEquals("/attachment/$hash-0.pdf", homeRef.url)
} @Test
fun `resolveAttachment picks first source path on basename collision`() {
val hashA = "d".repeat(64)
val hashB = "e".repeat(64)
val r = resolver(emptyMap(), attachments = listOf(
attachment("n2", "data/zzz/dup.jpg", hashB, 0, "jpg"),
attachment("n1", "data/aaa/dup.jpg", hashA, 0, "jpg")
))
val ref = r.resolveAttachment("dup.jpg")
assertNotNull(ref)
assertEquals("/attachment/$hashA-0.jpg", ref.url)
} @Test
fun `resolveAttachment returns null without repository`() {
val repo = AncestorFakeRepository(emptyMap())
val r = CrossDomainLinkResolver(emptyMap(), domainMap, repo, null, false)
assertNull(r.resolveAttachment("data/xx/foo.jpg"))
} @Test
fun `resolveAttachment returns null for unknown target`() {
val r = resolver(emptyMap(), attachments = listOf(
attachment("n1", "data/xx/foo.jpg", "f".repeat(64), 512, "jpg")
))
assertNull(r.resolveAttachment("nope.jpg"))
}}private class FakeAttachmentLookup(private val attachments: List<PublishedAttachment>) : PublishingRepository {
override suspend fun getAllRoutes(): List<RouteEntry> = emptyList()
override suspend fun getRoutesByPath(path: String): List<RouteEntry> = emptyList()
override suspend fun getPublishedRoutes(): List<RouteEntry> = emptyList()
override suspend fun getPublishedRoutesByPath(path: String): List<RouteEntry> = emptyList()
override suspend fun insertRoute(entry: RouteEntry) {}
override suspend fun deleteRoutesByFile(file: String) {}
override suspend fun countRoutes(): Long = 0
override suspend fun getAttachmentsByNode(nodeId: String): List<PublishedAttachment> =
attachments.filter { it.nodeId == nodeId }
override suspend fun getAttachmentsBySourcePath(sourcePath: String): List<PublishedAttachment> =
attachments.filter { it.sourcePath == sourcePath }
override suspend fun getAttachmentsByBasename(basename: String): List<PublishedAttachment> =
attachments.filter { it.sourcePath.substringAfterLast('/') == basename }
override suspend fun insertAttachment(entry: PublishedAttachment) {}
override suspend fun deleteAttachmentsByFile(file: String) {}
override suspend fun countAttachments(): Long = attachments.size.toLong()
}
private class AncestorFakeRepository(
private val ancestors: Map<String, List<String>>
) : RoamRepository {
override suspend fun getNodeAncestors(nodeId: String): List<String> =
ancestors[nodeId] ?: listOf(nodeId)
override suspend fun getAllFiles() = emptyList<computer.whatthefuck.arcology.domain.OrgFile>()
override suspend fun getFileByPath(path: String) = null
override suspend fun insertFile(file: computer.whatthefuck.arcology.domain.OrgFile) {}
override suspend fun deleteFile(path: String) {}
override suspend fun getAllNodes() = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun getNodeById(id: String) = null
override suspend fun getNodesByFile(file: String) = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun getRecentNodes(limit: Long) = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun searchNodesByTitle(query: String) = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun searchNodesByFilePath(query: String) = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun insertNode(node: computer.whatthefuck.arcology.domain.OrgNode) {}
override suspend fun deleteNode(id: String) {}
override suspend fun insertNodeAncestor(nodeId: String, ancestorId: String) {}
override suspend fun deleteNodeAncestorsByFile(file: String) {}
override suspend fun getLinksFrom(nodeId: String) = emptyList<computer.whatthefuck.arcology.domain.OrgLink>()
override suspend fun getLinksTo(nodeId: String) = emptyList<computer.whatthefuck.arcology.domain.OrgLink>()
override suspend fun insertLink(link: computer.whatthefuck.arcology.domain.OrgLink) {}
override suspend fun deleteLinksByFile(file: String) {}
override suspend fun getTagsByNode(nodeId: String) = emptyList<String>()
override suspend fun getTagsByNodes(nodeIds: List<String>) = emptyMap<String, List<String>>()
override suspend fun getNodesByTag(tag: String) = emptyList<String>()
override suspend fun getAllTags() = emptyList<String>()
override suspend fun getTagsWithCount() = emptyList<Pair<String, Long>>()
override suspend fun insertTag(tag: computer.whatthefuck.arcology.domain.OrgTag) {}
override suspend fun deleteTagsByFile(file: String) {}
override suspend fun getRefsByNode(nodeId: String) = emptyList<Pair<String, String>>()
override suspend fun getRefsByNodes(nodeIds: List<String>) = emptyMap<String, List<String>>()
override suspend fun insertRef(ref: computer.whatthefuck.arcology.domain.OrgRef) {}
override suspend fun deleteRefsByFile(file: String) {}
override suspend fun getAliasesByNode(nodeId: String) = emptyList<String>()
override suspend fun getNodesByAlias(alias: String) = emptyList<String>()
override suspend fun insertAlias(alias: computer.whatthefuck.arcology.domain.OrgAlias) {}
override suspend fun deleteAliasesByFile(file: String) {}
override suspend fun getHeadingProperties(nodeId: String) = emptyMap<String, String?>()
override suspend fun getHeadingProperty(nodeId: String, key: String) = null
override suspend fun insertHeadingProperty(property: computer.whatthefuck.arcology.domain.NodeProperty) {}
override suspend fun deleteHeadingProperty(nodeId: String, key: String) {}
override suspend fun deleteHeadingPropertiesByFile(file: String) {}
override suspend fun getFileProperties(file: String) = emptyMap<String, String?>()
override suspend fun insertFileProperty(property: computer.whatthefuck.arcology.domain.FileProperty) {}
override suspend fun getNodesByPropertyKey(key: String) = emptyList<Pair<String, String?>>()
override suspend fun getNodesWithLocation() = emptyList<Pair<computer.whatthefuck.arcology.domain.OrgNode, computer.whatthefuck.arcology.domain.GeoCoordinate>>()
override suspend fun searchNodes(query: String) = emptyList<computer.whatthefuck.arcology.domain.OrgNode>()
override suspend fun searchNodesByTitles(query: String, limit: Long) = emptyList<String>()
override suspend fun searchNodesByContent(query: String, limit: Long) = emptyList<String>()
override suspend fun searchNodesByTitlesBM25(query: String, limit: Long) = emptyList<computer.whatthefuck.arcology.domain.FtsSearchResult>()
override suspend fun searchNodesByContentBM25(query: String, limit: Long) = emptyList<computer.whatthefuck.arcology.domain.FtsSearchResult>()
override suspend fun insertNodeToFts(node: computer.whatthefuck.arcology.domain.OrgNode, tags: List<String>, aliases: List<String>, content: String) {}
override suspend fun updateNodeInFts(node: computer.whatthefuck.arcology.domain.OrgNode, tags: List<String>, aliases: List<String>, content: String) {}
override suspend fun deleteNodeFromFts(nodeId: String) {}
override suspend fun getFtsContentCount() = 0L
override suspend fun getFailedFile(path: String) = null
override suspend fun getAllFailedFiles() = emptyList<computer.whatthefuck.arcology.domain.FailedFile>()
override suspend fun insertFailedFile(failedFile: computer.whatthefuck.arcology.domain.FailedFile) {}
override suspend fun updateFailedFile(path: String, errorMessage: String, fileHash: String?) {}
override suspend fun deleteFailedFile(path: String) {}
override suspend fun deleteAllFailedFiles() {}
override suspend fun insertFtsStaging(nodeId: String, title: String, tags: String, aliases: String, content: String) {}
override suspend fun getFtsStagingBatch(limit: Long, offset: Long) = emptyList<computer.whatthefuck.arcology.domain.FtsStagingEntry>()
override suspend fun getFtsStagingCount() = 0L
override suspend fun clearFtsStaging() {}
override suspend fun clearAllTitleFts() {}
override suspend fun clearAllContentFts() {}
override suspend fun bulkInsertTitleFts(entries: List<computer.whatthefuck.arcology.domain.FtsTitleEntry>) {}
override suspend fun bulkInsertContentFts(entries: List<computer.whatthefuck.arcology.domain.FtsContentEntry>) {}
override suspend fun <T> transaction(block: suspend () -> T): T = block()
override suspend fun getAttachmentsByNode(nodeId: String) = emptyList<computer.whatthefuck.arcology.domain.OrgAttachment>()
override suspend fun getAttachmentsByType(type: String) = emptyList<computer.whatthefuck.arcology.domain.OrgAttachment>()
override suspend fun insertAttachment(attachment: computer.whatthefuck.arcology.domain.OrgAttachment) {}
override suspend fun deleteAttachmentsByNode(nodeId: String) {}
override suspend fun getAllDiscoveryCache() = emptyList<computer.whatthefuck.arcology.domain.FileDiscoveryCache>()
override suspend fun getDiscoveryCacheByUri(uri: String) = null
override suspend fun getDiscoveryCacheModifiedSince(timestamp: Long) = emptyList<computer.whatthefuck.arcology.domain.FileDiscoveryCache>()
override suspend fun insertDiscoveryCache(entry: computer.whatthefuck.arcology.domain.FileDiscoveryCache) {}
override suspend fun updateDiscoveryCacheHash(uri: String, contentHash: String?, lastSeenAt: Long) {}
override suspend fun deleteDiscoveryCacheByUri(uri: String) {}
override suspend fun deleteStaleDiscoveryCache(beforeTimestamp: Long) {}
override suspend fun clearDiscoveryCache() {}
override suspend fun countDiscoveryCache() = 0L
}ServeCommand
The Clikt command for arcology serve. Wires up all dependencies and starts the server.
package computer.whatthefuck.arcology.publishing
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.parameters.options.option
import com.github.ajalt.clikt.parameters.options.default
import com.github.ajalt.clikt.parameters.types.int
import com.github.ajalt.clikt.parameters.types.path
import computer.whatthefuck.arcology.database.DatabaseFactory
import computer.whatthefuck.arcology.database.RoamRepositoryImpl
import computer.whatthefuck.arcology.parser.OrgFileParser
import computer.whatthefuck.arcology.publishing.*
import io.pebbletemplates.pebble.PebbleEngine
import io.pebbletemplates.pebble.loader.ClasspathLoader
import kotlinx.coroutines.runBlocking
import java.io.File
class ServeCommand : CliktCommand(
name = "serve",
help = "Start the Arcology web server"
) {
private val dbPath: String by option("--db", help = "Path to SQLite database").default("~/org/arcology.db")
private val port: Int by option("--port", help = "Server port").int().default(8080)
private val domainsFile: String by option("--domains", help = "Path to domains.json file").default("web/domains.json")
private val cacheDir: String by option("--cache-dir", help = "HTML cache directory").default("/tmp/arcology-cache")
private val attachmentDir: String by option(
"--attachment-dir",
help = "Crushed attachment cache directory (default: \$ARCOLOGY_ATTACHMENT_DIR or /tmp/arcology-cache/attachments)"
).default(System.getenv("ARCOLOGY_ATTACHMENT_DIR") ?: "/tmp/arcology-cache/attachments")
private val orgDir: String by option("--org-dir", help = "Root org-mode directory").default("~/org")
override fun run() {
val expandedDb = dbPath.replace("~", System.getProperty("user.home"))
val expandedOrgDir = orgDir.replace("~", System.getProperty("user.home"))
val expandedDomains = domainsFile.replace("~", System.getProperty("user.home"))
println("Starting Arcology web server...")
println(" Database: $expandedDb")
println(" Org dir: $expandedOrgDir")
println(" Domains: $expandedDomains")
println(" Cache: $cacheDir")
println(" Attachments: $attachmentDir")
println(" Port: $port")
val database = DatabaseFactory.createDatabase(expandedDb)
val repository = RoamRepositoryImpl(database)
val publishingRepo = PublishingRepositoryImpl(database)
val parser = OrgFileParser()
val domainMap = File(expandedDomains).let { file ->
if (file.exists()) DomainMap.load(file) else {
println("Warning: domains file not found at $expandedDomains, using empty map")
DomainMap.fromMap(emptyMap())
}
}
val routeCount = runBlocking { publishingRepo.countRoutes() }
println(" Routes: $routeCount in the database (queried per-request)")
val htmlCache = HtmlCache(File(cacheDir))
val sidebarService = SidebarService(repository, domainMap)
val pebbleEngine = PebbleEngine.Builder()
.loader(ClasspathLoader().apply { prefix = "templates/" })
.autoEscaping(true)
.build()
val feedRepo = FeedRepositoryImpl(database)
val feedPublisher = FeedPublisher(
feedRepository = feedRepo,
publishingRepository = publishingRepo,
repository = repository,
domainMap = domainMap,
htmlCache = htmlCache,
orgDir = expandedOrgDir,
parser = parser
)
// Prometheus registry + JVM binders ([[file:metrics.org][MetricsConfiguration]]).
val metrics = ArcologyMetrics(MetricsConfiguration.registry())
val server = ArcologyServer(
repository = repository,
domainMap = domainMap,
htmlCache = htmlCache,
orgDir = expandedOrgDir,
parser = parser,
sidebarService = sidebarService,
pebbleEngine = pebbleEngine,
publishingRepository = publishingRepo,
attachmentDir = File(attachmentDir),
feedPublisher = feedPublisher,
feedRepository = feedRepo,
metrics = metrics
)
println("Server starting on http://0.0.0.0:$port")
server.start(port = port)
}
}Tangle Targets
ArcologyServer.kt
package computer.whatthefuck.arcology.publishing
<<server-imports>>
class ArcologyServer(
private val repository: RoamRepository,
private val domainMap: DomainMap,
private val htmlCache: HtmlCache,
private val orgDir: String,
private val parser: OrgFileParser,
private val sidebarService: SidebarService,
private val pebbleEngine: PebbleEngine,
private val publishingRepository: PublishingRepository,
private val attachmentDir: java.io.File = AttachmentCrusher.defaultCacheDir(),
private val feedPublisher: FeedPublisher? = null,
private val feedRepository: FeedRepository? = null,
val metrics: ArcologyMetrics? = null
) {
<<server-method>>
}
<<server-top-level>>NodeRendering.kt
Shared node-body pipeline functions (loadNodeParseResult, renderParsedNode) used by both the page paths and the FeedPublisher.
package computer.whatthefuck.arcology.publishing
import computer.whatthefuck.arcology.database.RoamRepository
import computer.whatthefuck.arcology.parser.OrgFileParser
import computer.whatthefuck.arcology.parser.ParseResult
import kotlinx.coroutines.runBlocking
import java.io.File
<<node-rendering>>PageModel.kt
<<page-model-preamble>>
<<page-model-data>>
<<html-cache-envelope>>
<<sidebar-service>>HtmlCache.kt
<<html-cache-preamble>>
<<html-cache-core>>
<<html-cache-private>>CrossDomainLinkResolver.kt
<<resolver-preamble>>
<<resolver-class>>
<<resolver-route-index>>ServeCommand.kt
<<serve-command>>HtmlCacheTest.kt
<<html-cache-test-prelude>>
<<html-cache-test>>
<<html-cache-test-end>>NestedSectionLookupTest.kt
<<nested-section-test-prelude>>
<<nested-section-test>>
<<nested-section-test-helpers>>
<<nested-section-test-end>>CrossDomainLinkResolverTest.kt
<<cross-domain-resolver-test-prelude>>
<<cross-domain-resolver-test>>
<<cross-domain-resolver-test-end>>
<<ancestor-fake-repo>>ArcologyServerRouteLookupTest.kt
Regression tests for the per-request route-lookup refactor. These verify that servePath and servePathLocalhost consult [PublishingRepository] on every request, so a route added to the database after the server was constructed is visible without a restart. They also lock in the localhost-vs-production draft-visibility split: localhost sees drafts and archived entries, production does not.
The test builds an ArcologyServer against a [MutableFakePublishingRepository] (a [FakePublishingRepository] variant whose backing list can be mutated after construction) and a minimal [RoamRepository] fake that returns empty/null for every read. renderNodeBody returns null when getNodeById returns null, so the rendered body is empty but servePath still produces a non-null PageModel — which is all the regression test needs to assert.
package computer.whatthefuck.arcology.publishing
import computer.whatthefuck.arcology.database.RoamRepository
import computer.whatthefuck.arcology.domain.OrgAlias
import computer.whatthefuck.arcology.domain.OrgAttachment
import computer.whatthefuck.arcology.domain.OrgFile
import computer.whatthefuck.arcology.domain.OrgLink
import computer.whatthefuck.arcology.domain.OrgNode
import computer.whatthefuck.arcology.domain.OrgRef
import computer.whatthefuck.arcology.domain.OrgTag
import computer.whatthefuck.arcology.domain.NodeProperty
import computer.whatthefuck.arcology.domain.FileProperty
import computer.whatthefuck.arcology.domain.FailedFile
import computer.whatthefuck.arcology.domain.FileDiscoveryCache
import computer.whatthefuck.arcology.domain.FtsSearchResult
import computer.whatthefuck.arcology.domain.FtsStagingEntry
import computer.whatthefuck.arcology.domain.FtsTitleEntry
import computer.whatthefuck.arcology.domain.FtsContentEntry
import computer.whatthefuck.arcology.domain.GeoCoordinate
import io.pebbletemplates.pebble.PebbleEngine
import io.pebbletemplates.pebble.loader.ClasspathLoader
import java.io.File
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertNull
class ArcologyServerRouteLookupTest {
private val cacheDir: File = File(System.getProperty("java.io.tmpdir"), "arcology-route-lookup-test-${System.nanoTime()}")
init {
cacheDir.mkdirs()
}
private fun routeEntry(
nodeId: String, site: String, path: String, file: String = "test.org",
title: String? = "Test", isDraft: Boolean = false, isArchived: Boolean = false
): RouteEntry = RouteEntry(
nodeId = nodeId, site = site, path = path, file = file, title = title,
timestamp = null, isDraft = isDraft, isArchived = isArchived
)
private fun server(repo: MutableFakePublishingRepository): ArcologyServer {
val roamRepo = EmptyRoamRepository()
return ArcologyServer(
repository = roamRepo,
domainMap = DomainMap.fromMap(mapOf("garden" to "arcology.garden", "lionsrear" to "thelionsrear.com")),
htmlCache = HtmlCache(cacheDir),
orgDir = "/tmp",
parser = computer.whatthefuck.arcology.parser.OrgFileParser(),
sidebarService = SidebarService(roamRepo, DomainMap.fromMap(mapOf("garden" to "arcology.garden", "lionsrear" to "thelionsrear.com"))),
pebbleEngine = PebbleEngine.Builder()
.loader(ClasspathLoader().apply { prefix = "templates/" })
.build(),
publishingRepository = repo
)
} @Test
fun `servePath returns null for unknown path`() {
val server = server(MutableFakePublishingRepository(emptyList()))
val model = server.servePathPublic("missing", "garden")
assertNull(model)
}
@Test
fun `servePath returns page when published route exists in DB`() {
val repo = MutableFakePublishingRepository(listOf(
routeEntry("node-1", "garden", "archive", title = "Archive")
))
val server = server(repo)
val model = server.servePathPublic("archive", "garden")
assertNotNull(model)
assertEquals("Archive", model.pageTitle)
}
@Test
fun `servePath sees route added to DB after server construction without restart`() {
// The bug: routes indexed after server startup were invisible until restart
// because the route table was built once at construction. With per-request
// DB lookups, a route added to the repository after the server was built
// is visible on the next request.
val repo = MutableFakePublishingRepository(emptyList())
val server = server(repo)
// No route yet:
assertNull(server.servePathPublic("archive", "garden"))
// Indexer writes the new route to the DB:
repo.routes = repo.routes + routeEntry("node-1", "garden", "archive", title = "Archive")
// The running server now sees it:
val model = server.servePathPublic("archive", "garden")
assertNotNull(model)
assertEquals("Archive", model.pageTitle)
}
@Test
fun `servePath excludes draft routes in production mode`() {
val repo = MutableFakePublishingRepository(listOf(
routeEntry("node-draft", "garden", "drafts", isDraft = true, title = "Draft")
))
val server = server(repo)
assertNull(server.servePathPublic("drafts", "garden"))
}
@Test
fun `servePathLocalhost includes draft routes`() {
// Localhost mode is "browse everything": drafts and archived entries
// are visible so you can preview them. The per-request refactor fixes a
// latent bug where the old startup-built RouteTable silently excluded
// drafts on localhost too (it was built from getPublishedRoutes()).
val repo = MutableFakePublishingRepository(listOf(
routeEntry("node-draft", "garden", "drafts", isDraft = true, title = "Draft"),
routeEntry("node-archived", "garden", "archived", isArchived = true, title = "Archived")
))
val server = server(repo)
assertNotNull(server.servePathLocalhostPublic("garden/drafts"))
assertNotNull(server.servePathLocalhostPublic("garden/archived"))
}
@Test
fun `servePathLocalhost filters by site`() {
val repo = MutableFakePublishingRepository(listOf(
routeEntry("g1", "garden", "shared"),
routeEntry("l1", "lionsrear", "shared")
))
val server = server(repo)
// localhost key is garden/shared — only the garden entry should render.
val model = server.servePathLocalhostPublic("garden/shared")
assertNotNull(model)
// The page still renders; the model's pageTitle comes from the first entry.
assertEquals("Test", model.pageTitle)
// And lionsrear/shared should find the lionsrear entry:
assertNotNull(server.servePathLocalhostPublic("lionsrear/shared"))
}/**
* [FakePublishingRepository] variant whose backing list is mutable so tests can
* simulate the indexer writing a new route to the database after the server
* has already been constructed. This is the scenario the per-request route
* lookup fixes: a route added after startup must be visible without a restart.
*/
class MutableFakePublishingRepository(
var routes: List<RouteEntry>
) : PublishingRepository {
override suspend fun getAllRoutes(): List<RouteEntry> = routes
override suspend fun getRoutesByPath(path: String): List<RouteEntry> = routes.filter { it.path == path }
override suspend fun getPublishedRoutes(): List<RouteEntry> = routes.filter { !it.isDraft && !it.isArchived }
override suspend fun getPublishedRoutesByPath(path: String): List<RouteEntry> =
routes.filter { it.path == path && !it.isDraft && !it.isArchived }
override suspend fun insertRoute(entry: RouteEntry) {}
override suspend fun deleteRoutesByFile(file: String) {}
override suspend fun countRoutes(): Long = routes.size.toLong()
override suspend fun getAttachmentsByNode(nodeId: String): List<PublishedAttachment> = emptyList()
override suspend fun getAttachmentsBySourcePath(sourcePath: String): List<PublishedAttachment> = emptyList()
override suspend fun getAttachmentsByBasename(basename: String): List<PublishedAttachment> = emptyList()
override suspend fun insertAttachment(entry: PublishedAttachment) {}
override suspend fun deleteAttachmentsByFile(file: String) {}
override suspend fun countAttachments(): Long = 0
}
/**
* Minimal [RoamRepository] that returns empty/null for every read and no-ops
* every write. [renderNodeBody] returns null when [getNodeById] returns null,
* so the rendered body is empty — but [servePath] still produces a non-null
* [PageModel], which is all the route-lookup tests need to assert.
*
* `open` so first-render fixtures can override [getNodeById] / [getNodesByFile]
* to serve a real node backed by an on-disk org file.
*/
open class EmptyRoamRepository : RoamRepository {
override suspend fun getAllFiles() = emptyList<OrgFile>()
override suspend fun getFileByPath(path: String) = null
override suspend fun insertFile(file: OrgFile) {}
override suspend fun deleteFile(path: String) {}
override suspend fun getAllNodes() = emptyList<OrgNode>()
override suspend fun getNodeById(id: String): OrgNode? = null
override suspend fun getNodesByFile(file: String): List<OrgNode> = emptyList()
override suspend fun getRecentNodes(limit: Long) = emptyList<OrgNode>()
override suspend fun searchNodesByTitle(query: String) = emptyList<OrgNode>()
override suspend fun searchNodesByFilePath(query: String) = emptyList<OrgNode>()
override suspend fun insertNode(node: OrgNode) {}
override suspend fun deleteNode(id: String) {}
override suspend fun insertNodeAncestor(nodeId: String, ancestorId: String) {}
override suspend fun deleteNodeAncestorsByFile(file: String) {}
override suspend fun getNodeAncestors(nodeId: String) = emptyList<String>()
override suspend fun getLinksFrom(nodeId: String) = emptyList<OrgLink>()
override suspend fun getLinksTo(nodeId: String) = emptyList<OrgLink>()
override suspend fun insertLink(link: OrgLink) {}
override suspend fun deleteLinksByFile(file: String) {}
override suspend fun getTagsByNode(nodeId: String) = emptyList<String>()
override suspend fun getTagsByNodes(nodeIds: List<String>) = emptyMap<String, List<String>>()
override suspend fun getNodesByTag(tag: String) = emptyList<String>()
override suspend fun getAllTags() = emptyList<String>()
override suspend fun getTagsWithCount() = emptyList<Pair<String, Long>>()
override suspend fun insertTag(tag: OrgTag) {}
override suspend fun deleteTagsByFile(file: String) {}
override suspend fun getRefsByNode(nodeId: String) = emptyList<Pair<String, String>>()
override suspend fun getRefsByNodes(nodeIds: List<String>) = emptyMap<String, List<String>>()
override suspend fun insertRef(ref: OrgRef) {}
override suspend fun deleteRefsByFile(file: String) {}
override suspend fun getAliasesByNode(nodeId: String) = emptyList<String>()
override suspend fun getNodesByAlias(alias: String) = emptyList<String>()
override suspend fun insertAlias(alias: OrgAlias) {}
override suspend fun deleteAliasesByFile(file: String) {}
override suspend fun getHeadingProperties(nodeId: String) = emptyMap<String, String?>()
override suspend fun getHeadingProperty(nodeId: String, key: String) = null
override suspend fun insertHeadingProperty(property: NodeProperty) {}
override suspend fun deleteHeadingProperty(nodeId: String, key: String) {}
override suspend fun deleteHeadingPropertiesByFile(file: String) {}
override suspend fun getFileProperties(file: String) = emptyMap<String, String?>()
override suspend fun insertFileProperty(property: FileProperty) {}
override suspend fun getNodesByPropertyKey(key: String) = emptyList<Pair<String, String?>>()
override suspend fun getNodesWithLocation() = emptyList<Pair<OrgNode, GeoCoordinate>>()
override suspend fun searchNodes(query: String) = emptyList<OrgNode>()
override suspend fun searchNodesByTitles(query: String, limit: Long) = emptyList<String>()
override suspend fun searchNodesByContent(query: String, limit: Long) = emptyList<String>()
override suspend fun searchNodesByTitlesBM25(query: String, limit: Long) = emptyList<FtsSearchResult>()
override suspend fun searchNodesByContentBM25(query: String, limit: Long) = emptyList<FtsSearchResult>()
override suspend fun insertNodeToFts(node: OrgNode, tags: List<String>, aliases: List<String>, content: String) {}
override suspend fun updateNodeInFts(node: OrgNode, tags: List<String>, aliases: List<String>, content: String) {}
override suspend fun deleteNodeFromFts(nodeId: String) {}
override suspend fun getFtsContentCount() = 0L
override suspend fun getFailedFile(path: String) = null
override suspend fun getAllFailedFiles() = emptyList<FailedFile>()
override suspend fun insertFailedFile(failedFile: FailedFile) {}
override suspend fun updateFailedFile(path: String, errorMessage: String, fileHash: String?) {}
override suspend fun deleteFailedFile(path: String) {}
override suspend fun deleteAllFailedFiles() {}
override suspend fun insertFtsStaging(nodeId: String, title: String, tags: String, aliases: String, content: String) {}
override suspend fun getFtsStagingBatch(limit: Long, offset: Long) = emptyList<FtsStagingEntry>()
override suspend fun getFtsStagingCount() = 0L
override suspend fun clearFtsStaging() {}
override suspend fun clearAllTitleFts() {}
override suspend fun clearAllContentFts() {}
override suspend fun bulkInsertTitleFts(entries: List<FtsTitleEntry>) {}
override suspend fun bulkInsertContentFts(entries: List<FtsContentEntry>) {}
override suspend fun <T> transaction(block: suspend () -> T): T = block()
override suspend fun getAttachmentsByNode(nodeId: String) = emptyList<OrgAttachment>()
override suspend fun getAttachmentsByType(type: String) = emptyList<OrgAttachment>()
override suspend fun insertAttachment(attachment: OrgAttachment) {}
override suspend fun deleteAttachmentsByNode(nodeId: String) {}
override suspend fun getAllDiscoveryCache() = emptyList<FileDiscoveryCache>()
override suspend fun getDiscoveryCacheByUri(uri: String) = null
override suspend fun getDiscoveryCacheModifiedSince(timestamp: Long) = emptyList<FileDiscoveryCache>()
override suspend fun insertDiscoveryCache(entry: FileDiscoveryCache) {}
override suspend fun updateDiscoveryCacheHash(uri: String, contentHash: String?, lastSeenAt: Long) {}
override suspend fun deleteDiscoveryCacheByUri(uri: String) {}
override suspend fun deleteStaleDiscoveryCache(beforeTimestamp: Long) {}
override suspend fun clearDiscoveryCache() {}
override suspend fun countDiscoveryCache() = 0L
}The tests call servePath and servePathLocalhost directly, but those are private on [ArcologyServer]. Rather than widen their visibility for production code, expose minimal internal test-only forwarders on the class so the test can exercise the same code path the request handlers use. The forwarders are visible from tests in the same module, not from consumers.
}<<route-lookup-test-prelude>>
<<route-lookup-test>>
<<route-lookup-test-end>>
<<route-lookup-test-mutable-repo>>ArcologyServerFirstRenderTest.kt
Regression tests for the first-render TOC bug. consumeHeadings() drains the renderer's accumulator (snapshot + clear), and the cache-miss branch of servePath / servePathLocalhost used to drain it a second time when building the live PageModel — so the first render of a page (cache miss) shipped with an empty headings list and no table of contents, while the second render (cache hit) hydrated headings from the [CachedPage] envelope and showed the TOC. The fix consumes once and feeds both the cache write and the live [PageModel] from the same envelope.
The fixture is a real org file on disk parsed by the real [OrgFileParser], a [RoamRepository] fake (subclass of [EmptyRoamRepository]) that returns the fixture node, and a cold [HtmlCache] in a fresh temp dir. The route entry's node ID matches the :ID: property of the fixture's root heading so findSectionByNodeId resolves it and the renderer records three headings (root + two sub-headings), clearing the sidebar.peb TOC gate {% if headings | length > 1 %}.
package computer.whatthefuck.arcology.publishing
import computer.whatthefuck.arcology.domain.OrgNode
import io.pebbletemplates.pebble.PebbleEngine
import io.pebbletemplates.pebble.loader.ClasspathLoader
import java.io.File
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
class ArcologyServerFirstRenderTest {
private val fixtureOrg = """
,#+TITLE: First Render Fixture
,* Root Heading
:PROPERTIES:
:ID: node-toc
:END:
Root body.
,** Sub One
Sub one body.
,** Sub Two
Sub two body.
""".trimIndent()
private data class Dirs(val orgDir: File, val cacheDir: File)
private fun dirs(): Dirs {
val stamp = System.nanoTime()
val orgDir = File(System.getProperty("java.io.tmpdir"), "arcology-first-render-org-$stamp")
val cacheDir = File(System.getProperty("java.io.tmpdir"), "arcology-first-render-cache-$stamp")
orgDir.mkdirs()
cacheDir.mkdirs()
File(orgDir, "fixture.org").writeText(fixtureOrg)
return Dirs(orgDir, cacheDir)
}
private fun newServer(dirs: Dirs): ArcologyServer {
val node = OrgNode(
id = "node-toc", file = "fixture.org", level = 1, position = 1,
title = "Root Heading"
)
val repo = FixtureRoamRepository(node)
val domainMap = DomainMap.fromMap(mapOf("garden" to "arcology.garden"))
return ArcologyServer(
repository = repo,
domainMap = domainMap,
htmlCache = HtmlCache(dirs.cacheDir),
orgDir = dirs.orgDir.absolutePath,
parser = computer.whatthefuck.arcology.parser.OrgFileParser(),
sidebarService = SidebarService(repo, domainMap),
pebbleEngine = PebbleEngine.Builder()
.loader(ClasspathLoader().apply { prefix = "templates/" })
.build(),
publishingRepository = MutableFakePublishingRepository(listOf(
RouteEntry(
nodeId = "node-toc", site = "garden", path = "first-render",
file = "fixture.org", title = "Root Heading",
timestamp = null, isDraft = false, isArchived = false
)
))
)
} @Test
fun `servePath includes TOC headings on the first render (cache miss)`() {
val server = newServer(dirs())
val model = server.servePathPublic("first-render", "garden")
assertNotNull(model)
// The bug: the first render consumed headings twice, shipping an
// empty TOC until the cache was warm.
assertEquals(
listOf("Root Heading", "Sub One", "Sub Two"),
model.headings.map { it.title }
)
}
@Test
fun `servePathLocalhost includes TOC headings on the first render (cache miss)`() {
val server = newServer(dirs())
val model = server.servePathLocalhostPublic("garden/first-render")
assertNotNull(model)
assertEquals(
listOf("Root Heading", "Sub One", "Sub Two"),
model.headings.map { it.title }
)
}
@Test
fun `first and second renders produce the same page model`() {
val dirs = dirs()
val server = newServer(dirs)
// First call: cold cache (miss) — exercises the restructured consume-once
// miss branch. Second call: warm cache (hit) — hydrates from the envelope.
val first = server.servePathPublic("first-render", "garden")
val second = server.servePathPublic("first-render", "garden")
assertNotNull(first)
assertNotNull(second)
// Cache-miss and cache-hit must agree: the TOC, body, and sidenotes are
// identical across the two paths. This also guards against the miss
// branch double-appending sidenotes (renderSidenotes is non-destructive,
// so a naive fix that keeps the trailing append would duplicate them).
assertEquals(first.headings, second.headings)
assertEquals(first.htmlContent, second.htmlContent)
assertEquals(first.sidenotesHtml, second.sidenotesHtml)
}/**
* [EmptyRoamRepository] variant that returns a single fixture node for
* [getNodeById] (so [ArcologyServer.renderNodeBody] finds it and parses the
* on-disk org file) and lists it under [getNodesByFile] (so [collectSidebar]
* sees the file's nodes). Everything else stays empty/null — the page has no
* tags, links, refs, or keywords, which is all the sidebar needs.
*/
class FixtureRoamRepository(private val node: OrgNode) : EmptyRoamRepository() {
override suspend fun getNodeById(id: String): OrgNode? = node.takeIf { it.id == id }
override suspend fun getNodesByFile(file: String): List<OrgNode> =
if (file == node.file) listOf(node) else emptyList()
}}<<first-render-test-prelude>>
<<first-render-test>>
<<first-render-test-fake>>
<<first-render-test-end>>Related Modules
Publishing Layer — provides
RouteTable,DomainMap,PublishingRepositoryHTML Renderer — provides
OrgHtmlRenderer,LinkResolverThe Arcology CLI — where
ServeCommandis registeredThe Indexer Pipeline — provides
RoamRepository,OrgFileParser