These pages are perhaps the most "dynamic" or inteactive of the Arcology's features right now.
First there is a set of views which renders an index of all the Pages' tags. Here we start to use HTMX to render partials dynamically. Rather than load every tag's page listing, the /tags/ endpoint allows you to drop-down tags with a button press to see posts with that tag.
Then there is the Sitemap page which uses SigmaJS to render a connected topology of all the pages in the Arcology Project.
Django View Setup
import logging
import json
import functools
from django.http import HttpResponse, HttpResponseNotFound, JsonResponse
from django.shortcuts import render, get_object_or_404
from django.db.models import Count
from arcology.models import Page, Feed, Site
from roam.models import Link, Tag, File
from sitemap.models import Node, Edge
# from prometheus_client import Counter, Histogram
logger = logging.getLogger(__name__)from django.contrib import admin
from django.urls import path, re_path, include
from sitemap import views
urlpatterns = [
path("tags", views.tags_index, name="sitemap"),
path("tags/", views.tags_index, name="sitemap"),
path("tags/<slug:tag>", views.tag_page, name="sitemap"),
path("sitemap", views.sitemap, name="sitemap"),
path("sitemap.json", views.sitemap_data, name="sitemap"),
]Tag aggregation pages
from django.shortcuts import render, get_object_or_404
def tags_index(request):
site = Site.from_request(request)
tags = Tag.objects.all().values('tag').annotate(total=Count('tag')).order_by('-total')
return render(request, "sitemap/tags.html", dict(
tags=tags,
site=site,
feeds=site.feed_set.all()
))The tags template lists all the tags and lets you click on them to see the pages in one. .o(it probably should just show all the pages underneath it.)
{% extends "arcology/app.html" %}The tab title is assembled from the page and site title:
{% block title %}The Arcology Project - Tag List{% endblock %}If the site has any feeds, they're injected in to the <head> along with any particular web-crawler rules.
{% load static %}
{% block extra_head %}
{% for feed in feeds %}
<link rel="alternate" type="application/atom+xml" href="{{ feed.url }}" title="{{ feed.title }}" />
{% endfor %}
<link rel="stylesheet" href="{% static 'sitemap/css/sitemap.css' %}"/>
<script src="{% static 'sitemap/js/htmx.js' %}" defer></script>
{% endblock %}The main content block contains the list of tags. It uses HTMX to make a dynamically loading list of pages with headings containing the Tag:
{% block content %}
<section>
<ul>
{% for tag in tags %}
<li>{{ tag.tag }}
(<a class="page_count"
href="/tags/{{ tag.tag }}"
hx-get="/tags/{{ tag.tag }}"
hx-swap="outerHTML"
hx-target="#{{tag.tag}}-pages">
<b>{{tag.total}}</b> Hits
</a>)
{# {% include tag-pages.html %} #}
<ul id="{{tag.tag}}-pages">
</ul>
</li>
{% endfor %}
</ul>
</section>
{% endblock %}
Individual Tag Pages (and list partial)
This renders a partial depending on whether or not it's called by the HTMX declarations above.
def tag_page(request, tag: str):
site = Site.from_request(request)
pages = Tag.weighted_pages_by_name(tag)
if request.htmx:
base_template = "sitemap/tag_partial.html"
else:
base_template = "arcology/app.html"
return render(request, "sitemap/tag.html", dict(
base_template=base_template,
tag=tag,
pages=pages,
site=site,
)){% extends base_template %}
{% load static %}
{% block h1 %}<h1>{{site.title}}<h2>Pages tagged with {{tag}}</h2></h1>{% endblock %}
{% block title %}Pages tagged with {{tag}} in the Arcology{% endblock %}
{% block extra_head %}
<link rel="stylesheet" href="{% static 'sitemap/css/sitemap.css' %}"/>
<script src="{% static 'sitemap/js/htmx.js' %}" defer></script>
{% endblock %}
{% block content %}
<section class="tag-index">
<a href="/tags/">← Show all tags</a>
<h1>Pages tagged with {{tag}}</h1>
{% block list %}
<ul id="{{tag}}-pages" class="tag-list">
{% for page, hit_count in pages.items %}
<li><a style="--size: {{hit_count}};" href="{{page.to_url}}">{{page.title}}</a></li>
{% endfor %}
</ul>
{% endblock %}
</section>
{% endblock %}With some minor CSS rules applied we get a sort of limited "tag cloud" effect where pages with more links to this heading get made larger
.tag-index ul.tag-list a {
--size: 1;
font-size: calc(log(var(--size) + 1) * 120%);
}
#sitemap-container {
height: 80em;
filter: saturate(500%);
}That request.htmx branch will make sure we only render the list if the HTMX partial is called by swapping the base HTML template to render only the list:
<div class="tag-list">
<a href="/tags/{{ tag }}">Show all...</a>
{% block list %}{% endblock %}
</div>Sitemap JSON
That this relies on Data Models for Sites, Web Features, and Feeds and Arcology Roam Models tells me it may be needs to be in a different module, idk... the structure of these projects really does need to be worked on.
This also lifts design and code whole-sale from Arcology FastAPI: Navigating the Arcology Site Graph with SigmaJS.
Nodes come from Page objects and are shaped like this:
#+begin_example json { "id": "n0", "label": "A node", "x": 0, "y": 0, "size": 3 } #+end_example
Edges come from Link objects and are shaped like this:
#+begin_example json { "id": "e0", "source": "n0", "target": "n1" } #+end_example
compose together and return in a single dict for JSON rendering shaped like:
#+begin_example json { "nodes": [ <<all nodes>> ] "edges": [ <<all edges>> ] } #+end_example
Making SigmaJS Nodes
import arcology.models
import roam.models
from arcology.cache_decorator import cache
import hashlib
def make_loc_hash(page: arcology.models.Page, salt, max_q=700):
key = page.file.path + str(salt)
hash = hashlib.sha224(key.encode("utf-8")).digest()
return int.from_bytes(hash, byteorder="big") % max_q
class Node():
@classmethod
def make_page_dict(cls, page):
@cache(key_prefix="sitemap_node", expire_secs=60*60*24)
def _make(page, hash):
link_cnt = page.file.outbound_links.count()
backlink_cnt = roam.models.Link.objects.filter(dest_heading__in=page.file.heading_set.all()).count()
return dict(
key=page.route_key,
attributes=dict(
label=page.title,
x=make_loc_hash(page, 1),
y=make_loc_hash(page, 2),
size=min((link_cnt + backlink_cnt) / 2, 20),
color=page.site.link_color,
href=page.to_url(),
)
)
return _make(page, page.file.digest)
@classmethod
def get_sigmajs_nodes(cls):
pages = arcology.models.Page.objects.all()
nodes = [
cls.make_page_dict(page)
for page in pages
]
return nodesMaking SigmaJS Edges
class hashabledict(dict):
def __hash__(self):
return hash(tuple(sorted(self.items())))
class Edge():
@classmethod
def get_sigmajs_edges(cls):
q_links = roam.models.Link.objects.all()
links = set()
for link in q_links:
try:
if link.dest_heading is None:
continue
source_page = link.source_file.page_set.first()
if source_page is None:
continue
source = source_page.route_key
dest_page = link.dest_heading.path.page_set.first()
if dest_page is None:
continue
dest = dest_page.route_key
links.add(
hashabledict(
key=f"{source}-{dest}",
source=source,
target=dest,
)
)
except roam.models.Heading.DoesNotExist:
pass
return list(links)JSON Handler/View
This tries to calculate a consistent cache key cheaply and probably fails.
import hashlib
from arcology.cache_decorator import cache
@cache(key_prefix="sitemap_resp", expire_secs=60*60*24)
def _cached(cache_key, hashes):
print(f"called w/ cache key {cache_key}")
ret = dict(
nodes=Node.get_sigmajs_nodes(),
edges=Edge.get_sigmajs_edges(),
)
print(f"finished call w/ cache key {cache_key}")
return ret
def sitemap_data(request):
hashes = [ file.digest for file in File.objects.order_by('path') ]
cache_key = hashlib.sha224(''.join(hashes).encode("utf-8")).hexdigest()
return JsonResponse(_cached(cache_key, cache_key))Sitemap HTML Page
def sitemap(request):
site = Site.from_request(request)
return render(request, "sitemap/map.html", dict(
site=site
))Sitemap Page Template
{% extends "arcology/app.html" %}
{% load static %}
{% block h1 %}<h1>{{site.title}}<h2>A Map of the Arcology Sites</h2></h1>{% endblock %}
{% block title %}Arcology Sites Map{% endblock %}
{% block extra_head %}
<link rel="stylesheet" href="{% static 'sitemap/css/sitemap.css' %}"/>
<script src="{% static 'sitemap/js/htmx.js' %}" defer></script>
<script src="{% static 'sitemap/js/graphology.min.js' %}" defer></script>
<script src="{% static 'sitemap/js/graphology-library.min.js' %}" defer></script>
<script src="{% static 'sitemap/js/sigmajs.min.js' %}" defer></script>
{% endblock %}
{% block content %}
<section>
<p>
This is a network graph of all the pages published in my
Arcology network. The color of the node corresponds to its
site, and you can click on any of them to jump to that page or
zoom in to see more of their labels. I will be adding more
features to this sitemap like search and highlighting
"relative" neighbors later on. Feel free to view
<a href="https://engine.arcology.garden/sitemaps">the implementation</a>
within the Arcology pages, as usual.
</p>
<p>
You may also like to see the <a href="/tags/">Tag Index</a>.
</p>
<div id="sitemap-container">
</div>
<script type="application/javascript" src="{% static 'sitemap/js/sitemap.js'%}"></script>
</section>
{% endblock %}Sitemap Frontend JS using Sigma JS
What is SigmaJS
Linked from
References
SigmaJS is a JavaScript library for visualizing graphs. My org-roam knowledge base is a graph and arroyo-db contains the public pages and links. As nodes and edges, this makes a fine graph and a lot more "logical" than an XML sitemap or linear walk through the sites.
Getting SigmaJS
Download and vendor the minified versions:
export GRAPHOLOGY_VER=0.24.1
export SIGMAJS_VER=v2.3.1
curl --location -v -o sitemap/static/sitemap/js/sigmajs.min.js https://github.com/jacomyal/sigma.js/releases/download/$SIGMAJS_VER/sigma.min.js
curl --location -v -o sitemap/static/sitemap/js/graphology.min.js https://github.com/graphology/graphology/releases/download/$GRAPHOLOGY_VER/graphology.min.js
curl --location -v -o sitemap/static/sitemap/js/graphology-library.min.js https://github.com/graphology/graphology/releases/download/$GRAPHOLOGY_VER/graphology-library.min.jsThese can now be used by the django static template helper.
JavaScript to set up and render the SigmaJS graph
The SigmaJS upstream code is loaded in the <head>, then this code runs after page load to fetch the graph data and cram it in to the =#sitemap-container= <div>.
const container = document.getElementById("sitemap-container");
fetch('/sitemap.json')
.then(response => response.json())
.then((data) => {
const graph = new graphology.Graph.from(data);
<<forceAtlas2-setup>>
const renderer = new Sigma(graph, container);
<<hoverReducer>>
<<forceAtlas2-invoke-worker>>
<<clickNode>>
return {renderer, graph}
});A ForceAtlas2 layout algorithm is applied to the page for 5 seconds to allow the nodes to coalesce in to a stable layout.
var FA2Layout = graphologyLibrary.FA2Layout;
var forceAtlas2 = graphologyLibrary.layoutForceAtlas2;
const sensibleSettings = forceAtlas2.inferSettings(graph);
const layout = new FA2Layout(graph, {
settings: sensibleSettings
});The layout work is done in a background task:
layout.start();
setTimeout(() => {
layout.stop()
}, 5000);An event handler is defined for the clickNode event to open the URL embedded in the node's attributes added in Making SigmaJS Nodes.
renderer.on("clickNode", ({ node }) => {
var realnode = graph._nodes.get(node);
window.location = realnode.attributes.href;
});Focusing on Nodes in the Graph
The <<hoverReducer>> code to "highlight" a node by de-emphasizing the nodes it does not neighbor is taken from the SigmaJS examples and converted from TypeScript to JavaScript by hand. No I will not set up a NodeJS build chain.
state = {};
function setHoveredNode(node) {
if (node) {
state.hoveredNode = node;
state.hoveredNeighbors = new Set(graph.neighbors(node));
} else {
state.hoveredNode = undefined;
state.hoveredNeighbors = undefined;
}
// Refresh rendering:
renderer.refresh();
}
renderer.on("enterNode", ({ node }) => {
setHoveredNode(node);
});
renderer.on("leaveNode", () => {
setHoveredNode(undefined);
});
// Render nodes accordingly to the internal state:
// 1. If a node is selected, it is highlighted
// 2. If there is query, all non-matching nodes are greyed
// 3. If there is a hovered node, all non-neighbor nodes are greyed
renderer.setSetting("nodeReducer", (node, data) => {
const res = { ...data };
if (state.hoveredNeighbors && !state.hoveredNeighbors.has(node) && state.hoveredNode !== node) {
res.label = "";
res.color = "#f6f6f6";
}
if (state.selectedNode === node) {
res.highlighted = true;
} else if (state.suggestions && !state.suggestions.has(node)) {
res.label = "";
res.color = "#f6f6f6";
}
return res;
});
// Render edges accordingly to the internal state:
// 1. If a node is hovered, the edge is hidden if it is not connected to the
// node
// 2. If there is a query, the edge is only visible if it connects two
// suggestions
renderer.setSetting("edgeReducer", (edge, data) => {
const res = { ...data };
if (state.hoveredNode && !graph.hasExtremity(edge, state.hoveredNode)) {
res.hidden = true;
}
if (state.suggestions && (!state.suggestions.has(graph.source(edge)) || !state.suggestions.has(graph.target(edge)))) {
res.hidden = true;
}
return res;
});