Arcology Engine

Posting Arcology Feeds to the Fediverse Automatically with feed2toot (deprecated)

Contents

Feed2toot automatically parses rss feeds, identifies new posts and posts them on the +Mastodon+ Fediverse social network.

I'll probably end up writing something similar myself one of these days since (somewhat critically) this does not support multiple (feeds->account) mappings right now and thus I will have to run one for each of my Arcology Sites' feeds.

feed2toot rixpkgs entry

We love a Python package that is pretty easy to get running:

nix:tangle ~/nix/pkgs/feed2toot.nix
{ lib,
  buildPythonPackage,
  fetchPypi,
  beautifulsoup4,
  feedparser,
  mastodon-py,
  python,
 }:

buildPythonPackage rec {
  pname = "feed2toot";
  version = "0.17";

  src = fetchPypi {
    inherit pname version;
    sha256 = "sha256-VKj0Wu5KdsfsywO1g4z3mFsQT9Svrt6wQHZSeNxbRAw=";
  };

  propagatedBuildInputs = [
    beautifulsoup4
    feedparser
    mastodon-py
    python.pkgs.influxdb
  ];

  meta = with lib; {
    homepage = "https://feed2toot.readthedocs.io/en/latest/";
    description = "Feed2toot parses a RSS feed, extracts the last entries and sends them to Mastodon.";
    license = licenses.gpl3;
    maintainers = with maintainers; [ rrix ];
  };
}

nix-build '<arroyo>' -A feed2toot -> /nix/store/fvw5m7q8y2kh0mkr2y52wv55lvv2hwym-python3.10-feed2toot-0.17

Generating the Feed URI Lists Programmatically

So I have a few Arcology Sites, each of these needs its own configuration file passed to the feed2toot invocation. This queries Arroyo Arcology Generator's arcology.arroyo.Feed to generate a file with each group of (site, visibility) of feeds in the database.

emacs-lisp#+name: site-filter:var site-domain="https://thelionsrear.com/":results drawer
(setq arcology-sites '(("lionsrear" . "https://thelionsrear.com/")
                       ("garden" . "https://arcology.garden/")
                       ("cce" . "https://cce.whatthefuck.computer/")
                       ("arcology" . "https://engine.arcology.garden/")))

(defun arcology-arroyo-feeds-to-urls (group rows)
  (let* ((file-name (format "~/arroyo-nix/files/feed2toot-uris-%s-%s.txt" (first group) (second group))))
    (list file-name
          (-map (lambda (row)
                  (let* ((feed-prefix (alist-get (nth 3 row) arcology-sites nil nil #'equal))
                         (site (nth 3 row))
                         (key (nth 1 row))
                         (feed-url (replace-regexp-in-string (concat site "/") feed-prefix key)))
                    feed-url
                    )
                  )
                rows))))

(defun arcology-arroyo-write-feed-list (file lines)
  (with-temp-buffer
    (insert (s-join "\n" lines))
    (write-file file)
    file))

(defun arcology-arroyo-make-feed-lists ()
  (->> (arroyo-db-query [:select * :from arcology-feeds])
       (-group-by (lambda (row)
                    (list (nth 3 row) (nth 4 row))))
       (-map (lambda (partitioned)
               (let* ((partition (first partitioned))
                      (rows (cdr partitioned)))
                 (arcology-arroyo-feeds-to-urls partition rows))))
       (-map (lambda (group)
               (apply #'arcology-arroyo-write-feed-list group)))))

(s-join "\n" (arcology-arroyo-make-feed-lists))

:results: ~/arroyo-nix/files/feed2toot-uris-garden-private.txt ~/arroyo-nix/files/feed2toot-uris-arcology-public.txt ~/arroyo-nix/files/feed2toot-uris-garden-public.txt ~/arroyo-nix/files/feed2toot-uris-lionsrear-unlisted.txt ~/arroyo-nix/files/feed2toot-uris-lionsrear-public.txt ~/arroyo-nix/files/feed2toot-uris-cce-public.txt :END:

INPROGRESS deploy to The Wobserver

feed2toot operates with INI configuration files and luckily enough nixpkgs includes a way to convert attrsets to ini files in lib.generators.toINI; this is wrapped in a function which takes the basic per-instance configuration details including the uri-list which are generated above and then creates a SystemD service manifest and timer to poll the site and post the toots. Easy peazy.

,#+ARROYO_NIXOS_MODULE: nixos/feed2toot.nix ,#+ARROYO_SYSTEM_ROLE: server

nix:tangle ~/nix/nixos/feed2toot.nix:noweb yes
{ pkgs, lib, config, ... }:

let
  mkFeed2TootIni = { instance,
                     uriList,
                     visibility ? "public",
                     credLoc ? config.users.users.feed2toot.home,
                     tagListFile ? pkgs.writeText "empty-taglist" ""}:
                       pkgs.writeText "feed2toot-${instance}"
                         (lib.generators.toINI {}
                           {
                             mastodon = {
                               instance_url = "https://notes.whatthefuck.computer";
                               user_credentials = "${credLoc}/${instance}_feed2toot_usercred.txt";
                               client_credentials = "${credLoc}/${instance}_feed2toot_clientcred.txt";
                               toot_visibility = visibility;
                             };
                             cache.cachefile = "/var/cache/feed2toot/${instance}.db";
                             lock.lock_file = "/run/feed2toot/${instance}.lock";
                             rss.uri_list = builtins.path { path = uriList; };
                             rss.toot = "NEW by @rrix@notes.whatthefuck.computer: {link} \\n {summary}";
                             rss.toot_max_len = 10000;
                             hashtaglist.several_words_hashtags_list = tagListFile;
                             feedparser.accept_bozo_exceptions = true;
                           });

  feeds = [
    (mkFeed2TootIni {
      instance = "garden";
      visibility = "public";
      uriList = ../files/feed2toot-uris-cce-public.txt;
    })
    (mkFeed2TootIni {
      instance = "garden";
      visibility = "public";
      uriList = ../files/feed2toot-uris-garden-public.txt;
    })
    (mkFeed2TootIni {
      instance = "lionsrear";
      visibility = "public";
      uriList = ../files/feed2toot-uris-lionsrear-public.txt;
    })
    (mkFeed2TootIni {
      instance = "garden";
      visibility = "unlisted";
      uriList = ../files/feed2toot-uris-arcology-public.txt;
    })
    (mkFeed2TootIni {
      instance = "lionsrear";
      visibility = "unlisted";
      uriList = ../files/feed2toot-uris-lionsrear-unlisted.txt;
    })
    (mkFeed2TootIni {
      instance = "garden";
      visibility = "private";
      uriList = ../files/feed2toot-uris-garden-private.txt;
    })
  ];
in {
  ids.uids.feed2toot = 902;
  ids.gids.bots = 902;

  users.groups.bots = {
    gid = config.ids.gids.bots;
  };
  users.users."feed2toot" = {
    home = "/srv/feed2toot/";
    group = "bots";
    uid = config.ids.uids.feed2toot;
    isSystemUser = true;
  };

  systemd.tmpfiles.rules = [
    "d /run/feed2toot 1777 feed2toot bots"       # lock file directory
    "d /var/cache/feed2toot 1777 feed2toot bots" # cache file directory
    "d /srv/feed2toot 1777 feed2toot bots"       # working directory
  ];

  systemd.services.feed2toot = {
    description = "Feeds to Toots";
    after = ["pleroma.service"];
    wantedBy = ["default.target"];
    script = 
      lib.concatMapStrings
        (feed: "\n${pkgs.feed2toot}/bin/feed2toot -c " + feed)
        feeds;
    serviceConfig = {
      User = "feed2toot";
      WorkingDirectory = "/srv/feed2toot";
    };
  };
  systemd.timers.feed2toot = {
    description = "Start feed2toot on the quarter-hour";
    timerConfig = {
      OnUnitActiveSec = "15 minutes";
      OnStartupSec = "15 minutes";
    };
    wantedBy = [ "default.target" ];
  };
}

NEXT remove bs4 html filtering

pleroma supports embedding HTML so we should use that.