Generating an EPUB out of unread feed entries using Python and Pandoc

[kartikynwa]

I came up with this crude script that uses Python modules and Pandoc to generate and epub from unread feed items.

The reason I went with Pandoc here is because it is generally reliable. It also takes care of downloading linked images and statically including them in the EPUB if that is something you care about.

# SPDX-License-Identifier: GPL-3.0-or-later

import os
import subprocess
import tomllib
from tempfile import TemporaryDirectory
from datetime import datetime
from io import StringIO

import miniflux
import dateutil.parser
import lxml.etree
from readability import Document


class ConfigException(Exception):
    """Raise this when the client is not configured properly"""

    pass


# Read
def read_config():
    """Read Miniflux API parameters from environment variables or ./config.toml"""
    miniflux_host = os.getenv("MINIFLUX_HOST")
    miniflux_api_key = os.getenv("MINIFLUX_API_KEY")
    if not (miniflux_api_key and miniflux_host):
        with open("config.toml", "rb") as f:
            config = tomllib.load(f)
            miniflux_host = miniflux_host or config.get("miniflux_host")
            miniflux_api_key = miniflux_api_key or config.get("miniflux_api_key")
    if not (miniflux_api_key and miniflux_host):
        raise ConfigException
    return {
        "miniflux_host": miniflux_host,
        "miniflux_api_key": miniflux_api_key,
    }


def clean_html(html: str) -> str:
    """Pass feed content through python-readability to remove unnecessary elements"""
    clean_html = Document(html).summary(html_partial=True)

    # I have to use lxml to convert the HTML document to XHTML document. This is because pandoc
    # does not respect XHTML spec regarding self-closing tags during this conversion for some
    # reason. Might be a bug,
    tree = lxml.etree.parse(
        StringIO(str(clean_html)), lxml.etree.HTMLParser(default_doctype=False)
    )
    div = tree.getroot().getchildren()[0].getchildren()[0]

    # Dump the parsed etree to XML. The [5:-6] is to remove the unnecessary <div> tags wrapping
    # the document
    return lxml.etree.tostring(div, method="xml").decode().strip()[5:-6].strip()


def main():
    # Read config
    config = read_config()
    client = miniflux.Client(
        config["miniflux_host"], api_key=config["miniflux_api_key"]
    )

    # Get the unread entries
    entries_response = client.get_entries(status=["unread"])
    entries = entries_response.get("entries")
    if not entries:
        print("No entries found")
        exit(0)

    index = 0
    markdown_files = []

    now = datetime.now()

    # Iterate over feed entries and generate Markdown files that will be provided to Pandoc later.
    with TemporaryDirectory() as tmpdir:
        for entry in entries:
            published_at = dateutil.parser.isoparse(entry["published_at"])
            published_at = published_at.strftime("%-d %B, %Y")
            content = f"""<h1>{entry['title']}</h1>\n<h3><em>{entry['feed']['title']}</em> on {entry['published_at']}</h3>\n{entry['content']}"""
            xhtml = clean_html(content)

            _prefix = f"{tmpdir}/{index:04}"
            xhtml_path = f"{_prefix}.xhtml"
            markdown_path = f"{_prefix}.md"
            with open(xhtml_path, "w") as f:
                f.write(xhtml)

            # This is a bit gnarly. Here I first convert XHTML document to Markdown using Pandoc.
            # Have to do this because I could not find a way to tell Pandoc about the chapter
            # metadata when feeding it HTML or XHTML files.
            subprocess.run(["pandoc", "--verbose", "-o", markdown_path, xhtml_path])
            markdown_files.append(markdown_path)

            index += 1

        # Generate a title.txt file containing some metadata for the EPUB file.
        metadata_lines = []
        metadata_lines.append("---")
        metadata_lines.append(f"title: Miniflux Digest {now.strftime('%Y-%m-%d')}")
        metadata_lines.append("author: miniflux-digest")
        metadata_lines.append("...")
        metadata = "\n".join(metadata_lines)
        metadata_path = f"{tmpdir}/title.txt"
        with open(metadata_path, "w") as f:
            f.write(metadata)

        # Generate an EPUB file using the title.txt and Markdown files
        subprocess.run(
            ["pandoc", "--verbose", "-o", "digest.epub", metadata_path, *markdown_files]
        )


if __name__ == "__main__":
    main()

It's very rough around the edges but I am gonna use it only under controlled circumstances.

The Python dependencies are as follows:

dependencies = [
  "miniflux @ git+https://github.com/miniflux/python-client@7b21455ee57167de500f285beeae81bee29fa342",
  "lxml >= 4",
  "readability-lxml @ git+https://github.com/buriy/python-readability@9ee6ef363c77284facba48744d627930d72ee418",
  "python-dateutil >= 2",
]

I don't have a reason to pin certain dependencies to specific commits. I just went that route because the modules had not had tagged releases for a while.