Blogging with RestructuredText, a Google Domain, and Sphinx

The previous approach to blogging will suffice for non-technical materials, but will quickly become cumbersome when one has to deal with bibtex, Jupyter, and documentation. Luckily, the only component that needs to change is Pelican; the other instructions still apply.

The replacement component, Sphinx, only requires a single configuration file:

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import datetime
import filecmp
import os
import pathlib
import shutil
import subprocess
import sys
import tempfile

import sphinx_bootstrap_theme

def generate_toctree(master_toctree='toctree-blog', max_depth=1,
                     crawl_path=['blog'], crawl_ext=['rst', 'ipynb'],
                     sort_results=True):
    toctree = []
    for _ in crawl_path:
        paths = []
        for ext in crawl_ext:
            p = list(pathlib.Path(_).rglob('*.{0}'.format(ext)))
            paths.extend(remove_excluded_patterns(p))

        #alphanumeric ordering of files grouped by crawl_path
        toctree.extend(sorted(paths, reverse=sort_results))

    _ = '{0}\n   {1}{2}\n\n'
    header = _.format('.. toctree::', ':maxdepth: ', max_depth)
    target_filename = '{0}/{1}'.format('_templates', master_toctree)
    with tempfile.NamedTemporaryFile(mode='w') as temp:
        temp.write(header)
        _ = (os.path.splitext('   {0}'.format(pp))[0] for pp in toctree)
        temp.write('\n'.join(_))
        temp.flush()

        _  = pathlib.Path(target_filename).is_file() and \
                     filecmp.cmp(temp.name, target_filename, shallow=False)
        if not _:
            shutil.copyfile(temp.name, target_filename)

def generate_nb_toctree(master_toctree='toctree-nb', max_depth=1,
                        crawl_path=['nb'], crawl_ext=['ipynb']):
    for cp in crawl_path:
        #assumes each subdirectory is a notebook
        notebooks = os.listdir(cp)
        for nb in notebooks:
            #create a toctree for each notebook
            toctree = []
            paths = []
            for ext in crawl_ext:
                _ = os.path.join(cp, nb)
                p = list(pathlib.Path(_).rglob('*.{0}'.format(ext)))
                paths.extend(p)

            #alphanumeric ordering of files
            toctree.extend(sorted(paths))

            _ = '{0}\n   {1}{2}\n\n'
            header = _.format('.. toctree::', ':maxdepth: ', max_depth)
            target_filename = '{0}/{1}-{2}'.format('_templates',
                                                   master_toctree, nb)
            with tempfile.NamedTemporaryFile(mode='w') as temp:
                temp.write(header)
                _ = ('   {0}'.format(pp.name) for pp in toctree)
                temp.write('\n'.join(_))
                temp.flush()

                _  = pathlib.Path(target_filename).is_file() and \
                     filecmp.cmp(temp.name, target_filename, shallow=False)
                if not _:
                    shutil.copyfile(temp.name, target_filename)

    generate_toctree(master_toctree=master_toctree, max_depth=max_depth,
                     crawl_path=crawl_path, crawl_ext=['rst'],
                     sort_results=False)

def asy2svg(asymptote='asy', epstopdf='epstopdf', pdf2svg='pdf2svg',
            crawl_path=['blog', 'nb'], crawl_ext=['asy']):
    for _ in crawl_path:
        for ext in crawl_ext:
            paths = pathlib.Path(_).rglob('*.{0}'.format(ext))
            for path in remove_excluded_patterns(paths):
                filename = os.path.splitext(path.name)[0]

                _ = [asymptote, str(path)]
                _ = subprocess.Popen(_).wait()

                _ = [
                    epstopdf, filename + '.eps'
                ]
                _ = subprocess.Popen(_).wait()

                _ = [
                    pdf2svg,
                    filename + '.pdf',
                    os.path.splitext(str(path))[0] + '.svg'
                ]
                _ = subprocess.Popen(_).wait()

                #remove intermediate files
                os.remove(filename + '.eps');
                os.remove(filename + '.pdf');

def ly2svg(lilypond='lilypond', crawl_path=['blog', 'nb'], crawl_ext=['ly']):
    for _ in crawl_path:
        for ext in crawl_ext:
            paths = pathlib.Path(_).rglob('*.{0}'.format(ext))
            for path in remove_excluded_patterns(paths):
                _ = [lilypond, '-dbackend=svg', '--silent', '-o', os.path.splitext(str(path))[0], str(path)]
                print(_)
                _ = subprocess.Popen(_).wait()

extensions = [
  'nbsphinx', 'IPython.sphinxext.ipython_console_highlighting',
  'sphinx.ext.todo',
  'sphinx.ext.mathjax',
  'sphinxcontrib.bibtex',
  'sphinx_sitemap',
]
mathjax_path = '{}?{}'.format(
                      'https://cdn.rawgit.com/mathjax/MathJax/2.7.1/MathJax.js',
                      'config=TeX-MML-AM_CHTML')
numfig = True

project = 'All Things Phi'
author = 'alphaXomega'
copyright = '2013-{0}, {1}'.format(datetime.datetime.now().year, author)
site_url = 'http://allthingsphi.com/'

source_suffix = '.rst'
exclude_patterns = ['blog/2013/01/11/example']
def remove_excluded_patterns(paths):
    return (path for path in paths if not
            all(str(path).startswith(prefix) for prefix in exclude_patterns))

master_doc = 'index'
generate_toctree()
generate_nb_toctree()
#asy2svg()
#ly2svg

language = None

todo_include_todos = True

templates_path = ['_templates']
html_static_path = ['_static']

html_title = 'All Things Phi'
htmlhelp_basename = 'AllThingsPhidoc'

html_favicon = '_static/phi.ico'

pygments_style = 'sphinx'

html_theme = 'bootstrap'
html_theme_options = {
  'navbar_links': [
    ('Bitbucket', 'https://bitbucket.org/alphaXomega/', True)
  ],
  'navbar_site_name': 'Archive',
  'source_link_position': 'footer',
}
html_theme_path = sphinx_bootstrap_theme.get_html_theme_path()

def setup(app):
    #make toc dropdown menu scrollable
    #https://github.com/ryan-roemer/sphinx-bootstrap-theme/issues/153
    app.add_stylesheet('my-styles.css')

There may be a lot of settings, but the majority of them are automatically generated via

pip install sphinx sphinxcontrib-bibtex nbsphinx sphinx_bootstrap_theme sphinx_sitemap
sphinx-quickstart

The extensions are meant to address the issues raised earlier:

Besides the configuration file, Sphinx requires a master_doc (e.g. source/index.rst) to describe the relations between the individual files. The sample conf.py will generate the files under source/_templates/ assuming the following project layout:

.
├── gae
│   ├── www
│   └── app.yaml
└── source
    ├── blog
    │   └── 2016
    │       └── 01
    │           └── 01
    │               └── some-content.rst
    ├── nb
    │   └── some-notebook
    │       ├── chapter-01.ipynb
    │       ├── index.rst
    │       └── refs.bib
    ├── _static
    │   └── phi.ico
    ├── _templates
    │   ├── toctree-blog
    │   ├── toctree-nb
    │   └── toctree-nb-some-notebook
    ├── conf.py
    └── index.rst

One way to get Sphinx to distinguish between document groupings is to organize each group (e.g. nb/some-notebook/index.rst) under a different toctree (e.g. toctree-nb-some-notebook) and reference the group from the master_doc.

Running the following command with the sample conf.py will scan the folders blog and nb for content, generate a static site, and launch a local web server:

sphinx-build -b html source gae/www
pushd gae/www; python3 -m http.server

To watch a Sphinx directory and rebuild the documentation when a change is detected, install and run sphinx-autobuild.

pip install sphinx-autobuild
sphinx-autobuild -b html source gae/www

To generate printable HTML slides from ReStructured Text, check out the Sphinx extension Hieroglyph.

Handling Graphical Elements

As part of documentation, one may want to include graphic elements that are generated programmatically. Unfortunately, there are no good Sphinx extension that handles this properly. The sample conf.py includes a function that uses asymptote, epstopdf, and pdf2svg to generate SVG graphics.

For any kind of flowcharts, use graphviz.