improved org2gemini

Makefile

@@ -7,7 +7,8 @@
 all: allatend
 SRC_DIR ?= src
 DST_DIR ?= _site
-SRC_RAW_FILES := $(shell find $(SRC_DIR) -type f)
+NO_DRAFT := -not -path '$(SRC_DIR)/drafts/*'
+SRC_RAW_FILES := $(shell find $(SRC_DIR) -type f $(NO_DRAFT))
 DST_RAW_FILES   := $(patsubst $(SRC_DIR)/%,$(DST_DIR)/%,$(SRC_RAW_FILES))
 ALL             += $(DST_RAW_FILES)
 
@@ -19,7 +20,7 @@ $(DST_DIR)/% : $(SRC_DIR)/%
 
 # ORG -> HTML
 EXT := .org
-SRC_PANDOC_FILES ?= $(shell find $(SRC_DIR) -type f -name "*$(EXT)")
+SRC_PANDOC_FILES ?= $(shell find $(SRC_DIR) -type f -name "*$(EXT)" $(NO_DRAFT))
 DST_PANDOC_FILES ?= $(subst $(EXT),.html, \
                         $(subst $(SRC_DIR),$(DST_DIR), \
                             $(SRC_PANDOC_FILES)))
@@ -55,7 +56,7 @@ ALL += $(HTML_INDEX)
 
 # ORG -> GEMINI
 EXT := .org
-SRC_GMI_FILES ?= $(shell find $(SRC_DIR) -type f -name "*$(EXT)")
+SRC_GMI_FILES ?= $(shell find $(SRC_DIR) -type f -name "*$(EXT)" $(NO_DRAFT))
 DST_GMI_FILES ?= $(subst $(EXT),.gmi, \
                         $(subst $(SRC_DIR),$(DST_DIR), \
                             $(SRC_GMI_FILES)))
@@ -63,7 +64,7 @@ DST_GMI_FILES ?= $(subst $(EXT),.gmi, \
 ALL              += $(DST_GMI_FILES)
 GMI := engine/org2gemini.sh
 
-$(DST_DIR)/%.gmi: $(SRC_DIR)/%.org $(GMI)
+$(DST_DIR)/%.gmi: $(SRC_DIR)/%.org $(GMI) engine/org2gemini_step1.sh
 	mkdir -p $(dir $@)
 	$(GMI) "$<" "$@"
 
@@ -71,8 +72,10 @@ $(DST_DIR)/%.gmi: $(SRC_DIR)/%.org $(GMI)
 # OPTIM PHASE
 
 OPTIM_DIR ?= _optim
+ENGINE_DIR ?= engine
+ENGINE_SCRIPTS := $(shell find $(ENGINE_DIR) -type f)
 OPTIM := engine/pre-deploy.sh
-$(OPTIM_DIR)/index.html:$(DST_RAW_FILES) $(DST_GMI_FILES) $(DST_PANDOC_FILES) $(HTML_INDEX) $(OPTIM)
+$(OPTIM_DIR)/index.html:$(DST_RAW_FILES) $(DST_GMI_FILES) $(DST_PANDOC_FILES) $(HTML_INDEX) $(ENGINE_SCRIPTS) $(OPTIM)
 	mkdir -p $(OPTIM_DIR)
 	$(OPTIM)
 

engine/mk-gemini-atom.sh

@@ -30,7 +30,7 @@ finddate(){ < $1 | awk '/^date: /' | head -n1 | perl -pe 's/^.*\[//;s/ .*$//;' }
 findtitle(){ < $1 | head -n1 | perl -pe 's/^# //' }
 getcontent(){
     < $1 perl -pe 'use URI; $base="'$2'"; s# (href|src)="((?!https?://)[^"]*)"#" ".$1."=\"".URI->new_abs($2,$base)->as_string."\""#eig' }
-findkeywords(){ < $1 | awk '/^keywords: /' | head -n1 | perl -pe 's/^[^:]\s+//' }
+findkeywords(){ < $1 | awk '/^keywords: /' | head -n1 | sed 's/keywords: //' }
 mkcategories(){
     for keyword in $*; do
         printf "\\n<category>%s</category>" $keyword

engine/mk-gemini-index.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env zsh
+
+cd "$(git rev-parse --show-toplevel)" || exit 1
+# Directory
+webdir="_optim"
+postsdir="$webdir/posts"
+indexfile="$webdir/index.gmi"
+
+# maximal number of articles to put in the RSS file
+maxarticles=100
+
+# RSS Metas
+rsstitle="her.esy.fun"
+websiteurl="gemini://her.esy.fun"
+rssdescription="her.esy.fun articles, mostly random personal thoughts"
+rsslang="en"
+rssauthor="yann@esposito.host (Yann Esposito)"
+
+# title and keyword shouldn't be changed
+
+formatdate() {
+    # format the date for RSS
+    local d=$1
+    LC_TIME=en_US date --date $d +'%Y-%m-%d'
+}
+
+finddate(){ < $1 | awk '/^date: /' | head -n1 | perl -pe 's/^.*\[//;s/ .*$//;' }
+findtitle(){ < $1 | head -n1 | perl -pe 's/^# //' }
+getcontent(){
+    < $1 perl -pe 'use URI; $base="'$2'"; s# (href|src)="((?!https?://)[^"]*)"#" ".$1."=\"".URI->new_abs($2,$base)->as_string."\""#eig' }
+findkeywords(){ < $1 | awk '/^keywords: /' | head -n1 | sed 's/keywords: //' }
+
+autoload -U colors && colors
+
+tmpdir=$(mktemp -d)
+typeset -a dates
+dates=( )
+for fic in $postsdir/**/*.gmi; do
+    postfile="$(echo "$fic"|sed 's#^'$postsdir'/##')"
+    blogfile="$(echo "$fic"|sed 's#^'$webdir'/##')"
+    printf "%-30s" $postfile
+    xfic="$fic"
+    d=$(finddate $xfic)
+    echo -n " [$d]"
+    rssdate=$(formatdate $d)
+    title=$(findtitle $xfic)
+    keywords=( $(findkeywords $xfic) )
+    printf ": %-55s" "$title ($keywords)"
+    absoluteurl="${websiteurl}/${blogfile}"
+    {
+      printf "=> %s %s: %s [%s]\n" "$absoluteurl" "$rssdate" "$title" "$keywords"
+    } >>  "$tmpdir/${d}-$(basename $fic).gmi"
+    dates=( $d $dates )
+    echo " [${fg[green]}OK${reset_color}]"
+done
+echo "Publishing"
+for fic in $(ls $tmpdir/*.gmi | sort -r | head -n $maxarticles ); do
+    echo "${fic:t}"
+    cat $fic >> $tmpdir/gmi
+done
+
+rssmaxdate=$(formatdate $(for d in $dates; do echo $d; done | sort -r | head -n 1))
+rssbuilddate=$(formatdate $(date))
+{
+cat <<END
+    ,---,
+   / <=> \\
+  (  (O)  )
+   \\     /
+    '---'
+  YOGSOTOTH
+
+The index of my articles.
+I talk about programming and sometime movies.
+Some articles are only intended for gemini.
+Enjoy!
+
+END
+cat $tmpdir/gmi
+} > "$indexfile"
+
+rm -rf $tmpdir
+echo "* Gemini Index [done]"

engine/mkrss.sh

@@ -121,5 +121,11 @@ cat <<END
 END
 } > "$rssfile"
 
+# HACK TO UPDATE OLD RSS FEEDS
+legacyenrss="$webdir/Scratch/en/blog/feed/feed.xml"
+legacyfrrss="$webdir/Scratch/fr/blog/feed/feed.xml"
+cp -f "$rssfile" "$legacyenrss"
+cp -f "$rssfile" "$legacyfrrss"
+
 rm -rf $tmpdir
 echo "* RSS [done]"

engine/org2gemini.sh

@@ -7,13 +7,14 @@ dst="$2"
 
 ./engine/org2gemini_step1.sh "$src" | \
   perl -pe 's#^email:\s+yann\@esposito.host\s*#$&=> /files/publickey.txt gpg\n#g;' | \
-  perl -pe 's#\[\[([^]]*)\]\[([^]]*)\]\]#\n=> $1 $2#g;' | \
+  perl -pe 's# ?\[\[([^]]*)\]\[([^]]*)\]\]#\n\n=> $1 $2\n#g;' | \
   perl -pe 's#=> file:([^ ]*)\.org#=> $1.gmi#g;' | \
   perl -pe 's#=> file:([^ ]*)#=> $1#g;' | \
   perl -pe 's#\[\[(file:)?([^]]*)\]\]#=> $2#g;' | \
-  perl -pe 's#^\* *\n##' | \
+  perl -pe 's#^\* *\n\n##' | \
+  perl -pe 's#^\s*[-\*\+\.]\s*$##' | \ # remove lines with a single special char
   perl -pe 's#^\**[ ]*:.*:$##' | \
-  perl -pe 's#^\s[- ]*$##;' > "$dst"
+  perl -pe 's#^\s[- ]*$#\n#;' > "$dst"
 
 { echo ""
   echo "=> /index.gmi Home"

engine/org2gemini_step1.sh

@@ -13,6 +13,8 @@ BEGIN { IGNORECASE=1; }
 
 /^#\+TITLE: / { gsub(/^#[^:]*: /,"# "); }
 /^ *:[a-zA-Z_0-9]*:/ { skip=1; }
+
+# title
 /^\* / { gsub(/^\* /,"# "); }
 /^\*\* / { gsub(/^\*\* /,"## "); }
 /^\*\*\* / { gsub(/^\*\*\* /,"### "); }
@@ -27,7 +29,7 @@ BEGIN { IGNORECASE=1; }
    $0=x" "$0;
 }
 /^- / { gsub(/^- /,"* "); }
-!skip && !htmlskip{ 
+!skip && !htmlskip{
   print;
 }
 /@@/ && !/@@html:/ { htmlskip = 0; }

engine/pre-deploy.sh

@@ -10,9 +10,11 @@ echo "Optim HTML size"
 # ./engine/dup-for-themes.sh
 echo "Optim Classes accross CSS/HTML"
 ./engine/optim-classes.sh
-echo "Update file size"
-./engine/update-file-size.sh
+# echo "Update file size"
+# ./engine/update-file-size.sh
 echo "Building RSS"
 ./engine/mkrss.sh
+echo "Building Gemini Index"
+./engine/mk-gemini-index.sh
 echo "Building Gemini Atom"
 ./engine/mk-gemini-atom.sh

src/css/y.css

@@ -43,7 +43,11 @@ footer { margin: 3em 0;
          line-height: 1em;
          font-size: 0.875em;
        }
-table { margin: 1rem 0; line-height: 1em; }
+table { margin: 1rem 0; line-height: 1em; max-width: 100%; overflow: scroll;
+        display: block;
+        border: solid;}
+table th { padding: .1em 1em; }
+table td { padding: .1em 1em; }
 nav { text-align: center; padding: 1em 0;  }
 pre { line-height: 1em; }
 
@@ -141,7 +145,10 @@ body, body > div {
 }
 label:hover, a,a:visited { color: var(--hl); }
 code { background: var(--rbg); }
-
+table th { background: var(--rbg); }
+blockquote { margin: 0 1em;
+             padding-left: 1em;
+             border-left: solid var(--rbg); }
 /* ---- SYNTAX HIGHLIGHTING ---- */
 #table-of-contents { text-align: left; }
 .ex { color:var(--v); }
@@ -154,3 +161,6 @@ code { background: var(--rbg); }
 
 .pubDate { font-size: .7em; color: var(--b1); }
 .tag { font-size: .7em; background-color: var(--b2); }
+
+.todo,.done { background: var(--r); color: #FFF; font-weight: bold; font-size: .66em; padding: .2em;}
+.done { background: var(--g); }

src/demo.org

@@ -97,7 +97,7 @@ In the middle of the text again $x$ and $x_i\times 0$.
 
 \(x^y / \log(x)\)
 
-\[  \prod_{i=0}^n \sum_{x_i\in E} \frac{1}{x_i} \]
+\( \prod_{i=0}^n \sum_{x_i\in E} \frac{1}{x_i} \)
 
 * Blocks
 :PROPERTIES:
@@ -394,8 +394,6 @@ CLOSED: [2019-07-09 Tue 13:45]
 :PROPERTIES:
 :CUSTOM_ID: canceled-status
 :END:
-- State "CANCELED"   from              [2019-07-09 Tue 13:45] \\
-cancel reason
 * Level 1
 :PROPERTIES:
 :CUSTOM_ID: level-1
@@ -480,15 +478,29 @@ cancel reason
 :PROPERTIES:
 :CUSTOM_ID: todo-9
 :END:
-********** TODO Todo 10
+:LOGBOOK:
+- State "TODO"       from "HOLD"       [2021-05-01 Sat 17:02]
+- State "HOLD"       from "TODO"       [2021-05-01 Sat 16:50] \\
+  reason
+:END:
+********** DONE Todo 10
 :PROPERTIES:
 :CUSTOM_ID: todo-10
 :END:
-*********** TODO Todo 11
+:LOGBOOK:
+- State "DONE"       from "CANCELED"   [2021-05-01 Sat 17:02]
+- State "CANCELED"   from "TODO"       [2021-05-01 Sat 16:49] \\
+  because
+:END:
+*********** DONE Todo 11
 :PROPERTIES:
 :CUSTOM_ID: todo-11
 :END:
-************ TODO Todo 12
+:LOGBOOK:
+- State "DONE"       from "HANDLED"    [2021-05-01 Sat 17:02]
+- State "HANDLED"    from "TODO"       [2021-05-01 Sat 16:49]
+:END:
+************ DONE Todo 12
 :PROPERTIES:
 :CUSTOM_ID: todo-12
 :END:

src/posts/0003-project-el/index.org

@@ -41,7 +41,7 @@ instead of 'n' and to load 3rd party script.
 Note that checking who signed a file with an external signature is not as
 straightforward as it should be:
 
-#+begin_src elisp
+#+begin_src lisp
 (defun auto-load-project/get-sign-key (file)
   "Return the fingerprint of they key that signed FILE.
 
@@ -84,7 +84,7 @@ The project is hosted here: https://gitlab.esy.fun/yogsototh/auto-load-project-e
 
 You can setup the emacs package in spacemacs with:
 
-#+begin_src elisp
+#+begin_src lisp
   ;; ...
   dotspacemacs-additional-packages
   '((auto-load-project :location
@@ -107,7 +107,7 @@ You can setup the emacs package in spacemacs with:
 The full current code should be easy to follow if you have basic notions
 of eLISP:
 
-#+begin_src elisp
+#+begin_src lisp
 (require 'projectile)
 
 (defvar auto-load-project/trusted-gpg-key-fingerprints

src/posts/0017-static-blog-builder/index.org

@@ -0,0 +1,226 @@
+#+TITLE: Static Blog Builder
+#+SUBTITLE: A few static blog rewrite experiences
+#+AUTHOR: Yann Esposito
+#+EMAIL: yann@esposito.host
+#+DATE: [2021-05-01 Sat]
+#+KEYWORDS: blog static
+#+DESCRIPTION: Minimal and fast static website builder with make.
+#+OPTIONS: auto-id:t toc:t
+
+As someone on the Internet said not so far ago.
+Building its own static building system is a rite of passage for many developers.
+It has a lot of nice features.
+It gives a goal with a feeling of accomplishment.
+It is simple enough so most developers could build their own system.
+But it could also become very complex when you go down the rabbit hole.
+
+Along the years I used different tools and used and wrote of few static
+website systems:
+
+- [[https://nanoc.app][nanoc]] (in Ruby), at that time it looked like this: [[https://web.archive.org/web/20081002071448/http://nanoc.stoneship.org/][old nanoc 2 website]]
+- [[https://jaspervdj.be/hakyll/][hakyll]] (haskell static website generator)
+- [[https://orgmode.org/worg/org-tutorials/org-publish-html-tutorial.html][org-publish]] (emacs package in conjunction with org-mode)
+- [[https://shakebuild.com][shake]] (haskell again)
+
+So if you look at the progression, I first used nanoc because I used ruby
+and it was a very new solution, the website looked really great.
+Also the main developer [[https://denisdefreyne.com][Denis Defreyne]] was really helpful.
+Ruby was really great at dealing with regular expressions for hacking my
+documents.
+
+Then I was interested in Haskell, and I switched to a Haskell-made
+solution.
+I used hakyll, and I wrote a bit about it in [[http://yannesposito.com/Scratch/en/blog/Hakyll-setup/][Hakyll Setup]].
+As a side note, the author of Hakyll [[https://jaspervdj.be/hakyll/][Jasper Van der Jeugt]] is apparently a
+friend of the author of nanoc.
+They both wrote a static site generators with their preferred programming
+language.
+I added a lot of personal features to my own site builder.
+It was a nice toy project.
+
+Then, due to a major disruption in my professional and private life I
+stopped to take care of my website.
+
+And a few years ago, I wanted to start a new website from scratch.
+In the meantime I switched my editor of choice from vim to Emacs.
+I started to work in Clojure and emacs is generally a natural choice
+because you can configure it with LISP.
+I discovered [[https://orgmode.org/worg/org-tutorials/org-publish-html-tutorial.html][org-mode]] (I don't think the homepage of org mode makes justice
+to how incredible it is).
+So org-mode comes with an export system.
+Thus I switched to org-publish.
+Again [[https://her.esy.fun/posts/0001-new-blog/index.html][I wrote a bit about it]].
+
+It was nice, but very slow.
+I improved a few things like writing a short script to
+[[https://her.esy.fun/posts/0005-rss-gen/index.html][Generate RSS from a tree of html files.]]
+But I still had the feeling it was too slow.
+
+Static site building is a specific usage of a build system.
+And as I knew I could use =pandoc= to build HTML out of org-mode files
+and still versed in the Haskell culture I decided to try [[https://shakebuild.com][shake]].
+You can learn more by reading this excellent paper about it, I
+think all developer should read it: [[https://github.com/snowleopard/build-systems/releases/download/icfp-submission/build-systems.pdf][Build System à la carte]].
+
+As a bonus, [[https://pandoc.org][pandoc]] is written in Haskell.
+I could then directly use the [[https://pandoc.org][pandoc]] library in my build program.
+It worked like a charm and it was *very fast* as compared to other
+solutions I tried.
+So really let me tell you shake is a great build system.
+
+But it was not perfect.
+While it was very fast, and I was able to use pandoc API directly.
+It made me dependent on Haskell.
+The best way I found to have Haskell reproducible build environment is to
+use [[https://nixos.org/nix][nix]].
+This was great until the Big Sur update.
+To keep it short, nix stopped working on my computers after I upgraded my
+to Big Sur.
+Gosh, it was painful to fix.
+
+Concurrently I discovered [[/posts/0016-gemini/index.html][gemini]] and wanted to duplicate my website into
+gemini sphere.
+So I tried to update my build system but my code was to oriented to use
+pandoc and it was painful to have gemini in the middle of it.
+Particularly, generating a gemini index file.
+My main goal was to have gemini file that could only be linked from withing
+gemini sphere.
+Because gemini is a lot smaller web where you could feel a bit more
+protected from what the Web has become along the years.
+Whatever, in the end, I just had two problems to tackles.
+
+1. Haskell became difficult to trust as very stable tool. Stable in the
+   sense that I would not have any support work to do in order to keep just
+   using it and not fixing/tweaking it.
+2. Simplify the overall system to have a simpler build description
+
+So a very stable tool that I am pretty sure will still work almost exactly
+as today in 10 years is *=make=* (more precisely gnumake).
+I expected a lot of people had already come to the same conclusion and
+wrote about it.
+To my great surprise, I found very few article about generating static
+website with make.
+I only found solutions a bit too specific for my need.
+This is why I would like to give you a more generic starting point
+solution.
+
+* The =Makefile=
+:PROPERTIES:
+:CUSTOM_ID: the--makefile-
+:END:
+
+Instead of copy/pasting my current =Makefile= entirely let me give you a
+more generic one.
+It should be a great start.
+
+The first part will be used to simply copy the files from =src/= to
+=_site/=.
+
+#+begin_src makefile
+all: website
+
+# directory containing my org files as well as my assets files
+SRC_DIR ?= src
+# directory where I will but the files for my website (HTML + assets)
+DST_DIR ?= _site
+
+# list all files in src
+# if you want to exclude .org files use the exclude from the find command
+SRC_RAW_FILES := $(shell find $(SRC_DIR) -type f)
+# generate all file that should be copied in the site
+# For my site, I want to publish my source files along the HTML files
+DST_RAW_FILES   := $(patsubst $(SRC_DIR)/%,$(DST_DIR)/%,$(SRC_RAW_FILES))
+ALL             += $(DST_RAW_FILES)
+
+# COPY EVERYTHING (.org file included)
+$(DST_DIR)/% : $(SRC_DIR)/%
+	mkdir -p "$(dir $@)"
+	cp "$<" "$@"
+#+end_src
+
+This part is about running the =pandoc= command for all =org= files in =src/=
+so they generate a html file in =_site/=.
+
+#+begin_src makefile
+# ORG -> HTML, If you prefer markdown replace .org by .md
+EXT := .org
+# all source file we'll pass to pandoc
+SRC_PANDOC_FILES ?= $(shell find $(SRC_DIR) -type f -name "*$(EXT)")
+# all destination files we expect (replace the extension by .html)
+DST_PANDOC_FILES ?= $(subst $(EXT),.html, \
+                        $(subst $(SRC_DIR),$(DST_DIR), \
+                            $(SRC_PANDOC_FILES)))
+ALL              += $(DST_PANDOC_FILES)
+
+# use a template (you should use one)
+TEMPLATE ?= templates/post.html
+# URL of the CSS put yours
+CSS = /css/y.css
+# The pandoc command to run to generate an html out of a source file
+PANDOC := pandoc \
+			-c $(CSS) \
+			--template=$(TEMPLATE) \
+			--from org \
+			--to html5 \
+			--standalone
+
+# Generate all html if the org file change or the template change
+$(DST_DIR)/%.html: $(SRC_DIR)/%.org $(TEMPLATE)
+	mkdir -p $(dir $@)
+	$(PANDOC) $< \
+		--output $@
+
+#+end_src
+
+A missing part is often the part where you would like to generate
+an index page to list the latest posts.
+Here you are a bit alone, you need to make one yourself.
+There is not generic way to do this one.
+
+#+begin_src makefile
+# Generating an index page is not difficult but not trivial either
+HTML_INDEX := $(DST_DIR)/index.html
+MKINDEX := engine/mk-index.sh
+$(HTML_INDEX): $(DST_PANDOC_FILES) $(MKINDEX)
+	mkdir -p $(DST_DIR)
+	$(MKINDEX)
+ALL += $(HTML_INDEX)
+#+end_src
+
+Finally, a few useful make commands. =make clean= and =make deploy=.
+
+#+begin_src makefile
+# make deploy will deploy the files to my website write your own script
+deploy: $(ALL)
+	engine/deploy.sh
+
+website: $(ALL)
+
+.PHONY: clean
+
+clean:
+	-rm -rf $(DST_DIR)/*
+#+end_src
+
+Limitation: =make= is old.
+So it really does not support spaces in filenames.
+Take care of that.
+
+But let me tell you.
+While this is quite a minimalist approach (<100 lines) it is nevertheless *very fast*.
+It will only generate the minimal amount of work to generate your website.
+I have a nice watcher script that update the website every time I save a
+file.
+It is almost instantaneous.
+
+The only risky dependencies for my website now is =pandoc=.
+Perhaps, they will change how they generate an HTML from the same org file
+in the future.
+I still use =nix= to pin my pandoc version.
+But the static site builder itself is very simple, very stable and still
+very efficient.
+
+As a conclusion, if you want to write your own static site builder that's great.
+There are plenty of things to learn along the way.
+Still if you want something stable for a long time, with a minimal amount
+of dependencies, I think this Makefile is really a great start.