Improve SEO by pre-rendering all HTML pages

* Change all internal links to use URLs ending in `/`
* Add the prerender script which uses PhantomJS to pre-render the
  HTML for all possible pages in the system
* Change the build process to incorporate the pre-rendered files
* Update the sitemap script to generate the new URLs

scripts/deploy

@@ -2,7 +2,7 @@
 
 export AWS_CONFIG_FILE="$HOME/Documents/CraftingGuide/aws.config"
 
-DIST="./dist"
+ROOT="./dist"
 SITE="crafting-guide.com"
 TARGET=""
 while [[ "$1" != "" ]]; do
@@ -19,9 +19,9 @@ if [[ "$TARGET" == "" ]]; then
 fi
 
 if [[ "$TARGET" == "new" ]]; then
-    find "$DIST" -name ".DS_Store" | xargs rm
+    find "$ROOT" -name ".DS_Store" | xargs rm
     grunt clean dist
-    s3cmd sync -r --delete-removed --add-header="cache-control:max-age=1800" "$DIST/" "s3://new.$SITE/"
+    s3cmd sync -r --delete-removed --add-header="cache-control:max-age=1800" "$ROOT/" "s3://new.$SITE/"
     grunt clean build
 elif [[ "$TARGET" == "promote" ]]; then
     aws s3 sync "s3://new.$SITE" "s3://$SITE" --delete

scripts/prerender

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+OUTPUT_DIR='./prerender'
+SCRIPT='/tmp/phantomjs_render.coffee'
+SITEMAP_FILE='./src/pages/sitemap.txt'
+
+cat > $SCRIPT <<END
+url      = require('system').args[1]
+page     = require('webpage').create()
+path     = "#{url}".replace(/http:\/\/[^/]*/, '').replace(/^\//, '').replace(/\/$/, '')
+localUrl = "http://localhost:8000/#{path}"
+
+page.open localUrl, (status)->
+    if status isnt 'success'
+        console.error "Could not load #{url}"
+        phantom.exit(1)
+
+    setTimeout (->
+        console.log page.content
+        phantom.exit(0)
+    ), 5000
+END
+
+./scripts/sitemap > $SITEMAP_FILE
+
+cat $SITEMAP_FILE | while read URL; do
+    OUTPUT_FILE=$(sed 's/http:\/\/[^/]*\///' <<< $URL)
+    OUTPUT_FILE="$OUTPUT_DIR/$OUTPUT_FILE/index.html"
+    OUTPUT_FILE=$(sed 's/\/\//\//' <<< $OUTPUT_FILE)
+
+    if [[ ! -e "$OUTPUT_FILE" ]]; then
+        mkdir -p $(dirname "$OUTPUT_FILE")
+        echo "prerendering $URL into $OUTPUT_FILE"
+        phantomjs --web-security=false --load-images=false $SCRIPT "$URL" >"$OUTPUT_FILE" 2>/dev/null
+    else
+        echo "Skipping $URL..."
+    fi
+done
+
+rm $SCRIPT

scripts/prerender.coffee

@@ -1,51 +0,0 @@
-#!/usr/bin/env phantomjs --web-security=false
-
-fs      = require 'fs'
-system  = require 'system'
-webPage = require 'webpage'
-
-if system.args.length isnt 2
-    console.error 'USAGE: prerender <path>'
-    phantom.exit 1
-
-########################################################################################################################
-
-urlsFileName      = system.args[1]
-urls              = fs.read(urlsFileName).split('\n')
-outputDir         = 'prerender'
-currentPage       = null
-currentOutputPath = null
-renderTime        = 5000
-
-urls = (url.trim() for url in urls when url.trim().length > 0)
-
-if not fs.exists outputDir
-    fs.makeDirectory outputDir
-
-startNextPage = ->
-    url = urls.pop()
-    if !url then phantom.exit()
-
-    path              = "#{url}".replace(/http:\/\/[^/]*/, '').replace(/^\//, '')
-    url               = "http://localhost:8000/#{path}"
-    currentOutputPath = "#{outputDir}/#{path}"
-    currentOutputPath = "#{outputDir}/index.html" if currentOutputPath is ''
-
-    if not fs.exists currentOutputPath
-        currentPage = webPage.create()
-        console.log "Opening #{url}..."
-        currentPage.open url, completePage()
-    else
-        setTimeout startNextPage, 0
-
-completePage = (status)->
-    if status is 'fail'
-        console.error "Could not load #{url}"
-        startNextPage()
-
-    setTimeout (->
-        fs.write currentOutputPath, currentPage.content, 'w'
-        startNextPage()
-    ), renderTime
-
-startNextPage()

scripts/serve-prerendered

@@ -0,0 +1,41 @@
+#!/usr/bin/env coffee
+#
+# Crafting Guide - serve-prerendered
+#
+# Copyright (c) 2014-2015 by Redwood Labs
+# All rights reserved.
+#
+
+express = require 'express'
+path    = require 'path'
+
+########################################################################################################################
+
+ROOT = './dist'
+
+########################################################################################################################
+
+app = express()
+
+app.get '*', (request, response)->
+    effectivePath = request.path.replace /\/$/, '/index.html'
+
+    extension = path.extname(effectivePath)
+    contentType = {
+        '.css':  'text/css'
+        '.html': 'text/html'
+        '.js':   'text/javascript'
+        '.json': 'application/json'
+        '.map':  'application/json'
+        '.png':  'image/png'
+        '.ttf':  'application/octet-stream'
+        '.scss': 'text/css'
+    }[extension]
+
+    console.log ">>> #{effectivePath}"
+
+    response.set 'Content-Type', contentType
+    setTimeout (-> response.sendFile effectivePath, root:ROOT), 100 # post files after a short delay
+
+server = app.listen 8001, ->
+    console.log "listening at: #{server.address().address}:#{server.address().port}"

scripts/server

@@ -1,6 +1,6 @@
 #!/usr/bin/env coffee
 #
-# Crafting Guide - server
+# Crafting Guide - serve-build
 #
 # Copyright (c) 2014-2015 by Redwood Labs
 # All rights reserved.
@@ -45,10 +45,7 @@ app.get '*', (request, response)->
     console.log ">>> #{request.path}"
 
     response.set 'Content-Type', contentType
-
-    sendFile = -> response.sendFile request.path, root:ROOT
-
-    setTimeout sendFile, 100 # change the zero to post files after a short delay
+    response.sendFile request.path, root:ROOT
 
 server = app.listen 8000, ->
     console.log "listening at: #{server.address().address}:#{server.address().port}"

scripts/sitemap

@@ -19,10 +19,10 @@ itemSlugs = {}
 base = 'http://crafting-guide.com'
 
 urls = [
-    base
-    "#{base}/configure"
-    "#{base}/browse"
-    "#{base}/craft"
+    "#{base}/"
+    "#{base}/configure/"
+    "#{base}/browse/"
+    "#{base}/craft/"
 ]
 
 for modSlug in fs.readdirSync './src/data'
@@ -41,11 +41,11 @@ for modSlug in fs.readdirSync './src/data'
             itemSlugs[item.slug] = item.slug
 
 for modSlug in _.values(modSlugs).sort()
-    urls.push base + "/browse/#{modSlug}"
+    urls.push base + "/browse/#{modSlug}/"
 
 for key in _.keys(itemSlugs).sort()
     itemSlug = itemSlugs[key]
-    urls.push base + "/browse/#{itemSlug.mod}/#{itemSlug.item}"
+    urls.push base + "/browse/#{itemSlug.mod}/#{itemSlug.item}/"
 
 for url in urls
     console.log url