Support GitHub access tokens and cache API results

.gitignore

@@ -7,3 +7,4 @@ node_modules/
 npm-debug.log
 .bundle/
 assets/
+.jekyll_get_cache/

Gemfile

@@ -3,7 +3,6 @@ source 'https://rubygems.org'
 gem 'jekyll'
 gem 'json'
 gem 'redcarpet'
-gem 'hash-joiner'
 gem 'open-uri-cached'
 gem 'jekyll-redirect-from'
 

README.md

@@ -76,6 +76,23 @@ https://federalist.fr.cloud.gov/preview/18f/web-design-standards-docs/develop/
 
 See the [`_posts` directory](_posts/#readme) for instructions on adding updates.
 
+### Dynamic content
+
+Some of the content on the documentation site is dynamically fetched from
+GitHub. If you want to ensure that its API won't rate-limit you, you
+may want to
+[create an access token](https://github.com/blog/1509-personal-api-tokens)
+and assign it to your `GITHUB_ACCESS_TOKEN` environment variable.
+
+The dynamic content is stored in the `.jekyll_get_cache` directory and
+won't be re-fetched once it's cached there. However, this means that your
+data can get stale over time, so if you want to ensure that your site
+is using the very latest data, you'll want to clear the cache by running:
+
+```
+rm -rf .jekyll_get_cache
+```
+
 ## Contributing
 
 Please read through our [contributing guidelines](CONTRIBUTING.md). These guidelines are directions for opening issues and submitting pull requests, and they also detail the coding and design standards we follow.

_config.yml

@@ -16,8 +16,10 @@ jekyll_get:
     json: 'https://api.github.com/repos/18F/web-design-standards/releases'
   - data: contributing
     json: 'https://api.github.com/repos/18F/web-design-standards/contents/CONTRIBUTING.md'
+    decode_content: true
   - data: standards-sites
     json: 'https://api.github.com/repos/18F/web-design-standards/contents/WHO_IS_USING_USWDS.md'
+    decode_content: true
 
 repos:
 - name: U.S. Web Design Standards

_plugins/jekyll_get.rb

@@ -1,5 +1,4 @@
 require 'json'
-require 'hash-joiner'
 require 'open-uri'
 require 'base64'
 
@@ -8,6 +7,45 @@ class Generator < Jekyll::Generator
     safe true
     priority :highest
 
+    def decode_content(source)
+      encoding = source['encoding']
+      target_content = source['content']
+      if encoding == 'base64'
+        source['decoded'] = Base64.decode64(target_content).force_encoding(Encoding::UTF_8)
+      else
+        source['decoded'] = target_content
+      end
+    end
+
+    def get_final_url(url)
+      if url.start_with? "https://api.github.com/"
+        access_token = ENV['GITHUB_ACCESS_TOKEN']
+        if access_token
+          return "#{url}?access_token=#{access_token}"
+        end
+      end
+      url
+    end
+
+    def load_json(site, d)
+      name = d['data']
+      url = d['json']
+      data_source = '.jekyll_get_cache'
+      path = "#{data_source}/#{name}.json"
+      if not File.exists?(path)
+        FileUtils.mkpath File.dirname(path)
+        print "Caching #{url} in #{path}...\n"
+        data = JSON.load(open(get_final_url(url)))
+        open(path, 'wb') do |file|
+          file << JSON.pretty_generate(data)
+        end
+      end
+      site.data[name] = JSON.load(open(path))
+      if d['decode_content']
+        decode_content site.data[name]
+      end
+    end
+
     def generate(site)
       config = site.config['jekyll_get']
       if !config
@@ -17,30 +55,11 @@ def generate(site)
         config = [config]
       end
       config.each do |d|
-        name_of_target = d['data']
+        url = d['json']
         begin
-          target = site.data[name_of_target]
-          source = JSON.load(open(d['json']))
-          if target
-            HashJoiner.deep_merge target, source
-          else
-            site.data[name_of_target] = source
-          end
-          encoding = site.data[name_of_target]['encoding']
-          target_content = site.data[name_of_target]['content']
-          if encoding == 'base64'
-            site.data[name_of_target]['decoded'] = Base64.decode64(target_content).force_encoding(Encoding::UTF_8)
-          else
-            site.data[name_of_target]['decoded'] = target_content
-          end
-          if d['cache']
-            data_source = (site.config['data_source'] || '_data')
-            path = "#{data_source}/#{name_of_target}.json"
-            open(path, 'wb') do |file|
-              file << JSON.generate(site.data[name_of_target])
-            end
-          end
-        rescue
+          load_json(site, d)
+        rescue => e
+          print "jekyll_get: error fetching #{url}: #{e}\n"
           next
         end
       end