From 815e675790c9d79eba2548a3679bc5ec09d261e7 Mon Sep 17 00:00:00 2001 From: Atul Varma Date: Mon, 3 Jul 2017 18:01:08 -0400 Subject: [PATCH 1/2] Support GitHub access tokens. Fixes #272. --- README.md | 8 ++++++++ _config.yml | 2 ++ _plugins/jekyll_get.rb | 34 ++++++++++++++++++++++++++-------- 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 8da2531b98..5d0982ee47 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,14 @@ https://federalist.fr.cloud.gov/preview/18f/web-design-standards-docs/develop/ See the [`_posts` directory](_posts/#readme) for instructions on adding updates. +### Dynamic content + +Some of the content on the documentation site is dynamically fetched from +GitHub. If you want to ensure that its API won't rate-limit you, you +may want to +[create an access token](https://github.com/blog/1509-personal-api-tokens) +and assign it to your `GITHUB_ACCESS_TOKEN` environment variable. + ## Contributing Please read through our [contributing guidelines](CONTRIBUTING.md). These guidelines are directions for opening issues and submitting pull requests, and they also detail the coding and design standards we follow. diff --git a/_config.yml b/_config.yml index b00fd1c471..f50c7edb3d 100644 --- a/_config.yml +++ b/_config.yml @@ -16,8 +16,10 @@ jekyll_get: json: 'https://api.github.com/repos/18F/web-design-standards/releases' - data: contributing json: 'https://api.github.com/repos/18F/web-design-standards/contents/CONTRIBUTING.md' + decode_content: true - data: standards-sites json: 'https://api.github.com/repos/18F/web-design-standards/contents/WHO_IS_USING_USWDS.md' + decode_content: true repos: - name: U.S. Web Design Standards diff --git a/_plugins/jekyll_get.rb b/_plugins/jekyll_get.rb index efb9be61e7..7d57b189f7 100644 --- a/_plugins/jekyll_get.rb +++ b/_plugins/jekyll_get.rb @@ -8,6 +8,26 @@ class Generator < Jekyll::Generator safe true priority :highest + def decode_content(source) + encoding = source['encoding'] + target_content = source['content'] + if encoding == 'base64' + source['decoded'] = Base64.decode64(target_content).force_encoding(Encoding::UTF_8) + else + source['decoded'] = target_content + end + end + + def get_final_url(url) + if url.start_with? "https://api.github.com/" + access_token = ENV['GITHUB_ACCESS_TOKEN'] + if access_token + return "#{url}?access_token=#{access_token}" + end + end + url + end + def generate(site) config = site.config['jekyll_get'] if !config @@ -18,20 +38,17 @@ def generate(site) end config.each do |d| name_of_target = d['data'] + url = d['json'] begin target = site.data[name_of_target] - source = JSON.load(open(d['json'])) + source = JSON.load(open(get_final_url(url))) if target HashJoiner.deep_merge target, source else site.data[name_of_target] = source end - encoding = site.data[name_of_target]['encoding'] - target_content = site.data[name_of_target]['content'] - if encoding == 'base64' - site.data[name_of_target]['decoded'] = Base64.decode64(target_content).force_encoding(Encoding::UTF_8) - else - site.data[name_of_target]['decoded'] = target_content + if d['decode_content'] + decode_content site.data[name_of_target] end if d['cache'] data_source = (site.config['data_source'] || '_data') @@ -40,7 +57,8 @@ def generate(site) file << JSON.generate(site.data[name_of_target]) end end - rescue + rescue => e + print "jekyll_get: error fetching #{url}: #{e}\n" next end end From 00be324cca890d3b3bb8be856748349c8b99bedc Mon Sep 17 00:00:00 2001 From: Atul Varma Date: Wed, 5 Jul 2017 07:46:28 -0400 Subject: [PATCH 2/2] Cache github data in .jekyll_get_cache. --- .gitignore | 1 + Gemfile | 1 - README.md | 9 +++++++++ _plugins/jekyll_get.rb | 39 ++++++++++++++++++++------------------- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index cb9d3e01ea..ae95f1046c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ node_modules/ npm-debug.log .bundle/ assets/ +.jekyll_get_cache/ diff --git a/Gemfile b/Gemfile index f32edceae7..5e2dfbea7d 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,6 @@ source 'https://rubygems.org' gem 'jekyll' gem 'json' gem 'redcarpet' -gem 'hash-joiner' gem 'open-uri-cached' gem 'jekyll-redirect-from' diff --git a/README.md b/README.md index 5d0982ee47..13ee3ea6f9 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,15 @@ may want to [create an access token](https://github.com/blog/1509-personal-api-tokens) and assign it to your `GITHUB_ACCESS_TOKEN` environment variable. +The dynamic content is stored in the `.jekyll_get_cache` directory and +won't be re-fetched once it's cached there. However, this means that your +data can get stale over time, so if you want to ensure that your site +is using the very latest data, you'll want to clear the cache by running: + +``` +rm -rf .jekyll_get_cache +``` + ## Contributing Please read through our [contributing guidelines](CONTRIBUTING.md). These guidelines are directions for opening issues and submitting pull requests, and they also detail the coding and design standards we follow. diff --git a/_plugins/jekyll_get.rb b/_plugins/jekyll_get.rb index 7d57b189f7..d9243e0206 100644 --- a/_plugins/jekyll_get.rb +++ b/_plugins/jekyll_get.rb @@ -1,5 +1,4 @@ require 'json' -require 'hash-joiner' require 'open-uri' require 'base64' @@ -28,6 +27,25 @@ def get_final_url(url) url end + def load_json(site, d) + name = d['data'] + url = d['json'] + data_source = '.jekyll_get_cache' + path = "#{data_source}/#{name}.json" + if not File.exists?(path) + FileUtils.mkpath File.dirname(path) + print "Caching #{url} in #{path}...\n" + data = JSON.load(open(get_final_url(url))) + open(path, 'wb') do |file| + file << JSON.pretty_generate(data) + end + end + site.data[name] = JSON.load(open(path)) + if d['decode_content'] + decode_content site.data[name] + end + end + def generate(site) config = site.config['jekyll_get'] if !config @@ -37,26 +55,9 @@ def generate(site) config = [config] end config.each do |d| - name_of_target = d['data'] url = d['json'] begin - target = site.data[name_of_target] - source = JSON.load(open(get_final_url(url))) - if target - HashJoiner.deep_merge target, source - else - site.data[name_of_target] = source - end - if d['decode_content'] - decode_content site.data[name_of_target] - end - if d['cache'] - data_source = (site.config['data_source'] || '_data') - path = "#{data_source}/#{name_of_target}.json" - open(path, 'wb') do |file| - file << JSON.generate(site.data[name_of_target]) - end - end + load_json(site, d) rescue => e print "jekyll_get: error fetching #{url}: #{e}\n" next