Data Conversion + Gatsby

This commit adds convert.py that reads the old datasets.yml and writes it as a set of markdown files in src/datasets/ Some data massaging needed to be done, especially to supply missing dates. Also included in this commit is a barebone Gatsby application which is replacing Jekyll so that it can work more like an web application (sorting, filtering, etc).
DocNow · Mar 29, 2020 · ba04be8 · ba04be8
1 parent 276b921
commit ba04be8
Show file tree

Hide file tree

Showing 135 changed files with 20,186 additions and 205 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
-_site
-.sass-cache
-.jekyll-metadata
+node_modules
+public
+*.log
+.cache
+.vscode
diff --git a/Gemfile b/Gemfile
diff --git a/_config.yml b/_config.yml
diff --git a/_includes/footer.html b/_includes/footer.html
diff --git a/_includes/head.html b/_includes/head.html
diff --git a/_includes/header.html b/_includes/header.html
diff --git a/_layouts/default.html b/_layouts/default.html
diff --git a/convert.py b/convert.py
@@ -0,0 +1,45 @@
+# A one-off script used to convert the old catalog YAML into separate Markdown files.
+
+import re
+import yaml
+from datetime import date, datetime
+
+from dateutil.parser import parse
+
+datasets = yaml.load(open('static/data/datasets.yml'), Loader=yaml.Loader)
+datasets.sort(key=lambda d: d['added'])
+
+def json_serial(obj):
+    if isinstance(obj, (datetime, date)):
+        return obj.isoformat()
+    raise TypeError ("Type %s not serializable" % type(obj))
+
+def unpack_date(s):
+    dates = s.split(' - ')
+    dates = list(map(lambda d: parse(d).date(), dates))
+    return {"start": dates[0], "end": dates[1]}
+
+id = 0
+for d in datasets:
+    id += 1
+    print(id, d['title'])
+
+    desc = d['description']
+    del d['description']
+
+    if type(d['creator']) != list:
+        d['creator'] = [d['creator']]
+
+    if type(d['dates']) == list:
+        d['dates'] = list(map(unpack_date, d['dates']))
+    else:
+        d['dates'] = [unpack_date(d['dates'])]
+
+    meta = yaml.dump(d, default_flow_style=False)
+
+    path = "src/datasets/{0:05n}.md".format(id)
+    fh = open(path, 'w')
+    fh.write('---\n')
+    fh.write(meta)
+    fh.write('---\n\n')
+    fh.write(desc)
diff --git a/css/style.css b/css/style.css
diff --git a/datasets/README.md b/datasets/README.md
diff --git a/feed.xml b/feed.xml
diff --git a/gatsby-browser.js b/gatsby-browser.js
@@ -0,0 +1,7 @@
+/**
+ * Implement Gatsby's Browser APIs in this file.
+ *
+ * See: https://www.gatsbyjs.org/docs/browser-apis/
+ */
+
+// You can delete this file if you're not using it
diff --git a/gatsby-config.js b/gatsby-config.js
@@ -0,0 +1,26 @@
+module.exports = {
+  siteMetadata: {
+    title: `DocNow Tweet Catalog`,
+    description: `A catalog of tweet identifier datasets`,
+    author: `Documenting the Now`,
+  },
+  plugins: [
+    `gatsby-plugin-react-helmet`,
+    `gatsby-transformer-json`,
+    `gatsby-transformer-remark`,
+    {
+      resolve: `gatsby-source-filesystem`,
+      options: {
+        name: `datasets`,
+        path: `${__dirname}/src/datasets`
+      }
+    },
+    {
+      resolve: `gatsby-source-filesystem`,
+      options: {
+        name: `images`,
+        path: `${__dirname}/src/images`,
+      },
+    }
+  ]
+}
diff --git a/gatsby-node.js b/gatsby-node.js
@@ -0,0 +1,49 @@
+const fs = require('fs')
+const path = require('path')
+
+exports.createPages = async ({ actions: { createPage }, graphql, pathPrefix }) => {
+  await makeDatasets(createPage, graphql, pathPrefix)
+}
+
+async function makeDatasets(createPage, graphql, pathPrefix) {
+  const results = await graphql(`
+    query {
+      allMarkdownRemark {
+        nodes {
+          frontmatter {
+            title
+            creator
+            added
+            published
+            dates {
+              start
+              end
+            }
+            tweets
+            tags
+            url
+          }
+          html
+          fileAbsolutePath
+        }
+      }
+    }
+  `)
+
+  const datasets = []
+  for (let dataset of results.data.allMarkdownRemark.nodes) {
+    const id = Number.parseInt(path.basename(dataset.fileAbsolutePath))
+    const context = {
+      id: id,
+      description: dataset.html,
+      ...dataset.frontmatter
+    }
+    createPage({
+      path: `/datasets/${id}/`,
+      component: require.resolve(`./src/templates/dataset.js`),
+      context: context
+    })
+    datasets.push(context)
+  }
+  fs.writeFileSync(`static/data/datasets.json`, JSON.stringify(datasets, null, 2))
+}
diff --git a/gatsby-ssr.js b/gatsby-ssr.js
@@ -0,0 +1,7 @@
+/**
+ * Implement Gatsby's SSR (Server Side Rendering) APIs in this file.
+ *
+ * See: https://www.gatsbyjs.org/docs/ssr-apis/
+ */
+
+// You can delete this file if you're not using it
diff --git a/index.html b/index.html