Enable pandoc spellcheck filter

merges manubot/rootstock#333 Installs and runs aspell via the pandoc spellcheck filter when the SPELLCHECK environment variable is true. Spellcheck outputs candidate spelling errors and the filenames and line numbers in which they occur. Initially only supported in Travis CI and AppVeyor builds.
greenelab · Apr 29, 2020 · 2c28689 · 2c28689
2 parents 073ce0e + 8dee14c
commit 2c28689
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 6 deletions.
diff --git a/.appveyor.yml b/.appveyor.yml
@@ -23,20 +23,30 @@ image: ubuntu
 services:
   - docker
 
+# Set SPELLCHECK to true to enable Pandoc spellchecking
+environment:
+  SPELLCHECK: true
+
 install:
   # Create the message with the triggering commit before install so it is
   # available if the build fails
   - TRIGGERING_COMMIT=${APPVEYOR_PULL_REQUEST_HEAD_COMMIT:-APPVEYOR_REPO_COMMIT}
-  - appveyor AddMessage "commit $TRIGGERING_COMMIT"
+  - JOB_MESSAGE=" for commit $TRIGGERING_COMMIT "
   - source ci/install.sh
-    
+
 test_script:
   - bash build/build.sh
   - MANUSCRIPT_FILENAME=manuscript-$APPVEYOR_BUILD_VERSION-${TRIGGERING_COMMIT:0:7}
   - cp output/manuscript.html $MANUSCRIPT_FILENAME.html
   - cp output/manuscript.pdf $MANUSCRIPT_FILENAME.pdf
   - appveyor PushArtifact $MANUSCRIPT_FILENAME.html
   - appveyor PushArtifact $MANUSCRIPT_FILENAME.pdf
+  - |
+      if [ "${SPELLCHECK:-}" = "true" ]; then
+        SPELLING_ERRORS_FILENAME=spelling-errors-$APPVEYOR_BUILD_VERSION-${TRIGGERING_COMMIT:0:7}.txt
+        cp output/spelling-errors.txt $SPELLING_ERRORS_FILENAME;
+        appveyor PushArtifact $SPELLING_ERRORS_FILENAME
+      fi
 
 build: off
 
@@ -46,6 +56,16 @@ cache:
 on_success:
   - echo "Artifacts available from $APPVEYOR_URL/project/$APPVEYOR_ACCOUNT_NAME/$APPVEYOR_PROJECT_SLUG/builds/$APPVEYOR_BUILD_ID/artifacts"
   - echo "Updated PDF available from $APPVEYOR_URL/api/buildjobs/$APPVEYOR_JOB_ID/artifacts/$MANUSCRIPT_FILENAME.pdf"
+  - appveyor AddMessage "$JOB_MESSAGE is now complete."
+  - |
+      if [ "${SPELLCHECK:-}" = "true" ]; then
+        SPELLING_ERROR_COUNT=($(wc -l $SPELLING_ERRORS_FILENAME))
+        appveyor AddMessage " <details><summary>Found $SPELLING_ERROR_COUNT potential spelling error(s). Preview:</summary>$(cat $SPELLING_ERRORS_FILENAME)"
+        appveyor AddMessage "... </details>"
+      fi
+
+on_failure:
+  - appveyor AddMessage "$JOB_MESSAGE failed."
 
 # The following lines can be safely deleted, which will disable AppVeyorBot
 # notifications in GitHub pull requests
@@ -55,7 +75,6 @@ on_success:
 notifications:
   - provider: GitHubPullRequest
     template: "AppVeyor [build {{buildVersion}}]({{buildUrl}})
-      {{#jobs}}{{#messages}} for {{message}} by @{{&commitAuthorUsername}}{{/messages}}{{/jobs}}
-      {{#passed}} is now complete. The rendered manuscript from this build is temporarily available for download at:\n\n{{/passed}}
-      {{#failed}} failed.{{/failed}}
-      {{#jobs}}{{#artifacts}}- [`{{fileName}}`]({{permalink}})\n{{/artifacts}}{{/jobs}}"
+      {{#jobs}}{{#messages}}{{{message}}}{{/messages}}{{/jobs}}
+      {{#passed}}The rendered manuscript from this build is temporarily available for download at:\n\n
+      {{#jobs}}{{#artifacts}}- [`{{fileName}}`]({{permalink}})\n{{/artifacts}}{{/jobs}}{{/passed}}"
diff --git a/.travis.yml b/.travis.yml
@@ -6,6 +6,8 @@ services:
 branches:
   only:
     - master
+env:
+  - SPELLCHECK=true
 install:
   - source ci/install.sh
 script:

diff --git a/USAGE.md b/USAGE.md
@@ -276,6 +276,13 @@ Changing the citation style or which interactive HTML plugins are loaded require
 The citation style is determined by the Citation Style Language file specified by `CSL_PATH`.
 It can be changed to use other existing styles as [described here](https://github.com/manubot/rootstock/issues/242#issuecomment-507688339).
 
+## Spellchecking
+
+When the `SPELLCHECK` environment variable is `true`, the pandoc [spellcheck filter](https://github.com/pandoc/lua-filters/tree/master/spellcheck) is run.
+Potential spelling errors will be printed in the continuous integration log along with the files and line numbers in which they appeared.
+Words in `build/assets/custom-dictionary.txt` are ignored during spellchecking.
+Spellchecking is currently only supported for English language manuscripts and with Travis CI and AppVeyor continuous integration services.
+
 ## Manubot feedback
 
 If you experience any issues with the Manubot or would like to contribute to its source code, please visit [`manubot/manubot`](https://github.com/manubot/manubot) or [`manubot/rootstock`](https://github.com/manubot/rootstock).

diff --git a/build/assets/custom-dictionary.txt b/build/assets/custom-dictionary.txt
@@ -0,0 +1,23 @@
+personal_ws-1.1 en 22
+al
+doi
+eq
+et
+github
+isbn
+latex
+manubot
+orcid
+permalink
+pmc
+pmcid
+pmid
+pubmed
+rootstock
+s
+strikethrough
+svg
+svgs
+tbl
+unicode
+wikidata
diff --git a/build/build.sh b/build/build.sh
@@ -82,4 +82,15 @@ if [ "${BUILD_DOCX:-}" = "true" ]; then
     --defaults="$PANDOC_DEFAULTS_DIR/docx.yaml"
 fi
 
+# Spellcheck
+if [ "${SPELLCHECK:-}" = "true" ]; then
+  export ASPELL_CONF="add-extra-dicts $(pwd)/build/assets/custom-dictionary.txt; ignore-case true"
+  # Use "|| true" after grep because otherwise this step of the pipeline will
+  # return exit code 1 if any of the markdown files do not contain a
+  # misspelled word
+  pandoc --lua-filter spellcheck.lua output/manuscript.md | uniq | while read word; do grep -ion "\<$word\>" content/*.md; done | sort -h -t ":" -k 1b,1 -k2,2 > output/spelling-errors.txt || true
+  echo >&2 "Filenames and line numbers with potential spelling errors:"
+  cat output/spelling-errors.txt
+fi
+
 echo >&2 "Build complete"
diff --git a/ci/install.sh b/ci/install.sh
@@ -17,3 +17,10 @@ conda config \
 conda env create --quiet --file build/environment.yml
 conda list --name manubot
 conda activate manubot
+
+# Install Spellcheck filter for Pandoc
+if [ "${SPELLCHECK:-}" = "true" ]; then
+  sudo apt-get update -y
+  sudo apt-get install -y aspell aspell-en
+  wget https://raw.githubusercontent.com/pandoc/lua-filters/1c553017ecc58914c22bf2372902dca4a456929b/spellcheck/spellcheck.lua
+fi