diff --git a/.appveyor.yml b/.appveyor.yml
new file mode 100644
index 00000000..d57e0cb8
--- /dev/null
+++ b/.appveyor.yml
@@ -0,0 +1,83 @@
+# See https://www.appveyor.com/docs/getting-started-with-appveyor-for-linux/
+# Don't build branches with a PR, since their build will be created with the PR itself.
+# Otherwise there would be two builds -- one for the PR and one for the branch.
+# If you're having issues with getting your PR to build, make sure there are no merge conflicts.
+skip_branch_with_pr: true
+
+# Enable 'Do not build on "Push" events' in the AppVeyor project settings
+# to only build commits from pull requests
+branches:
+ only:
+ - master
+
+# Only run AppVeyor on commits that modify at least one of the following files
+# Delete these lines to run AppVeyor on all master branch commits
+only_commits:
+ files:
+ - .appveyor.yml
+ - build/
+ - ci/install.sh
+ - content/
+
+image: ubuntu
+services:
+ - docker
+
+# Set SPELLCHECK to true to enable Pandoc spellchecking
+environment:
+ SPELLCHECK: true
+
+install:
+ # Create the message with the triggering commit before install so it is
+ # available if the build fails
+ - TRIGGERING_COMMIT=${APPVEYOR_PULL_REQUEST_HEAD_COMMIT:-APPVEYOR_REPO_COMMIT}
+ - JOB_MESSAGE=" for commit $TRIGGERING_COMMIT "
+ - source ci/install.sh
+
+test_script:
+ - bash build/build.sh
+ - MANUSCRIPT_FILENAME=manuscript-$APPVEYOR_BUILD_VERSION-${TRIGGERING_COMMIT:0:7}
+ - cp output/manuscript.html $MANUSCRIPT_FILENAME.html
+ - cp output/manuscript.pdf $MANUSCRIPT_FILENAME.pdf
+ - appveyor PushArtifact $MANUSCRIPT_FILENAME.html
+ - appveyor PushArtifact $MANUSCRIPT_FILENAME.pdf
+ - |
+ if [ "${SPELLCHECK:-}" = "true" ]; then
+ SPELLING_ERRORS_FILENAME=spelling-errors-$APPVEYOR_BUILD_VERSION-${TRIGGERING_COMMIT:0:7}.txt
+ cp output/spelling-errors.txt $SPELLING_ERRORS_FILENAME
+ appveyor PushArtifact $SPELLING_ERRORS_FILENAME
+ SPELLING_ERROR_LOCATIONS_FILENAME=spelling-error-locations-$APPVEYOR_BUILD_VERSION-${TRIGGERING_COMMIT:0:7}.txt
+ cp output/spelling-error-locations.txt $SPELLING_ERROR_LOCATIONS_FILENAME
+ appveyor PushArtifact $SPELLING_ERROR_LOCATIONS_FILENAME
+ fi
+
+build: off
+
+cache:
+ - ci/cache
+
+on_success:
+ - echo "Artifacts available from $APPVEYOR_URL/project/$APPVEYOR_ACCOUNT_NAME/$APPVEYOR_PROJECT_SLUG/builds/$APPVEYOR_BUILD_ID/artifacts"
+ - echo "Updated PDF available from $APPVEYOR_URL/api/buildjobs/$APPVEYOR_JOB_ID/artifacts/$MANUSCRIPT_FILENAME.pdf"
+ - appveyor AddMessage "$JOB_MESSAGE is now complete."
+ - |
+ if [ "${SPELLCHECK:-}" = "true" ]; then
+ SPELLING_ERROR_COUNT=($(wc -l $SPELLING_ERROR_LOCATIONS_FILENAME))
+ appveyor AddMessage " Found $SPELLING_ERROR_COUNT potential spelling error(s). Preview:
$(head -n 100 $SPELLING_ERROR_LOCATIONS_FILENAME)"
+ appveyor AddMessage "... "
+ fi
+
+on_failure:
+ - appveyor AddMessage "$JOB_MESSAGE failed."
+
+# The following lines can be safely deleted, which will disable AppVeyorBot
+# notifications in GitHub pull requests
+# Notifications use Mustache templates http://mustache.github.io/mustache.5.html
+# See https://www.appveyor.com/docs/notifications/#customizing-message-template
+# for available variables
+notifications:
+ - provider: GitHubPullRequest
+ template: "AppVeyor [build {{buildVersion}}]({{buildUrl}})
+ {{#jobs}}{{#messages}}{{{message}}}{{/messages}}{{/jobs}}
+ {{#passed}}The rendered manuscript from this build is temporarily available for download at:\n\n
+ {{#jobs}}{{#artifacts}}- [`{{fileName}}`]({{permalink}})\n{{/artifacts}}{{/jobs}}{{/passed}}"
diff --git a/.github/workflows/manubot.yaml b/.github/workflows/manubot.yaml
new file mode 100644
index 00000000..42509c5c
--- /dev/null
+++ b/.github/workflows/manubot.yaml
@@ -0,0 +1,64 @@
+name: Manubot
+on:
+ push:
+ branches:
+ - master
+ pull_request:
+ branches:
+ - master
+jobs:
+ manubot:
+ name: Manubot
+ runs-on: ubuntu-latest
+ env:
+ GITHUB_PULL_REQUEST_SHA: ${{ github.event.pull_request.head.sha }}
+ SPELLCHECK: true
+ steps:
+ - name: Set Environment Variables
+ run: |
+ TRIGGERING_SHA=${GITHUB_PULL_REQUEST_SHA:-$GITHUB_SHA}
+ TRIGGERING_SHA_7=${TRIGGERING_SHA::7}
+ echo "::set-env name=TRIGGERING_SHA_7::$TRIGGERING_SHA_7"
+ echo "TRIGGERING_SHA: $TRIGGERING_SHA"
+ - name: Checkout Repository
+ uses: actions/checkout@v2
+ with:
+ # fetch entire commit history to support get_rootstock_commit
+ fetch-depth: 0
+ - name: Cache
+ uses: actions/cache@v1
+ with:
+ path: ci/cache
+ key: ci-cache-${{ github.ref }}
+ restore-keys: |
+ ci-cache-refs/heads/master
+ - name: Install Environment
+ uses: goanpeca/setup-miniconda@v1
+ with:
+ activate-environment: manubot
+ environment-file: build/environment.yml
+ auto-activate-base: false
+ miniconda-version: 'latest'
+ - name: Install Spellcheck
+ shell: bash --login {0}
+ run: |
+ if [ "${SPELLCHECK:-}" = "true" ]; then
+ bash ci/install-spellcheck.sh
+ fi
+ - name: Build Manuscript
+ shell: bash --login {0}
+ run: bash build/build.sh
+ - name: Upload Artifacts
+ uses: actions/upload-artifact@v1
+ with:
+ name: manuscript-${{ github.run_id }}-${{ env.TRIGGERING_SHA_7 }}
+ path: output
+ - name: Deploy Manuscript
+ if: github.ref == 'refs/heads/master' && github.event_name == 'push' && !github.event.repository.fork
+ env:
+ MANUBOT_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ MANUBOT_SSH_PRIVATE_KEY: ${{ secrets.MANUBOT_SSH_PRIVATE_KEY }}
+ CI_BUILD_WEB_URL: https://github.com/${{ github.repository }}/commit/${{ github.sha }}/checks
+ CI_JOB_WEB_URL: https://github.com/${{ github.repository }}/runs/${{ github.run_id }}
+ shell: bash --login {0}
+ run: bash ci/deploy.sh
diff --git a/.gitignore b/.gitignore
index ac39263a..e48c49e4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,9 @@ webpage/v
# Manubot cache directory
ci/cache
+# Pandoc filters downloaded during continuous integration setup
+build/pandoc/filters/spellcheck.lua
+
# Python
__pycache__/
*.pyc
diff --git a/.travis.yml b/.travis.yml
index ab7f1609..a8421383 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,19 +6,10 @@ services:
branches:
only:
- master
-before_install:
- - wget https://repo.continuum.io/miniconda/Miniconda3-4.6.14-Linux-x86_64.sh
- --output-document miniconda.sh
- - bash miniconda.sh -b -p $HOME/miniconda
- - source $HOME/miniconda/etc/profile.d/conda.sh
- - hash -r
- - conda config
- --set always_yes yes
- --set changeps1 no
+env:
+ - SPELLCHECK=true
install:
- - conda env create --quiet --file build/environment.yml
- - conda list --name manubot
- - conda activate manubot
+ - source ci/install.sh
script:
- bash build/build.sh
cache:
@@ -26,8 +17,8 @@ cache:
- ci/cache
deploy:
provider: script
- script: bash -o xtrace ci/deploy.sh
+ script: bash ci/deploy.sh
skip_cleanup: true
on:
branch: master
- condition: $TRAVIS_EVENT_TYPE = "push"
+ condition: $TRAVIS_EVENT_TYPE = "push" && (-v MANUBOT_SSH_PRIVATE_KEY || "${!encrypted_*}" != "")
diff --git a/README.md b/README.md
index 0c245743..cccaffa7 100644
--- a/README.md
+++ b/README.md
@@ -68,7 +68,6 @@ The directories are as follows:
+ [`webpage`](webpage) is a directory meant to be rendered as a static webpage for viewing the HTML manuscript.
+ [`build`](build) contains commands and tools for building the manuscript.
+ [`ci`](ci) contains files necessary for deployment via continuous integration.
- For the CI configuration, see [`.travis.yml`](.travis.yml).
### Local execution
@@ -88,7 +87,7 @@ bash build/build.sh
# This is required to view local images in the HTML output.
# Configure the webpage directory
-python build/webpage.py
+manubot webpage
# You can now open the manuscript webpage/index.html in a web browser.
# Alternatively, open a local webserver at http://localhost:8000/ with the
@@ -98,7 +97,7 @@ python -m http.server
```
Sometimes it's helpful to monitor the content directory and automatically rebuild the manuscript when a change is detected.
-The following command, while running, will trigger both the `build.sh` and `webpage.py` scripts upon content changes:
+The following command, while running, will trigger both the `build.sh` script and `manubot webpage` command upon content changes:
```sh
bash build/autobuild.sh
diff --git a/USAGE.md b/USAGE.md
index 900a94d7..06b7eafb 100644
--- a/USAGE.md
+++ b/USAGE.md
@@ -12,6 +12,9 @@ Markdown files are identified by their `.md` extension and ordered according to
For basic formatting, check out the [CommonMark Help](https://commonmark.org/help/) page for an introduction to the formatting options provided by standard markdown.
In addition, Manubot supports an extended version of markdown, tailored for scholarly writing, which includes [Pandoc's Markdown](https://pandoc.org/MANUAL.html#pandocs-markdown) and the extensions discussed below.
+The `content/02.delete-me.md` file in the Rootstock repository shows many of the elements and formatting options supported by Manubot.
+See the [raw markdown](https://gitlab.com/manubot/rootstock/blob/master/content/02.delete-me.md#L) in this file and compare it to the [rendered manuscript](https://manubot.github.io/rootstock/).
+
Within a paragraph in markdown, single newlines are interpreted as whitespace (same as a space).
A paragraph's source does not need to contain newlines.
However, "one paragraph per line" makes the git diff less precise, leading to less granular review commenting, and makes conflicts more likely.
@@ -73,9 +76,15 @@ We recommend always specifying the width of SVG images (even if just `width="100
### Citations
-Manubot supports Pandoc [citations](https://pandoc.org/MANUAL.html#citations) via `pandoc-citeproc`.
-However, Manubot performs automated citation processing and metadata retrieval on in-text citations.
-Therefore, citations must be of the following form: `@source:identifier`, where `source` is one of the options described below.
+Manubot supports [Pandoc citations](https://pandoc.org/MANUAL.html#citations), but with added support for citing persistent identifiers directly.
+Citations are processed in 3 stages:
+
+1. Pandoc parses the input Markdown to locate citation keys.
+2. The [`pandoc-manubot-cite` filter](https://github.com/manubot/manubot#pandoc-filter) automatically retrieves the bibliographic metadata for citation keys.
+3. The [`pandoc-citeproc` filter](https://github.com/jgm/pandoc-citeproc/blob/master/man/pandoc-citeproc.1.md) renders in-text citations and generates styled references.
+
+When citing persistent identifiers, citation keys should be formatted like `@prefix:accession`,
+where `prefix` is one of the options described below.
When choosing which source to use for a citation, we recommend the following order:
1. DOI (Digital Object Identifier), cite like `@doi:10.15363/thinklab.4`.
@@ -83,62 +92,108 @@ When choosing which source to use for a citation, we recommend the following ord
shortDOIs begin with `10/` rather than `10.` and can also be cited.
For example, Manubot will expand `@doi:10/993` to the DOI above.
We suggest using shortDOIs to cite DOIs containing forbidden characters, such as `(` or `)`.
-2. PubMed Central ID, cite like `@pmcid:PMC4497619`.
-3. PubMed ID, cite like `@pmid:26158728`.
+2. PubMed Central ID, cite like `@pmc:PMC4497619`.
+3. PubMed ID, cite like `@pubmed:26158728`.
4. _arXiv_ ID, cite like `@arxiv:1508.06576v2`.
5. ISBN (International Standard Book Number), cite like `@isbn:9781339919881`.
-6. URL / webpage, cite like `@url:https://nyti.ms/1QUgAt1`.
+6. URL / webpage, cite like `@https://nyti.ms/1QUgAt1`.
+ URL citations can be helpful if the above methods return incorrect metadata.
+ For example, `@doi:10.1038/ng.3834` [incorrectly handles](https://github.com/manubot/manubot/issues/158) the consortium name resulting in a blank author, while `@https://doi.org/10.1038/ng.3834` succeeds.
+ Similarly, `@https://doi.org/10.1101/142760` is a [workaround](https://github.com/manubot/manubot/issues/16) to set the journal name of bioRxiv preprints to _bioRxiv_.
7. Wikidata Items, cite like `@wikidata:Q50051684`.
- Note that anyone can edit or add records on [Wikidata](https://www.wikidata.org), so users are encouraged to contribute metadata for hard-to-cite works to Wikidata as an alternative to using a `raw` citation.
-8. For references that do not have any of the persistent identifiers above, use a raw citation like `@raw:old-manuscript`.
- Metadata for raw citations must be provided manually.
+ Note that anyone can edit or add records on [Wikidata](https://www.wikidata.org), so users are encouraged to contribute metadata for hard-to-cite works to Wikidata.
+8. Any other compact identifier supported by .
+ Manubot uses the Identifiers.org Resolution Service to support [hundreds](https://github.com/manubot/manubot/blob/7055bcc6524fdf1ef97d838cf0158973e2061595/manubot/cite/handlers.py#L122-L831 "Actual prefix support is determined by this manubot source code.") of [prefixes](https://registry.identifiers.org/registry "Identifiers.org prefix search").
+ For example, citing `@clinicaltrials:NCT04280705` will produce the same bibliographic metadata as `@https://identifiers.org/clinicaltrials:NCT04280705` or `@https://clinicaltrials.gov/ct2/show/NCT04280705`.
+9. For references that do not have any of the above persistent identifiers, the citation key does not need to include a prefix.
+ Citing `@old-manuscript` will work, but only if reference metadata is [provided manually](#reference-metadata).
Cite multiple items at once like:
```md
-Here is a sentence with several citations [@doi:10.15363/thinklab.4; @pmid:26158728; @arxiv:1508.06576; @isbn:9780394603988].
+Here is a sentence with several citations [@doi:10.15363/thinklab.4; @pubmed:26158728; @arxiv:1508.06576; @isbn:9780394603988].
```
Note that multiple citations must be semicolon separated.
Be careful not to cite the same study using identifiers from multiple sources.
-For example, the following citations all refer to the same study, but will be treated as separate references: `[@doi:10.7717/peerj.705; @pmcid:PMC4304851; @pmid:25648772]`.
+For example, the following citations all refer to the same study, but will be treated as separate references: `[@doi:10.7717/peerj.705; @pmc:PMC4304851; @pubmed:25648772]`.
+
+Citation keys must adhere to the syntax described in the [Pandoc manual](https://pandoc.org/MANUAL.html#citations):
+
+> The citation key must begin with a letter, digit, or `_`, and may contain alphanumerics, `_`, and internal punctuation characters (`:.#$%&-+?<>~/`).
+
+To evaluate whether a citation key fully matches this syntax, try [this online regex](https://regex101.com/r/mXZyY2/latest).
+If the citation key is not valid, use the [citation aliases](#citation-aliases) workaround below.
+This is required for citation keys that contain forbidden characters such as `;` or `=` or end with a non-alphanumeric character such as `/`.
+
+
+Prior to Rootstock commit [`6636b91`](https://github.com/manubot/rootstock/commit/6636b912c6b41593acd2041d34cd4158c1b317fb) on 2020-01-14, Manubot processed citations separately from Pandoc.
+Switching to a Pandoc filter improved reliability on complex documents, but restricted the syntax of citation keys slightly.
+Therefore, users upgrading Rootstock may find some citations become invalid.
+By default, `pandoc-manubot-cite` does not fail upon invalid citations, although this can be changed by adding the following to `metadata.yaml`:
+
+```yaml
+pandoc:
+ manubot-fail-on-errors: True
+```
-#### Citation tags
+#### Citation aliases
-The system also supports citation tags, which are recommended for the following applications:
+The system also supports citation aliases, which map from one citation key (the "alias" or "tag") to another.
+Aliases are recommended for the following applications:
-1. A citation's identifier contains forbidden characters, such as `;` or `=`, or ends with a non-alphanumeric character other than `/`.
- In these instances, you must use a tag.
+1. A citation key contains forbidden characters.
2. A single reference is cited many times.
- Therefore, it might make sense to define a tag, so if the citation updates (e.g. a newer version becomes available), only a single change is required.
+ Therefore, it might make sense to define an alias, so if the citation updates (e.g. a newer version becomes available), only a single change is required.
+
+Aliases can be defined using Markdown's [link reference syntax](https://spec.commonmark.org/0.29/#link-reference-definitions) as follows:
+
+```markdown
+Citing a URL containing a `?` character [@my-url].
+Citing a DOI containing parentheses [@my-doi].
-Tags should be defined in [`content/citation-tags.tsv`](content/citation-tags.tsv).
-If `citation-tags.tsv` defines the tag `study-x`, then this study can be cited like `@tag:study-x`.
+[@my-url]: https://openreview.net/forum?id=HkwoSDPgg
+[@my-doi]: doi:10.1016/S0022-2836(05)80360-2
+```
+
+This syntax is also used by [`pandoc-url2cite`](https://github.com/phiresky/pandoc-url2cite).
+Make sure to place these link reference definitions in their own paragraphs.
+These paragraphs can be in any of the content Markdown files.
+
+Another method for defining aliases is to define `pandoc.citekey-aliases` in `metadata.yaml`:
+
+```yaml
+pandoc:
+ citekey-aliases:
+ my-url: https://openreview.net/forum?id=HkwoSDPgg
+ my-doi: doi:10.1016/S0022-2836(05)80360-2
+```
## Reference metadata
Manubot stores the bibliographic details for references (the set of all cited works) as CSL JSON ([Citation Style Language Items](http://citeproc-js.readthedocs.io/en/latest/csl-json/markup.html#csl-json-items)).
-For all citation sources besides `raw`, Manubot automatically generates CSL JSON.
+Manubot automatically generates CSL JSON for most persistent identifiers (as described in [Citations](#citations) above).
In some cases, automatic metadata retrieval fails or provides incorrect or incomplete information.
-Errors are most common for `url` references.
+Errors are most common for references generated from scraping HTML metadata from websites.
+This occurs most frequently for `https`/`http`/`url` citations as well as identifiers.org prefixes without explicit support listed above.
Therefore, Manubot supports user-provided metadata, which we refer to as "manual references".
When a manual reference is provided, Manubot uses the supplied metadata and does not attempt to generate it.
Manubot searches the `content` directory for files that match the glob pattern `manual-references*.*` and expects that these files contain manual references.
[`content/manual-references.json`](content/manual-references.json) is the default file to specify custom CSL JSON metadata.
Manual references are matched to citations using their "id" field.
-For example, to manually specify the metadata for the citation `@url:https://github.com/manubot/rootstock`, add a CSL JSON Item to `manual-references.json` that contains the following excerpt:
+For example, to manually specify the metadata for the citation `@https://github.com/manubot/rootstock`, add a CSL JSON Item to `manual-references.json` that contains the following excerpt:
```json
-"id": "url:https://github.com/manubot/rootstock",
+"id": "https://github.com/manubot/rootstock",
```
-The metadata for `raw` citations must be provided in a manual reference file (e.g. `manual-references.json`) or an error will occur.
-For example, to cite `@raw:private-message` in a manuscript, a corresponding CSL JSON Item is required, such as:
+The metadata for unhandled citations — any citation key that is a not a supported persistent ID — must be provided in a manual reference file (e.g. `manual-references.json`) or an error will occur.
+For example, to cite `@private-message` in a manuscript, a corresponding CSL JSON Item is required, such as:
```json
{
- "id": "raw:private-message",
+ "id": "private-message",
"type": "personal_communication",
"title": "Personal communication with Doctor X"
}
@@ -149,10 +204,10 @@ For guidance on what CSL JSON should be like for different document types, refer
Manubot offers some support for other bibliographic metadata formats besides CSL JSON, by delegating conversion to the `pandoc-citeproc --bib2json` [utility](https://github.com/jgm/pandoc-citeproc/blob/master/man/pandoc-citeproc.1.md#convert-mode).
Formats are inferred from filename extensions.
-So, for example, to provide metadata for `@url:https://github.com/manubot/rootstock` in BibTeX format, create the file `content/manual-references.bib` and create an item whose definition starts with the excerpt:
+So, for example, to provide metadata for `@https://github.com/manubot/rootstock` in BibTeX format, create the file `content/manual-references.bib` and create an item whose definition starts with the excerpt:
```latex
-@misc{url:https://github.com/manubot/rootstock,
+@misc{https://github.com/manubot/rootstock,
```
Processed reference metadata in CSL JSON format, either generated by Manubot or specified via manual references, is exported to `references.json`.
@@ -176,27 +231,69 @@ The following YAML shows the supported key–value pairs for an author:
```yaml
github: dhimmel # strongly suggested
name: Daniel S. Himmelstein # mandatory
-initials: DSH # strongly suggested
+initials: DSH # optional
orcid: 0000-0002-3012-7446 # mandatory
twitter: dhimmel # optional
email: daniel.himmelstein@gmail.com # suggested
affiliations: # as a list, strongly suggested
- Department of Systems Pharmacology and Translational Therapeutics, University of Pennsylvania
- Department of Biological & Medical Informatics, University of California, San Francisco
-funders: GBMF4552 # optional
+funders:
+ - GBMF4552 # optional list of author's funding
```
Note that `affiliations` should be a list to allow for multiple affiliations per author.
+### Thumbnail
+
+A thumbnail is an image used to visually represent the manuscript,
+such as when a manuscript is shared on social media or added to the [Manubot catalog](https://manubot.org/catalog/).
+Specify a thumbnail in any of the following ways:
+
+1. placing an image named `thumbnail.png` anywhere in the manuscript repository (for example, in the root directory).
+2. setting `thumbnail` in `metadata.yaml` to a path, relative to the repository root, where the image file is located.
+ Example:
+ ```yaml
+ thumbnail: build/assets/thumbnail-1000x1000.png
+ ```
+3. setting `thumbnail` in `metadata.yaml` to an absolute URL where the image is located.
+ Example:
+ ```yaml
+ thumbnail: https://github.com/greenelab/meta-review/raw/master/thumbnail.png
+ ```
+
+Methods 2 and 3 take precedence over method 1.
+View the [guidelines here](https://github.com/manubot/catalog#thumbnail-guidelines) for suggestions on how to create a good thumbnail.
+Key points are that thumbnails should be 1000 × 1000 pixels, PNG formatted, and striking.
+
## Custom formatting
Modifying the manuscript formatting requires modifying the CSS in the file [`build/themes/default.html`](build/themes/default.html).
Common formatting changes, such as [font size](https://github.com/manubot/rootstock/issues/239) and [double spacing](https://github.com/manubot/rootstock/issues/244), can be found by searching the [Rootstock issues](https://github.com/manubot/rootstock/issues).
Open a [new issue](https://github.com/manubot/rootstock/issues/new) if you have a new formatting question.
-Changing the citation style or which interactive HTML plugins are loaded requires editing the build script [`build/build.sh`](build/build.sh).
-The citation style is determined by the Citation Style Language file specified by `CSL_PATH`.
-It can be changed to use other existing styles as [described here](https://github.com/manubot/rootstock/issues/242#issuecomment-507688339).
+Changing the citation style or which interactive HTML plugins are loaded requires editing the options specified by Pandoc defaults files in [`build/pandoc/defaults`](build/pandoc/defaults).
+The citation style is determined by the Citation Style Language file specified in [`common.yaml`](build/pandoc/defaults/common.yaml):
+
+```yaml
+metadata:
+ csl: build/assets/style.csl
+```
+
+The value for `metadata.csl` can be a URL, allowing access to thousands of existing styles hosted by [Zotero](https://www.zotero.org/styles) or the [CSL GitHub](https://github.com/citation-style-language/styles).
+For example, the following options replace the Manubot citation style with the _PeerJ_ style:
+
+```yaml
+metadata:
+ csl: https://github.com/citation-style-language/styles/raw/906cd6d43d0c136190ecfbb12f6af0ca794e3c5b/peerj.csl
+```
+
+## Spellchecking
+
+When the `SPELLCHECK` environment variable is `true`, the pandoc [spellcheck filter](https://github.com/pandoc/lua-filters/tree/master/spellcheck) is run.
+Potential spelling errors will be printed in the continuous integration log along with the files and line numbers in which they appeared.
+Words in `build/assets/custom-dictionary.txt` are ignored during spellchecking.
+Spellchecking is currently only supported for English language manuscripts.
## Manubot feedback
diff --git a/build/README.md b/build/README.md
index 1a886f1d..27bf76d2 100644
--- a/build/README.md
+++ b/build/README.md
@@ -1,32 +1,44 @@
# Building the manuscript
[`build.sh`](build.sh) builds the repository.
-`sh build/build.sh` should be executed from the root directory of the repository.
+`bash build/build.sh` should be executed from the root directory of the repository.
By default, `build.sh` creates HTML and PDF outputs.
However, setting the `BUILD_PDF` environment variable to `false` will suppress PDF output.
For example, run local builds using the command `BUILD_PDF=false bash build/build.sh`.
To build a DOCX file of the manuscript, set the `BUILD_DOCX` environment variable to `true`.
For example, use the command `BUILD_DOCX=true bash build/build.sh`.
-To export DOCX for all Travis builds, set a [Travis environment variable](https://docs.travis-ci.com/user/environment-variables/#Defining-Variables-in-Repository-Settings).
+To export DOCX for all CI builds, set an environment variable (see docs for [GitHub Actions](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-environment-variables) or [Travis CI](https://docs.travis-ci.com/user/environment-variables/#Defining-Variables-in-Repository-Settings)).
Currently, equation numbers via `pandoc-eqnos` are not supported for DOCX output.
-There is varying support for embedding images in DOCX output.
-Please reference [Pull Request #40](https://github.com/manubot/rootstock/pull/40) for possible solutions and continued discussion.
+
+Format conversion is done using [Pandoc](https://pandoc.org/MANUAL.html).
+`build.sh` calls `pandoc` commands using the options specified in [`pandoc/defaults`](pandoc/defaults).
+Each file specifies a set of pandoc `--defaults` options for a given format.
+To change the options, either edit the YAML files directly or add additional `--defaults` files.
## Environment
Note: currently, **Windows is not supported**.
-Install or update the [conda](https://conda.io) environment specified in [`environment.yml`](environment.yml) by running the following commands from this directory:
+Install the [conda](https://conda.io) environment specified in [`environment.yml`](environment.yml) by running the following commands from this directory:
```sh
-# If the manubot environment already exists, remove it first
-conda env remove --name manubot
-
# Install the environment
conda env create --file environment.yml
```
+If the `manubot` environment is already installed, but needs to be updated to reflect changes to `environment.yml`, use one of the following options:
+
+```shell
+# option 1: update the existing environment.
+conda env update --file environment.yml
+
+# option 2: remove and reinstall the manubot environment.
+# Slower than option 1, but guarantees a fresh environment.
+conda env remove --name manubot
+conda env create --file environment.yml
+```
+
Activate with `conda activate manubot` (assumes `conda` version of [at least](https://github.com/conda/conda/blob/9d759d8edeb86569c25f6eb82053f09581013a2a/CHANGELOG.md#440-2017-12-20) 4.4).
The environment should successfully install on both Linux and macOS.
However, it will fail on Windows due to the [`pango`](https://anaconda.org/conda-forge/pango) dependency.
@@ -40,7 +52,8 @@ If Docker is available, `build.sh` uses the [Athena](https://www.athenapdf.com/)
Otherwise, `build.sh` uses [WeasyPrint](https://weasyprint.org/) to build the PDF.
It is common for WeasyPrint to generate many warnings and errors that can be safely ignored.
Examples are shown below:
-```
+
+```text
WARNING: Ignored `pointer-events: none` at 3:16, unknown property.
WARNING: Ignored `font-display:auto` at 1:53114, descriptor not supported.
ERROR: Failed to load font at "https://use.fontawesome.com/releases/v5.7.2/webfonts/fa-brands-400.eot#iefix"
diff --git a/build/assets/custom-dictionary.txt b/build/assets/custom-dictionary.txt
new file mode 100644
index 00000000..0e761ed9
--- /dev/null
+++ b/build/assets/custom-dictionary.txt
@@ -0,0 +1,23 @@
+personal_ws-1.1 en 22
+al
+doi
+eq
+et
+github
+isbn
+latex
+manubot
+orcid
+permalink
+pmc
+pmcid
+pmid
+pubmed
+rootstock
+s
+strikethrough
+svg
+svgs
+tbl
+unicode
+wikidata
diff --git a/build/assets/redirect-template.html b/build/assets/redirect-template.html
deleted file mode 100644
index 9fd83c23..00000000
--- a/build/assets/redirect-template.html
+++ /dev/null
@@ -1,19 +0,0 @@
-
-
-
-
-
-
-
- Page Redirection
-
-
- If you are not redirected automatically, follow this link.
-
-
diff --git a/build/autobuild.sh b/build/autobuild.sh
index e82dbfb6..144775ab 100755
--- a/build/autobuild.sh
+++ b/build/autobuild.sh
@@ -5,5 +5,5 @@
watchmedo shell-command \
--wait \
- --command='bash build/build.sh && python build/webpage.py' \
+ --command='bash build/build.sh && manubot webpage' \
content
diff --git a/build/build.sh b/build/build.sh
index eec2bd87..c3028c0e 100755
--- a/build/build.sh
+++ b/build/build.sh
@@ -17,74 +17,50 @@ manubot process \
--content-directory=content \
--output-directory=output \
--cache-directory=ci/cache \
+ --skip-citations \
--log-level=INFO
-# pandoc settings
-CSL_PATH=build/assets/style.csl
-BIBLIOGRAPHY_PATH=output/references.json
-INPUT_PATH=output/manuscript.md
+# Pandoc's configuration is specified via files of option defaults
+# located in the $PANDOC_DATA_DIR/defaults directory.
+PANDOC_DATA_DIR="${PANDOC_DATA_DIR:-build/pandoc}"
# Make output directory
mkdir -p output
# Create HTML output
-# http://pandoc.org/MANUAL.html
+# https://pandoc.org/MANUAL.html
echo >&2 "Exporting HTML manuscript"
pandoc --verbose \
- --from=markdown \
- --to=html5 \
- --filter=pandoc-fignos \
- --filter=pandoc-eqnos \
- --filter=pandoc-tablenos \
- --bibliography="$BIBLIOGRAPHY_PATH" \
- --csl="$CSL_PATH" \
- --metadata link-citations=true \
- --include-after-body=build/themes/default.html \
- --include-after-body=build/plugins/table-scroll.html \
- --include-after-body=build/plugins/anchors.html \
- --include-after-body=build/plugins/accordion.html \
- --include-after-body=build/plugins/tooltips.html \
- --include-after-body=build/plugins/jump-to-first.html \
- --include-after-body=build/plugins/link-highlight.html \
- --include-after-body=build/plugins/table-of-contents.html \
- --include-after-body=build/plugins/lightbox.html \
- --mathjax \
- --variable math="" \
- --include-after-body=build/plugins/math.html \
- --include-after-body=build/plugins/hypothesis.html \
- --include-after-body=build/plugins/analytics.html \
- --output=output/manuscript.html \
- "$INPUT_PATH"
+ --data-dir="$PANDOC_DATA_DIR" \
+ --defaults=common.yaml \
+ --defaults=html.yaml
# Return null if docker command is missing, otherwise return path to docker
DOCKER_EXISTS="$(command -v docker || true)"
# Create PDF output (unless BUILD_PDF environment variable equals "false")
+# If Docker is not available, use WeasyPrint to create PDF
if [ "${BUILD_PDF:-}" != "false" ] && [ -z "$DOCKER_EXISTS" ]; then
echo >&2 "Exporting PDF manuscript using WeasyPrint"
if [ -L images ]; then rm images; fi # if images is a symlink, remove it
ln -s content/images
pandoc \
- --from=markdown \
- --to=html5 \
- --pdf-engine=weasyprint \
- --pdf-engine-opt=--presentational-hints \
- --filter=pandoc-fignos \
- --filter=pandoc-eqnos \
- --filter=pandoc-tablenos \
- --bibliography="$BIBLIOGRAPHY_PATH" \
- --csl="$CSL_PATH" \
- --metadata link-citations=true \
- --webtex=https://latex.codecogs.com/svg.latex? \
- --include-after-body=build/themes/default.html \
- --output=output/manuscript.pdf \
- "$INPUT_PATH"
+ --data-dir="$PANDOC_DATA_DIR" \
+ --defaults=common.yaml \
+ --defaults=html.yaml \
+ --defaults=pdf-weasyprint.yaml
rm images
fi
-# Create PDF output (unless BUILD_PDF environment variable equals "false")
+# If Docker is available, use athenapdf to create PDF
if [ "${BUILD_PDF:-}" != "false" ] && [ -n "$DOCKER_EXISTS" ]; then
echo >&2 "Exporting PDF manuscript using Docker + Athena"
+ if [ "${CI:-}" = "true" ]; then
+ # Incease --delay for CI builds to ensure the webpage fully renders, even when the CI server is under high load.
+ # Local builds default to a shorter --delay to minimize runtime, assuming proper rendering is less crucial.
+ MANUBOT_ATHENAPDF_DELAY="${MANUBOT_ATHENAPDF_DELAY:-5000}"
+ echo >&2 "Continuous integration build detected. Setting athenapdf --delay=$MANUBOT_ATHENAPDF_DELAY"
+ fi
if [ -d output/images ]; then rm -rf output/images; fi # if images is a directory, remove it
cp -R -L content/images output/
docker run \
@@ -94,7 +70,8 @@ if [ "${BUILD_PDF:-}" != "false" ] && [ -n "$DOCKER_EXISTS" ]; then
--security-opt=seccomp:unconfined \
arachnysdocker/athenapdf:2.16.0 \
athenapdf \
- --delay=2000 \
+ --delay=${MANUBOT_ATHENAPDF_DELAY:-1100} \
+ --pagesize=A4 \
manuscript.html manuscript.pdf
rm -rf output/images
fi
@@ -103,18 +80,40 @@ fi
if [ "${BUILD_DOCX:-}" = "true" ]; then
echo >&2 "Exporting Word Docx manuscript"
pandoc --verbose \
- --from=markdown \
- --to=docx \
- --filter=pandoc-fignos \
- --filter=pandoc-eqnos \
- --filter=pandoc-tablenos \
- --bibliography="$BIBLIOGRAPHY_PATH" \
- --csl="$CSL_PATH" \
- --metadata link-citations=true \
- --reference-doc=build/themes/default.docx \
- --resource-path=.:content \
- --output=output/manuscript.docx \
- "$INPUT_PATH"
+ --data-dir="$PANDOC_DATA_DIR" \
+ --defaults=common.yaml \
+ --defaults=docx.yaml
+fi
+
+# Spellcheck
+if [ "${SPELLCHECK:-}" = "true" ]; then
+ export ASPELL_CONF="add-extra-dicts $(pwd)/build/assets/custom-dictionary.txt; ignore-case true"
+
+ # Identify and store spelling errors
+ pandoc \
+ --data-dir="$PANDOC_DATA_DIR" \
+ --lua-filter spellcheck.lua \
+ output/manuscript.md \
+ | sort -fu > output/spelling-errors.txt
+ echo >&2 "Potential spelling errors:"
+ cat output/spelling-errors.txt
+
+ # Add additional forms of punctuation that Pandoc converts so that the
+ # locations can be detected
+ # Create a new expanded spelling errors file so that the saved artifact
+ # contains only the original misspelled words
+ cp output/spelling-errors.txt output/expanded-spelling-errors.txt
+ grep "’" output/spelling-errors.txt | sed "s/’/'/g" >> output/expanded-spelling-errors.txt || true
+
+ # Find locations of spelling errors
+ # Use "|| true" after grep because otherwise this step of the pipeline will
+ # return exit code 1 if any of the markdown files do not contain a
+ # misspelled word
+ cat output/expanded-spelling-errors.txt | while read word; do grep -ion "\<$word\>" content/*.md; done | sort -h -t ":" -k 1b,1 -k2,2 > output/spelling-error-locations.txt || true
+ echo >&2 "Filenames and line numbers with potential spelling errors:"
+ cat output/spelling-error-locations.txt
+
+ rm output/expanded-spelling-errors.txt
fi
echo >&2 "Build complete"
diff --git a/build/environment.yml b/build/environment.yml
index fa807dba..b156d6ca 100644
--- a/build/environment.yml
+++ b/build/environment.yml
@@ -6,29 +6,28 @@ dependencies:
- cairocffi=0.8.0
- cffi=1.12.3
- ghp-import=0.5.5
- - jinja2=2.10
- - jsonschema=3.0.1
- - pandas=0.24.2
- - pandoc=2.7.2
+ - jinja2=2.11.2
+ - jsonschema=3.2.0
+ - pandoc=2.9.2
- pango=1.40.14
- - pip=19.1
- - psutil=5.6.2
- - python=3.6.7
- - pyyaml=5.1
- - requests=2.22.0
- - watchdog=0.8.3
+ - pip=20.0
+ - psutil=5.7.0
+ - python=3.7.6
+ - pyyaml=5.3
+ - requests=2.23.0
+ - watchdog=0.10.2
+ - yamllint=1.21.0
- pip:
- errorhandler==2.0.1
- - git+https://github.com/manubot/manubot@2a99f4af4a352d2e082988d364e01e2e2809a87b
- - jsonref==0.2
- - opentimestamps-client==0.6.0
- - opentimestamps==0.3.0
- - pandoc-eqnos==1.4.3
- - pandoc-fignos==1.4.2
- - pandoc-tablenos==1.4.2
- - pandoc-xnos==1.2.0
- - pybase62==0.4.0
+ - git+https://github.com/manubot/manubot@31968197d1ccd96a46bf092cdba4b575764bb954
+ - opentimestamps-client==0.7.0
+ - opentimestamps==0.4.1
+ - pandoc-eqnos==2.1.1
+ - pandoc-fignos==2.2.0
+ - pandoc-tablenos==2.1.1
+ - pandoc-xnos==2.2.0
+ - pybase62==0.4.3
- pysha3==1.0.2
- - python-bitcoinlib==0.9.0
- - requests-cache==0.5.0
+ - python-bitcoinlib==0.10.2
+ - requests-cache==0.5.2
- weasyprint==0.42.3
diff --git a/build/pandoc/defaults/common.yaml b/build/pandoc/defaults/common.yaml
new file mode 100644
index 00000000..12c0c343
--- /dev/null
+++ b/build/pandoc/defaults/common.yaml
@@ -0,0 +1,13 @@
+# Pandoc --defaults shared between Manubot output formats.
+from: markdown
+input-file: output/manuscript.md
+filters:
+- pandoc-fignos
+- pandoc-eqnos
+- pandoc-tablenos
+- pandoc-manubot-cite
+- pandoc-citeproc
+wrap: preserve
+metadata:
+ csl: build/assets/style.csl
+ link-citations: true
diff --git a/build/pandoc/defaults/docx.yaml b/build/pandoc/defaults/docx.yaml
new file mode 100644
index 00000000..b8f2f932
--- /dev/null
+++ b/build/pandoc/defaults/docx.yaml
@@ -0,0 +1,8 @@
+# Pandoc --defaults for DOCX output.
+# Load on top of common defaults.
+to: docx
+output-file: output/manuscript.docx
+reference-doc: build/themes/default.docx
+resource-path:
+ - '.'
+ - content
diff --git a/build/pandoc/defaults/html.yaml b/build/pandoc/defaults/html.yaml
new file mode 100644
index 00000000..be2a4239
--- /dev/null
+++ b/build/pandoc/defaults/html.yaml
@@ -0,0 +1,21 @@
+# Pandoc --defaults for HTML output.
+# Load on top of common defaults.
+to: html5
+output-file: output/manuscript.html
+include-after-body:
+- build/themes/default.html
+- build/plugins/anchors.html
+- build/plugins/accordion.html
+- build/plugins/tooltips.html
+- build/plugins/jump-to-first.html
+- build/plugins/link-highlight.html
+- build/plugins/table-of-contents.html
+- build/plugins/lightbox.html
+- build/plugins/attributes.html
+- build/plugins/math.html
+- build/plugins/hypothesis.html
+- build/plugins/analytics.html
+variables:
+ math: ''
+html-math-method:
+ method: mathjax
diff --git a/build/pandoc/defaults/pdf-weasyprint.yaml b/build/pandoc/defaults/pdf-weasyprint.yaml
new file mode 100644
index 00000000..4f7666a8
--- /dev/null
+++ b/build/pandoc/defaults/pdf-weasyprint.yaml
@@ -0,0 +1,9 @@
+# Pandoc --defaults for PDF output via weasyprint.
+# Load on top of HTML defaults.
+output-file: output/manuscript.pdf
+pdf-engine: weasyprint
+pdf-engine-opts:
+- '--presentational-hints'
+html-math-method:
+ method: webtex
+ url: 'https://latex.codecogs.com/svg.latex?'
diff --git a/build/plugins/accordion.html b/build/plugins/accordion.html
index ebd6ec8d..eaddd546 100644
--- a/build/plugins/accordion.html
+++ b/build/plugins/accordion.html
@@ -99,17 +99,15 @@
function getHashTarget(link) {
const hash = link ? link.hash : window.location.hash;
const id = hash.slice(1);
- let target = document.querySelector(
- '[id="' + id + '"], [name="' + id + '"]'
- );
+ let target = document.querySelector('[id="' + id + '"]');
if (!target)
return;
// if figure or table, modify target to get expected element
- if (hash.indexOf('#fig:') === 0)
- target = target.parentNode;
- if (hash.indexOf('#tbl:') === 0)
- target = target.nextElementSibling;
+ if (id.indexOf('fig:') === 0)
+ target = target.querySelector('figure');
+ if (id.indexOf('tbl:') === 0)
+ target = target.querySelector('table');
return target;
}
diff --git a/build/plugins/anchors.html b/build/plugins/anchors.html
index 92015d1b..4f2e6ba3 100644
--- a/build/plugins/anchors.html
+++ b/build/plugins/anchors.html
@@ -23,7 +23,7 @@
const options = {
// which types of elements to add anchors next to, in
// "document.querySelector" format
- typesQuery: 'h1, h2, h3, figure, table',
+ typesQuery: 'h1, h2, h3, [id^="fig:"], [id^="tbl:"], [id^="eq:"]',
// whether plugin is on or not
enabled: 'true'
};
@@ -50,11 +50,13 @@
function onScroll() {
// if url has hash and user has scrolled out of view of hash
// target, remove hash from url
+ const tolerance = 100;
const target = getHashTarget();
if (target) {
if (
- target.getBoundingClientRect().top > window.innerHeight ||
- target.getBoundingClientRect().bottom < 0
+ target.getBoundingClientRect().top >
+ window.innerHeight + tolerance ||
+ target.getBoundingClientRect().bottom < 0 - tolerance
)
history.pushState(null, null, ' ');
}
@@ -62,24 +64,20 @@
// add anchor to element
function addAnchor(element) {
- let withId; // element with unique id
let addTo; // element to add anchor button to
// if figure or table, modify withId and addTo to get expected
// elements
- if (element.tagName.toLowerCase() === 'figure') {
- withId = element.querySelector('img');
+ if (element.id.indexOf('fig:') === 0) {
addTo = element.querySelector('figcaption');
- } else if (element.tagName.toLowerCase() === 'table') {
- withId =
- element.previousElementSibling ||
- element.parentNode.previousElementSibling;
+ } else if (element.id.indexOf('tbl:') === 0) {
addTo = element.querySelector('caption');
+ } else if (element.id.indexOf('eq:') === 0) {
+ addTo = element.querySelector('.eqnos-number');
}
- withId = withId || element;
addTo = addTo || element;
- const id = withId.id || withId.name || null;
+ const id = element.id || null;
// do not add anchor if element doesn't have assigned id.
// id is generated by pandoc and is assumed to be unique and
@@ -101,17 +99,15 @@
function getHashTarget() {
const hash = window.location.hash;
const id = hash.slice(1);
- let target = document.querySelector(
- '[id="' + id + '"], [name="' + id + '"]'
- );
+ let target = document.querySelector('[id="' + id + '"]');
if (!target)
return;
// if figure or table, modify target to get expected element
- if (hash.indexOf('#fig:') === 0)
- target = target.parentNode;
- if (hash.indexOf('#tbl:') === 0)
- target = target.nextElementSibling;
+ if (id.indexOf('fig:') === 0)
+ target = target.querySelector('figure');
+ if (id.indexOf('tbl:') === 0)
+ target = target.querySelector('table');
return target;
}
diff --git a/build/plugins/attributes.html b/build/plugins/attributes.html
new file mode 100644
index 00000000..b968198c
--- /dev/null
+++ b/build/plugins/attributes.html
@@ -0,0 +1,130 @@
+
+
+
diff --git a/build/plugins/jump-to-first.html b/build/plugins/jump-to-first.html
index 90e88ba7..e8a8088f 100644
--- a/build/plugins/jump-to-first.html
+++ b/build/plugins/jump-to-first.html
@@ -102,11 +102,11 @@
// add button next to each figure
function makeFigureButtons() {
- const figures = document.querySelectorAll('img[id^="fig:"]');
+ const figures = document.querySelectorAll('[id^="fig:"]');
for (const figure of figures) {
// get figure id and element to add button to
const id = figure.id;
- const container = figure.nextElementSibling;
+ const container = figure.querySelector('figcaption') || figure;
const first = getFirstOccurrence(id);
// if can't find link to figure, ignore
@@ -130,13 +130,11 @@
// add button next to each figure
function makeTableButtons() {
- const tables = document.querySelectorAll('a[name^="tbl:"]');
+ const tables = document.querySelectorAll('[id^="tbl:"]');
for (const table of tables) {
// get ref id and element to add button to
- const id = table.name;
- const container = table.nextElementSibling.querySelector(
- 'caption'
- );
+ const id = table.id;
+ const container = table.querySelector('caption') || table;
const first = getFirstOccurrence(id);
// if can't find link to table, ignore
diff --git a/build/plugins/link-highlight.html b/build/plugins/link-highlight.html
index a8a3ac5b..4462f509 100644
--- a/build/plugins/link-highlight.html
+++ b/build/plugins/link-highlight.html
@@ -86,18 +86,10 @@
function getHashTarget(link) {
const hash = link ? link.hash : window.location.hash;
const id = hash.slice(1);
- let target = document.querySelector(
- '[id="' + id + '"], [name="' + id + '"]'
- );
+ let target = document.querySelector('[id="' + id + '"]');
if (!target)
return;
- // if figure or table, modify target to get expected element
- if (hash.indexOf('#fig:') === 0)
- target = target.parentNode;
- else if (hash.indexOf('#tbl:') === 0)
- target = target.nextElementSibling.querySelector('caption');
-
return target;
}
diff --git a/build/plugins/math.html b/build/plugins/math.html
index f1a38616..d7bc6ed7 100644
--- a/build/plugins/math.html
+++ b/build/plugins/math.html
@@ -4,7 +4,8 @@
MathJax.Hub.Config({
"CommonHTML": { linebreaks: { automatic: true } },
"HTML-CSS": { linebreaks: { automatic: true } },
- "SVG": { linebreaks: { automatic: true } }
+ "SVG": { linebreaks: { automatic: true } },
+ "fast-preview": { disabled: true }
});
diff --git a/build/plugins/table-scroll.html b/build/plugins/table-scroll.html
deleted file mode 100644
index 2a4473ab..00000000
--- a/build/plugins/table-scroll.html
+++ /dev/null
@@ -1,68 +0,0 @@
-
-
-
diff --git a/build/plugins/tooltips.html b/build/plugins/tooltips.html
index 2afc2cba..f9cbc60e 100644
--- a/build/plugins/tooltips.html
+++ b/build/plugins/tooltips.html
@@ -398,19 +398,15 @@
function getSource(link) {
const hash = link ? link.hash : window.location.hash;
const id = hash.slice(1);
- let target = document.querySelector(
- '[id="' + id + '"], [name="' + id + '"]'
- );
+ let target = document.querySelector('[id="' + id + '"]');
if (!target)
return;
- // if figure or table, modify target to get expected element
- if (hash.indexOf('#ref-') === 0)
+ // if ref or figure, modify target to get expected element
+ if (id.indexOf('ref-') === 0)
target = target.querySelector('p');
- else if (hash.indexOf('#fig:') === 0)
- target = target.parentNode;
- else if (hash.indexOf('#tbl:') === 0)
- return;
+ else if (id.indexOf('fig:') === 0)
+ target = target.querySelector('figure');
return target;
}
diff --git a/build/themes/default.html b/build/themes/default.html
index eb41df24..26675bdf 100644
--- a/build/themes/default.html
+++ b/build/themes/default.html
@@ -102,6 +102,7 @@
/* links */
a {
color: #2196f3;
+ overflow-wrap: break-word;
}
/* normal links (not empty, not button link, not syntax highlighting link) */
@@ -227,7 +228,7 @@
border: solid 1px #bdbdbd;
padding: 10px;
/* squash table if too wide for page by forcing line breaks */
- word-break: break-all;
+ overflow-wrap: break-word;
word-break: break-word;
}
@@ -617,6 +618,43 @@
margin-bottom: 20px;
}
+ /* -------------------------------------------------- */
+ /* tablenos */
+ /* -------------------------------------------------- */
+
+ /* tablenos wrapper */
+ .tablenos {
+ /* show scrollbar on tables if necessary to prevent overflow */
+ width: 100%;
+ margin: 20px 0;
+ }
+
+ .tablenos > table {
+ /* move margins from table to table_wrapper to allow margin collapsing */
+ margin: 0;
+ }
+
+ @media only screen {
+ /* tablenos wrapper */
+ .tablenos {
+ /* show scrollbar on tables if necessary to prevent overflow */
+ overflow-x: auto !important;
+ }
+
+ .tablenos th,
+ .tablenos td {
+ overflow-wrap: unset !important;
+ word-break: unset !important;
+ }
+
+ /* table in wrapper */
+ .tablenos table,
+ .tablenos table * {
+ /* don't break table words */
+ overflow-wrap: normal !important;
+ }
+ }
+
/* -------------------------------------------------- */
/* mathjax */
/* -------------------------------------------------- */
@@ -646,34 +684,8 @@
/* equation */
span[id^="eq:"] > span.math.display > span {
- /* nudge to make room for equation auto-number */
- margin-right: 40px !important;
- }
-
- /* -------------------------------------------------- */
- /* table scroll plugin */
- /* -------------------------------------------------- */
-
- @media only screen {
- /* table wrapper */
- .table_wrapper {
- /* show scrollbar on tables if necessary to prevent overflow */
- overflow: auto;
- width: 100%;
- margin: 20px 0;
- }
-
- /* table within table wrapper */
- .table_wrapper table,
- .table_wrapper table * {
- /* don't break table words */
- word-break: normal !important;
- }
-
- .table_wrapper > table {
- /* move margins from table to table_wrapper to allow margin collapsing */
- margin: 0;
- }
+ /* nudge to make room for equation auto-number and anchor */
+ margin-right: 60px !important;
}
/* -------------------------------------------------- */
@@ -776,7 +788,7 @@
border: solid 1px #bdbdbd;
box-shadow: 0 0 20px rgba(0, 0, 0, 0.05);
background: #ffffff;
- word-break: break-word;
+ overflow-wrap: break-word;
}
/* tooltip copy of paragraphs and figures */
diff --git a/build/webpage.py b/build/webpage.py
deleted file mode 100644
index 9f26b375..00000000
--- a/build/webpage.py
+++ /dev/null
@@ -1,215 +0,0 @@
-import argparse
-import os
-import pathlib
-import shutil
-import subprocess
-
-
-def parse_arguments():
- """
- Read and process command line arguments.
- """
- parser = argparse.ArgumentParser()
- parser.add_argument(
- '--checkout',
- nargs='?', const='gh-pages', default=None,
- help='branch to checkout /v directory contents from. For example, --checkout=upstream/gh-pages. --checkout is equivalent to --checkout=gh-pages. If --checkout is ommitted, no checkout is performed.',
- )
- parser.add_argument(
- '--version',
- default=os.environ.get('TRAVIS_COMMIT', 'local'),
- help="Used to create webpage/v/{version} directory. "
- "Generally a commit hash, tag, or 'local'. "
- "(default: '%(default)s')"
- )
- cache_group = parser.add_mutually_exclusive_group()
- cache_group.add_argument(
- '--no-ots-cache',
- action='store_true',
- help="disable the timestamp cache."
- )
- cache_group.add_argument(
- '--ots-cache',
- default=pathlib.Path('ci/cache/ots'),
- type=pathlib.Path,
- help="location for the timestamp cache (default: %(default)s)."
- )
- args = parser.parse_args()
- return args
-
-
-def configure_directories(args):
- """
- Add directories to args and create them if neccessary.
- Note that versions_directory is the parent of version_directory.
- """
- args_dict = vars(args)
-
- # Directory where Manubot outputs reside
- args_dict['output_directory'] = pathlib.Path('output')
-
- # Set webpage directory
- args_dict['webpage_directory'] = pathlib.Path('webpage')
-
- # Create webpage/v directory (if it doesn't already exist)
- args_dict['versions_directory'] = args.webpage_directory.joinpath('v')
- args.versions_directory.mkdir(exist_ok=True)
-
- # Checkout existing version directories
- checkout_existing_versions(args)
-
- # Create empty webpage/v/version directory
- version_directory = args.versions_directory.joinpath(args.version)
- if version_directory.is_dir():
- print(f'{version_directory} exists: replacing it with an empty directory')
- shutil.rmtree(version_directory)
- version_directory.mkdir()
- args_dict['version_directory'] = version_directory
-
- # Symlink webpage/v/latest to point to webpage/v/commit
- latest_directory = args.versions_directory.joinpath('latest')
- if latest_directory.is_symlink() or latest_directory.is_file():
- latest_directory.unlink()
- elif latest_directory.is_dir():
- shutil.rmtree(latest_directory)
- latest_directory.symlink_to(args.version, target_is_directory=True)
- args_dict['latest_directory'] = latest_directory
-
- # Create freeze directory
- freeze_directory = args.versions_directory.joinpath('freeze')
- freeze_directory.mkdir(exist_ok=True)
- args_dict['freeze_directory'] = freeze_directory
-
- return args
-
-
-def checkout_existing_versions(args):
- """
- Must populate webpage/v from the gh-pages branch to get history
-
- References:
- http://clubmate.fi/git-checkout-file-or-directories-from-another-branch/
- https://stackoverflow.com/a/2668947/4651668
- https://stackoverflow.com/a/16493707/4651668
-
- Command modeled after:
- git --work-tree=webpage checkout upstream/gh-pages -- v
- """
- if not args.checkout:
- return
- command = [
- 'git',
- f'--work-tree={args.webpage_directory}',
- 'checkout',
- args.checkout,
- '--',
- 'v',
- ]
- print('Attempting checkout with the following command:', ' '.join(command), sep='\n')
- process = subprocess.run(command, stderr=subprocess.PIPE)
- if process.returncode == 0:
- # Addresses an odd behavior where git checkout stages v/* files that don't actually exist
- subprocess.run(['git', 'add', 'v'])
- else:
- stderr = process.stderr.decode()
- print(f'Checkout returned a nonzero exit status. See stderr:\n{stderr.rstrip()}')
- if 'pathspec' in stderr:
- print(
- 'Manubot note: if there are no preexisting webpage versions (like for a newly created manuscript), '
- 'the pathspec error above is expected and can be safely ignored.'
- ) # see https://github.com/manubot/rootstock/issues/183
-
-
-def create_version(args):
- """
- Populate the version directory for a new version.
- """
-
- # Copy content/images to webpage/v/commit/images
- shutil.copytree(
- src=pathlib.Path('content/images'),
- dst=args.version_directory.joinpath('images'),
- )
-
- # Copy output files to to webpage/v/version/
- renamer = {
- 'manuscript.html': 'index.html',
- 'manuscript.pdf': 'manuscript.pdf',
- }
- for src, dst in renamer.items():
- src_path = args.output_directory.joinpath(src)
- if not src_path.exists():
- continue
- shutil.copy2(
- src=src_path,
- dst=args.version_directory.joinpath(dst),
- )
-
- # Create v/freeze to redirect to v/commit
- path = pathlib.Path('build/assets/redirect-template.html')
- redirect_html = path.read_text()
- redirect_html = redirect_html.format(url=f'../{args.version}/')
- args.freeze_directory.joinpath('index.html').write_text(redirect_html)
-
-
-def get_versions(args):
- """
- Extract versions from the webpage/v directory, which should each contain
- a manuscript.
- """
- versions = {x.name for x in args.versions_directory.iterdir() if x.is_dir()}
- versions -= {'freeze', 'latest'}
- versions = sorted(versions)
- return versions
-
-
-def ots_upgrade(args):
- """
- Upgrade OpenTimestamps .ots files in versioned commit directory trees.
-
- Upgrades each .ots file with a separate ots upgrade subprocess call due to
- https://github.com/opentimestamps/opentimestamps-client/issues/71
- """
- ots_paths = list()
- for version in get_versions(args):
- ots_paths.extend(args.versions_directory.joinpath(version).glob('**/*.ots'))
- ots_paths.sort()
- for ots_path in ots_paths:
- process_args = ['ots']
- if args.no_ots_cache:
- process_args.append('--no-cache')
- else:
- process_args.extend(['--cache', str(args.ots_cache)])
- process_args.extend([
- 'upgrade',
- str(ots_path),
- ])
- process = subprocess.run(
- process_args,
- stderr=subprocess.PIPE,
- universal_newlines=True,
- )
- if process.returncode != 0:
- print(f"OpenTimestamp upgrade command returned nonzero code ({process.returncode}).")
- if not process.stderr.strip() == 'Success! Timestamp complete':
- print(
- f">>> {' '.join(map(str, process.args))}\n"
- f"{process.stderr}"
- )
- backup_path = ots_path.with_suffix('.ots.bak')
- if backup_path.exists():
- if process.returncode == 0:
- backup_path.unlink()
- else:
- # Restore original timestamp if failure
- backup_path.rename(ots_path)
-
-
-if __name__ == '__main__':
- args = parse_arguments()
- configure_directories(args)
- print(args)
- create_version(args)
- versions = get_versions(args)
- print(versions)
- ots_upgrade(args)
diff --git a/ci/README.md b/ci/README.md
index 1c34f3a7..e8a91b5f 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -5,4 +5,4 @@ Specifically, [`deploy.sh`](deploy.sh) runs on successful `master` branch builds
The contents of `../webpage` are committed to the `gh-pages` branch.
The contents of `../output` are committed to the `output` branch.
-For more information on the CI implementation, see the [Travis CI configuration](../.travis.yml) and the CI setup documentation in `SETUP.md`.
\ No newline at end of file
+For more information on the CI implementation, see the [Travis CI configuration](../.travis.yml).
\ No newline at end of file
diff --git a/ci/deploy.sh b/ci/deploy.sh
index f7ba215c..57ac8efc 100644
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -1,16 +1,23 @@
#!/usr/bin/env bash
-## deploy.sh: run during a Travis CI build to deploy manuscript outputs to the output and gh-pages branches on GitHub.
+## deploy.sh: run during a CI build to deploy manuscript outputs to the output and gh-pages branches on GitHub.
# Set options for extra caution & debugging
set -o errexit \
-o nounset \
-o pipefail
+# set environment variables for either Travis or GitHub Actions
+REPO_SLUG=${TRAVIS_REPO_SLUG:-$GITHUB_REPOSITORY}
+COMMIT=${TRAVIS_COMMIT:-$GITHUB_SHA}
+CI_BUILD_WEB_URL=${CI_BUILD_WEB_URL:-$TRAVIS_BUILD_WEB_URL}
+CI_JOB_WEB_URL=${CI_JOB_WEB_URL:-$TRAVIS_JOB_WEB_URL}
+BRANCH=${TRAVIS_BRANCH:-master}
+
# Add commit hash to the README
-OWNER_NAME="$(dirname "$TRAVIS_REPO_SLUG")"
-REPO_NAME="$(basename "$TRAVIS_REPO_SLUG")"
-export OWNER_NAME REPO_NAME
+OWNER_NAME="$(dirname "$REPO_SLUG")"
+REPO_NAME="$(basename "$REPO_SLUG")"
+export REPO_SLUG COMMIT OWNER_NAME REPO_NAME
envsubst < webpage/README.md > webpage/README-complete.md
mv webpage/README-complete.md webpage/README.md
@@ -18,9 +25,22 @@ mv webpage/README-complete.md webpage/README.md
git config --global push.default simple
git config --global user.email "$(git log --max-count=1 --format='%ae')"
git config --global user.name "$(git log --max-count=1 --format='%an')"
-git checkout "$TRAVIS_BRANCH"
-git remote set-url origin "git@github.com:$TRAVIS_REPO_SLUG.git"
+git checkout "$BRANCH"
+
+# Configure deployment credentials
+MANUBOT_DEPLOY_VIA_SSH=true
+git remote set-url origin "git@github.com:$REPO_SLUG.git"
+if [ -v MANUBOT_SSH_PRIVATE_KEY ] && [ "$MANUBOT_SSH_PRIVATE_KEY" != "" ]; then
+ echo >&2 "[INFO] Detected MANUBOT_SSH_PRIVATE_KEY. Will deploy via SSH."
+elif [ -v MANUBOT_ACCESS_TOKEN ] && [ "$MANUBOT_ACCESS_TOKEN" != "" ]; then
+ echo >&2 "[INFO] Detected MANUBOT_ACCESS_TOKEN. Will deploy via HTTPS."
+ MANUBOT_DEPLOY_VIA_SSH=false
+ git remote set-url origin "https://$MANUBOT_ACCESS_TOKEN@github.com/$REPO_SLUG.git"
+else
+ echo >&2 "[INFO] Missing MANUBOT_SSH_PRIVATE_KEY and MANUBOT_ACCESS_TOKEN. Will deploy via SSH."
+fi
+if [ $MANUBOT_DEPLOY_VIA_SSH = "true" ]; then
# Decrypt and add SSH key
eval "$(ssh-agent -s)"
(
@@ -28,8 +48,8 @@ set +o xtrace # disable xtrace in subshell for private key operations
if [ -v MANUBOT_SSH_PRIVATE_KEY ]; then
base64 --decode <<< "$MANUBOT_SSH_PRIVATE_KEY" | ssh-add -
else
-echo "DeprecationWarning: Loading deploy.key from an encrypted file.
-In the future, using the MANUBOT_SSH_PRIVATE_KEY environment variable may be required."
+echo >&2 "DeprecationWarning: Loading deploy.key from an encrypted file.
+In the future, using the MANUBOT_ACCESS_TOKEN or MANUBOT_SSH_PRIVATE_KEY environment variable may be required."
openssl aes-256-cbc \
-K $encrypted_cdc27f6ebb6e_key \
-iv $encrypted_cdc27f6ebb6e_iv \
@@ -39,40 +59,32 @@ chmod 600 ci/deploy.key
ssh-add ci/deploy.key
fi
)
+fi
# Fetch and create gh-pages and output branches
# Travis does a shallow and single branch git clone
git remote set-branches --add origin gh-pages output
-git fetch origin gh-pages:gh-pages output:output
+git fetch origin gh-pages:gh-pages output:output || \
+ echo >&2 "[INFO] could not fetch gh-pages or output from origin."
-# Configure versioned webpage
-python build/webpage.py \
+# Configure versioned webpage and timestamp
+manubot webpage \
+ --timestamp \
--no-ots-cache \
--checkout=gh-pages \
- --version="$TRAVIS_COMMIT"
-
-# Generate OpenTimestamps
-ots stamp "webpage/v/$TRAVIS_COMMIT/index.html"
-if [ "${BUILD_PDF:-}" != "false" ]; then
- ots stamp "webpage/v/$TRAVIS_COMMIT/manuscript.pdf"
-fi
+ --version="$COMMIT"
# Commit message
MESSAGE="\
$(git log --max-count=1 --format='%s')
-
-This build is based on
-https://github.com/$TRAVIS_REPO_SLUG/commit/$TRAVIS_COMMIT.
-
-This commit was created by the following Travis CI build and job:
-$TRAVIS_BUILD_WEB_URL
-$TRAVIS_JOB_WEB_URL
-
[ci skip]
-The full commit message that triggered this build is copied below:
+This build is based on
+https://github.com/$REPO_SLUG/commit/$COMMIT.
-$TRAVIS_COMMIT_MESSAGE
+This commit was created by the following CI build and job:
+$CI_BUILD_WEB_URL
+$CI_JOB_WEB_URL
"
# Deploy the manubot outputs to output
@@ -84,11 +96,14 @@ ghp-import \
# Deploy the webpage directory to gh-pages
ghp-import \
+ --no-jekyll \
--follow-links \
--push \
--branch=gh-pages \
--message="$MESSAGE" \
webpage
-# Workaround https://github.com/travis-ci/travis-ci/issues/8082
-ssh-agent -k
+if [ $MANUBOT_DEPLOY_VIA_SSH = "true" ]; then
+ # Workaround https://github.com/travis-ci/travis-ci/issues/8082
+ ssh-agent -k
+fi
diff --git a/ci/install-spellcheck.sh b/ci/install-spellcheck.sh
new file mode 100755
index 00000000..e12661ec
--- /dev/null
+++ b/ci/install-spellcheck.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+## install-spellcheck.sh: run during a CI build to install Pandoc spellcheck dependencies.
+
+# Set options for extra caution & debugging
+set -o errexit \
+ -o pipefail
+
+sudo apt-get update --yes
+sudo apt-get install --yes aspell aspell-en
+wget --directory-prefix=build/pandoc/filters \
+ https://github.com/pandoc/lua-filters/raw/13c3fa7e97206413609a48a82575cb43137e037f/spellcheck/spellcheck.lua
diff --git a/ci/install.sh b/ci/install.sh
new file mode 100755
index 00000000..61928d75
--- /dev/null
+++ b/ci/install.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+## install.sh: run during a Travis CI or AppVeyor build to install the conda environment
+## and the optional Pandoc spellcheck dependencies.
+
+# Set options for extra caution & debugging
+set -o errexit \
+ -o pipefail
+
+wget https://repo.continuum.io/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh \
+ --output-document miniconda.sh
+bash miniconda.sh -b -p $HOME/miniconda
+source $HOME/miniconda/etc/profile.d/conda.sh
+hash -r
+conda config \
+ --set always_yes yes \
+ --set changeps1 no
+conda env create --quiet --file build/environment.yml
+conda list --name manubot
+conda activate manubot
+
+# Install Spellcheck filter for Pandoc
+if [ "${SPELLCHECK:-}" = "true" ]; then
+ bash ci/install-spellcheck.sh
+fi
diff --git a/content/00.front-matter.md b/content/00.front-matter.md
index bdcde928..2007da68 100644
--- a/content/00.front-matter.md
+++ b/content/00.front-matter.md
@@ -11,21 +11,23 @@ _A DOI-citable version of this manuscript is available at
This manuscript
-{% if ci_source is defined -%}
-([permalink](https://{{ci_source.repo_owner}}.github.io/{{ci_source.repo_name}}/v/{{ci_source.commit}}/))
+{% if manubot.ci_source is defined and manubot.ci_source.provider == "appveyor" -%}
+([permalink]({{manubot.ci_source.artifact_url}}))
+{% elif manubot.html_url_versioned is defined -%}
+([permalink]({{manubot.html_url_versioned}}))
{% endif -%}
was automatically generated
-{% if ci_source is defined -%}
-from [{{ci_source.repo_slug}}@{{ci_source.commit | truncate(length=7, end='', leeway=0)}}](https://github.com/{{ci_source.repo_slug}}/tree/{{ci_source.commit}})
+{% if manubot.ci_source is defined -%}
+from [{{manubot.ci_source.repo_slug}}@{{manubot.ci_source.commit | truncate(length=7, end='', leeway=0)}}](https://github.com/{{manubot.ci_source.repo_slug}}/tree/{{manubot.ci_source.commit}})
{% endif -%}
-on {{date}}.
+on {{manubot.date}}.
## Authors
Please note the current author order is chronological and does not reflect the final order.
{## Template for listing authors ##}
-{% for author in authors %}
+{% for author in manubot.authors %}
+ **{{author.name}}**
{%- if author.orcid is defined and author.orcid is not none %}
{.inline_icon}
@@ -44,7 +46,7 @@ Please note the current author order is chronological and does not reflect the f
{{author.affiliations | join('; ')}}
{%- endif %}
{%- if author.funders is defined and author.funders|length %}
- · Funded by {{author.funders}}
+ · Funded by {{author.funders | join('; ')}}
{%- endif %}
{% endfor %}
diff --git a/output/README.md b/output/README.md
index 8c0fd1c6..fbdc3a6d 100644
--- a/output/README.md
+++ b/output/README.md
@@ -2,7 +2,7 @@
The `output` branch contains files automatically generated by the manuscript build process.
It consists of the contents of the `output` directory of the `master` branch.
-These files are not tracked in `master`, but instead written to the `output` branch by continuous Travis CI builds.
+These files are not tracked in `master`, but instead written to the `output` branch by continuous integration builds.
## Files
diff --git a/webpage/README.md b/webpage/README.md
index 6153fe59..9e22c56d 100644
--- a/webpage/README.md
+++ b/webpage/README.md
@@ -1,15 +1,14 @@
# Output directory containing the formatted manuscript
-The [`gh-pages`](https://github.com/$TRAVIS_REPO_SLUG/tree/gh-pages) branch hosts the contents of this directory at https://$OWNER_NAME.github.io/$REPO_NAME/.
-The permalink for this webpage version is https://$OWNER_NAME.github.io/$REPO_NAME/v/$TRAVIS_COMMIT/.
-To redirect to the permalink for the latest manuscript version at anytime, use the link https://$OWNER_NAME.github.io/$REPO_NAME/v/freeze/.
+The [`gh-pages`](https://github.com/$REPO_SLUG/tree/gh-pages) branch hosts the contents of this directory at .
+The permalink for this webpage version is .
+To redirect to the permalink for the latest manuscript version at anytime, use the link .
## Files
This directory contains the following files, which are mostly ignored on the `master` branch:
+ [`index.html`](index.html) is an HTML manuscript.
-+ [`github-pandoc.css`](github-pandoc.css) sets the display style for `index.html`.
+ [`manuscript.pdf`](manuscript.pdf) is a PDF manuscript.
The `v` directory contains directories for each manuscript version.
@@ -19,13 +18,13 @@ In general, a version is identified by the commit hash of the source content tha
The `*.ots` files in version directories are OpenTimestamps which can be used to verify manuscript existence at or before a given time.
[OpenTimestamps](https://opentimestamps.org/) uses the Bitcoin blockchain to attest to file hash existence.
-The `deploy.sh` script run during continuous deployment creates the `.ots` files.
+The `deploy.sh` script run during continuous deployment creates the `.ots` files through its `manubot webpage` call.
There is a delay before timestamps get confirmed by a Bitcoin block.
Therefore, `.ots` files are initially incomplete and should be upgraded at a later time, so that they no longer rely on the availability of a calendar server to verify.
-`webpage.py`, which is run during continuous deployment, identifies files matched by `webpage/v/**/*.ots` and attempts to upgrade them.
+The `manubot webpage` call during continuous deployment identifies files matched by `webpage/v/**/*.ots` and attempts to upgrade them.
You can also manually upgrade timestamps, by running the following in the `gh-pages` branch:
-```sh
+```shell
ots upgrade v/*/*.ots
rm v/*/*.ots.bak
git add v/*/*.ots
@@ -36,4 +35,4 @@ Verifying timestamps with the `ots verify` command requires running a local bitc
## Source
The manuscripts in this directory were built from
-[`$TRAVIS_COMMIT`](https://github.com/$TRAVIS_REPO_SLUG/commit/$TRAVIS_COMMIT).
+[`$COMMIT`](https://github.com/$REPO_SLUG/commit/$COMMIT).