diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..94143827 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +Dockerfile diff --git a/.gitignore b/.gitignore index 2535ee2a..589d00dc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,6 @@ pkg/ .bundle Gemfile.lock -vendor/ .project .buildpath +*~ diff --git a/.travis.yml b/.travis.yml index f118d5cc..15942b74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,16 +6,16 @@ rvm: - 2.2.7 - 2.3.4 - 2.4.1 - - jruby-9.1.9.0 -jdk: - - oraclejdk8 notifications: email: false git: depth: 10 before_install: + - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 379CE192D401AB61 + - echo "deb https://dl.bintray.com/nxadm/rakudo-pkg-debs `lsb_release -cs` main" | sudo tee -a /etc/apt/sources.list.d/rakudo-pkg.list - sudo apt-get update -qq - - sudo apt-get install perl + - sudo apt-get install perl rakudo-pkg + - export PATH=$PATH:/.perl6/bin:/opt/rakudo-pkg/bin - curl -L http://cpanmin.us | perl - --sudo App::cpanminus - sudo cpanm --installdeps --notest Pod::Simple - sudo pip install docutils @@ -25,6 +25,3 @@ cache: env: global: - "JRUBY_OPTS=-Xcext.enabled=true" -matrix: - allow_failures: - - rvm: jruby-9.1.9.0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..aa0f09aa --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM ubuntu:trusty + +RUN apt-get update -qq +RUN apt-get install -y apt-transport-https + +RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 379CE192D401AB61 +RUN echo "deb https://dl.bintray.com/nxadm/rakudo-pkg-debs `lsb_release -cs` main" | tee -a /etc/apt/sources.list.d/rakudo-pkg.list +RUN apt-get update -qq + +RUN apt-get install -y \ + perl rakudo-pkg curl git build-essential python python-pip \ + libssl-dev libreadline-dev zlib1g-dev \ + libicu-dev cmake pkg-config + +ENV PATH $PATH:/opt/rakudo-pkg/bin +RUN install-zef-as-user && zef install Pod::To::HTML + +RUN curl -L http://cpanmin.us | perl - App::cpanminus +RUN cpanm --installdeps --notest Pod::Simple + +RUN pip install docutils + +ENV PATH $PATH:/root/.rbenv/bin:/root/.rbenv/shims +RUN curl -fsSL https://github.com/rbenv/rbenv-installer/raw/master/bin/rbenv-installer | bash +RUN rbenv install 2.4.1 +RUN rbenv global 2.4.1 +RUN rbenv rehash + +RUN gem install bundler + +WORKDIR /data/github-markup +COPY github-markup.gemspec . +COPY Gemfile . +COPY Gemfile.lock . +COPY lib/github-markup.rb lib/github-markup.rb +RUN bundle + +ENV LC_ALL en_US.UTF-8 +RUN locale-gen en_US.UTF-8 diff --git a/Gemfile b/Gemfile index cbd97b7b..61920ab8 100644 --- a/Gemfile +++ b/Gemfile @@ -11,5 +11,5 @@ gem "org-ruby", "= 0.9.9" gem "creole", "~>0.3.6" gem "wikicloth", "=0.8.3" gem "twitter-text", "~> 1.14" -gem "asciidoctor", "= 1.5.6.1" +gem "asciidoctor", "~> 2.0.5" gem "rake" diff --git a/HISTORY.md b/HISTORY.md index 97f3e825..c296602b 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,29 @@ +## 4.0.0 - 2021-03-31 + +* Drop support for Python 2 in RST rendering [#1456](https://github.com/github/markup/pull/1456) + +## 3.0.5 - 2020-11-12 + +* Add commonmarker_exts to commonmarker options [#1268](https://github.com/github/markup/pull/1268) +* Check whether filename is set when rendering Asciidoc. [#1290](https://github.com/github/markup/pull/1290) + +## 3.0.4 - 2019-04-03 + +* Expose options in #render_s [#1249](https://github.com/github/markup/pull/1249) +* Upgrade to Asciidoctor 2.0.x [#1264](https://github.com/github/markup/pull/1264) + +## 3.0.3 - 2018-12-17 + +* Temporarily remove support for POD6 [#1248](https://github.com/github/markup/pull/1248) + +## 3.0.2 - 2018-12-12 + +* Add support for POD6 [#1173](https://github.com/github/markup/pull/1173) + +## 3.0.1 - 2018-10-19 + +* Remove linguist-detected RMarkdown files from the Markdown renderer [#1237](https://github.com/github/markup/pull/1237) + ## 3.0.0 - 2018-10-18 * Allow passing options through to CommonMarker [#1236](https://github.com/github/markup/pull/1236) @@ -50,8 +76,8 @@ ### Added -* Re-introduce [#537](https://github.com/github/markup/pull/537) to detect language of markup document - However `github-linguist` is optional and this gem will fallback to extensions for detection. +* Re-introduce [#537](https://github.com/github/markup/pull/537) to detect language of markup document + However `github-linguist` is optional and this gem will fallback to extensions for detection. [Full changelog](https://github.com/github/markup/compare/v1.4.9...v1.5.0) diff --git a/README.md b/README.md index 8d619414..b2bcc809 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This library is the **first step** of a journey that every markup file in a repo 1. `github-markup` selects an _underlying library_ to convert the raw markup to HTML. See the list of [supported markup formats](#markups) below. 1. The HTML is sanitized, aggressively removing things that could harm you and your kin—such as `script` tags, inline-styles, and `class` or `id` attributes. 1. Syntax highlighting is performed on code blocks. See [github/linguist](https://github.com/github/linguist#syntax-highlighting) for more information about syntax highlighting. -1. The HTML is passed through other filters in the [html-pipeline](https://github.com/jch/html-pipeline) that add special sauce, such as [emoji](https://github.com/jch/html-pipeline/blob/master/lib/html/pipeline/emoji_filter.rb), [task lists](https://github.com/github/task_list/blob/master/lib/task_list/filter.rb), [named anchors](https://github.com/jch/html-pipeline/blob/master/lib/html/pipeline/toc_filter.rb), [CDN caching for images](https://github.com/jch/html-pipeline/blob/master/lib/html/pipeline/camo_filter.rb), and [autolinking](https://github.com/jch/html-pipeline/blob/master/lib/html/pipeline/autolink_filter.rb). +1. The HTML is passed through other filters that add special sauce, such as emoji, task lists, named anchors, CDN caching for images, and autolinking. 1. The resulting HTML is rendered on GitHub.com. Please note that **only the first step** is covered by this gem — the rest happens on GitHub.com. In particular, `markup` itself does no sanitization of the resulting HTML, as it expects that to be covered by whatever pipeline is consuming the HTML. @@ -30,7 +30,6 @@ you wish to run the library. You can also run `script/bootstrap` to fetch them a * [.pod](http://search.cpan.org/dist/perl/pod/perlpod.pod) -- `Pod::Simple::XHTML` comes with Perl >= 5.10. Lower versions should install Pod::Simple from CPAN. - Installation ----------- @@ -38,6 +37,14 @@ Installation gem install github-markup ``` +or + +``` +bundle install +``` + +from this directory. + Usage ----- diff --git a/github-markup.gemspec b/github-markup.gemspec index 7c663d09..594d3507 100644 --- a/github-markup.gemspec +++ b/github-markup.gemspec @@ -4,14 +4,17 @@ Gem::Specification.new do |s| s.name = "github-markup" s.version = GitHub::Markup::VERSION s.summary = "The code GitHub uses to render README.markup" - s.description = "This gem is used by GitHub to render any fancy markup such " + - "as Markdown, Textile, Org-Mode, etc. Fork it and add your own!" + s.description = <<~DESC + This gem is used by GitHub to render any fancy markup such as Markdown, + Textile, Org-Mode, etc. Fork it and add your own! + DESC s.authors = ["Chris Wanstrath"] s.email = "chris@ozmm.org" s.homepage = "https://github.com/github/markup" s.license = "MIT" s.files = `git ls-files`.split($\) + s.files += Dir['vendor/**/*'] s.executables = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) } s.test_files = s.files.grep(%r{^(test|spec|features)/}) s.require_paths = %w[lib] @@ -20,8 +23,8 @@ Gem::Specification.new do |s| s.add_development_dependency 'activesupport', '~> 4.0' s.add_development_dependency 'minitest', '~> 5.4', '>= 5.4.3' s.add_development_dependency 'html-pipeline', '~> 1.0' - s.add_development_dependency 'sanitize', '~> 2.1', '>= 2.1.0' + s.add_development_dependency 'sanitize', '>= 4.6.3' s.add_development_dependency 'nokogiri', '~> 1.8.1' s.add_development_dependency 'nokogiri-diff', '~> 0.2.0' - s.add_development_dependency "github-linguist", "~> 6.0" + s.add_development_dependency "github-linguist", ">= 7.1.3" end diff --git a/lib/github-markup.rb b/lib/github-markup.rb index cb58c742..df6aa41d 100644 --- a/lib/github-markup.rb +++ b/lib/github-markup.rb @@ -1,6 +1,6 @@ module GitHub module Markup - VERSION = '3.0.0' + VERSION = '4.0.0' Version = VERSION end end diff --git a/lib/github/commands/pod62html b/lib/github/commands/pod62html new file mode 100755 index 00000000..b2d27764 --- /dev/null +++ b/lib/github/commands/pod62html @@ -0,0 +1,5 @@ +#!/usr/bin/env perl6 + +use v6; + +slurp.put; diff --git a/lib/github/markup.rb b/lib/github/markup.rb index 5a78d006..1c63c8bd 100644 --- a/lib/github/markup.rb +++ b/lib/github/markup.rb @@ -19,6 +19,7 @@ module Markups MARKUP_RDOC = :rdoc MARKUP_RST = :rst MARKUP_TEXTILE = :textile + MARKUP_POD6 = :pod6 end module Markup @@ -46,11 +47,11 @@ def render(filename, content, symlink: false, options: {}) end end - def render_s(symbol, content) + def render_s(symbol, content, options: {}) raise ArgumentError, 'Can not render a nil.' if content.nil? if markups.key?(symbol) - markups[symbol].render(nil, content) + markups[symbol].render(nil, content, options: options) else content end diff --git a/lib/github/markup/implementation.rb b/lib/github/markup/implementation.rb index f2854e22..458ab0d8 100644 --- a/lib/github/markup/implementation.rb +++ b/lib/github/markup/implementation.rb @@ -8,7 +8,11 @@ def initialize(regexp, languages) @regexp = regexp if defined?(::Linguist) - @languages = languages.map {|l| Linguist::Language[l]} + @languages = languages.map do |l| + lang = Linguist::Language[l] + raise "no match for language #{l.inspect}" if lang.nil? + lang + end end end diff --git a/lib/github/markup/markdown.rb b/lib/github/markup/markdown.rb index aa012d48..7952bd3b 100644 --- a/lib/github/markup/markdown.rb +++ b/lib/github/markup/markdown.rb @@ -6,7 +6,8 @@ class Markdown < Implementation MARKDOWN_GEMS = { "commonmarker" => proc { |content, options: {}| commonmarker_opts = [:GITHUB_PRE_LANG].concat(options.fetch(:commonmarker_opts, [])) - CommonMarker.render_html(content, commonmarker_opts, [:tagfilter, :autolink, :table, :strikethrough]) + commonmarker_exts = options.fetch(:commonmarker_exts, [:tagfilter, :autolink, :table, :strikethrough]) + CommonMarker.render_html(content, commonmarker_opts, commonmarker_exts) }, "github/markdown" => proc { |content, options: {}| GitHub::Markdown.render(content) @@ -31,7 +32,7 @@ class Markdown < Implementation def initialize super( /md|mkdn?|mdwn|mdown|markdown|litcoffee/i, - ["Markdown", "RMarkdown", "Literate CoffeeScript"]) + ["Markdown", "Literate CoffeeScript"]) end def load diff --git a/lib/github/markups.rb b/lib/github/markups.rb index 440e0dbf..2c30c99d 100644 --- a/lib/github/markups.rb +++ b/lib/github/markups.rb @@ -32,23 +32,28 @@ 'showtitle' => '@', 'idprefix' => '', 'idseparator' => '-', - 'docname' => File.basename(filename, (extname = File.extname(filename))), - 'docfilesuffix' => extname, - 'outfilesuffix' => extname, + 'sectanchors' => nil, 'env' => 'github', 'env-github' => '', 'source-highlighter' => 'html-pipeline' } + if filename + attributes['docname'] = File.basename(filename, (extname = File.extname(filename))) + attributes['docfilesuffix'] = attributes['outfilesuffix'] = extname + else + attributes['outfilesuffix'] = '.adoc' + end Asciidoctor::Compliance.unique_id_start_index = 1 Asciidoctor.convert(content, :safe => :secure, :attributes => attributes) end command( ::GitHub::Markups::MARKUP_RST, - "python2 -S #{Shellwords.escape(File.dirname(__FILE__))}/commands/rest2html", + "python3 #{Shellwords.escape(File.dirname(__FILE__))}/commands/rest2html", /re?st(\.txt)?/, ["reStructuredText"], "restructuredtext" ) +command(::GitHub::Markups::MARKUP_POD6, :pod62html, /pod6/, ["Pod 6"], "pod6") command(::GitHub::Markups::MARKUP_POD, :pod2html, /pod/, ["Pod"], "pod") diff --git a/script/bootstrap b/script/bootstrap index 8092d517..f89f3181 100755 --- a/script/bootstrap +++ b/script/bootstrap @@ -5,4 +5,4 @@ set -e cd $(dirname "$0")/.. bundle install -easy_install docutils +pip3 install docutils diff --git a/test/markup_test.rb b/test/markup_test.rb index eedaccdf..24e78000 100644 --- a/test/markup_test.rb +++ b/test/markup_test.rb @@ -52,7 +52,6 @@ def call define_method "test_#{markup}" do skip "Skipping MediaWiki test because wikicloth is currently not compatible with JRuby." if markup == "mediawiki" && RUBY_PLATFORM == "java" - source = File.read(readme) expected_file = "#{readme}.html" expected = File.read(expected_file).rstrip @@ -67,14 +66,13 @@ def call f.close_write f.read end - assert_html_equal expected, actual, <Title') assert_equal true, GitHub::Markup.can_render?('README.markdown', '=== Title') @@ -91,11 +89,15 @@ def test_each_render_has_a_name assert_equal "wikicloth", GitHub::Markup.renderer('README.wiki', '

Title

').name assert_equal "asciidoctor", GitHub::Markup.renderer('README.adoc', '== Title').name assert_equal "restructuredtext", GitHub::Markup.renderer('README.rst', 'Title').name - assert_equal "pod", GitHub::Markup.renderer('README.pod', '=begin').name + assert_equal "pod", GitHub::Markup.renderer('README.pod', '=head1').name + assert_equal "pod6", GitHub::Markup.renderer('README.pod6', '=begin pod').name end - + def test_rendering_by_symbol - assert_equal '

test

', GitHub::Markup.render_s(GitHub::Markups::MARKUP_MARKDOWN, '`test`').strip + markup = '`test`' + result = /

test<\/code><\/p>/ + assert_match result, GitHub::Markup.render_s(GitHub::Markups::MARKUP_MARKDOWN, markup).strip + assert_match result, GitHub::Markup.render_s(GitHub::Markups::MARKUP_ASCIIDOC, markup).split.join end def test_raises_error_if_command_exits_non_zero @@ -118,5 +120,14 @@ def test_preserve_markup def test_commonmarker_options assert_equal "

hello world

\n", GitHub::Markup.render("test.md", "hello world") assert_equal "

hello world

\n", GitHub::Markup.render("test.md", "hello world", options: {commonmarker_opts: [:UNSAFE]}) + + assert_equal "

hello world

\n", GitHub::Markup.render_s(GitHub::Markups::MARKUP_MARKDOWN, "hello world") + assert_equal "

hello world

\n", GitHub::Markup.render_s(GitHub::Markups::MARKUP_MARKDOWN, "hello world", options: {commonmarker_opts: [:UNSAFE]}) + + assert_equal "<style>.red{color: red;}</style>\n", GitHub::Markup.render("test.md", "", options: {commonmarker_opts: [:UNSAFE]}) + assert_equal "\n", GitHub::Markup.render("test.md", "", options: {commonmarker_opts: [:UNSAFE], commonmarker_exts: [:autolink, :table, :strikethrough]}) + + assert_equal "<style>.red{color: red;}</style>\n", GitHub::Markup.render_s(GitHub::Markups::MARKUP_MARKDOWN, "", options: {commonmarker_opts: [:UNSAFE]}) + assert_equal "\n", GitHub::Markup.render_s(GitHub::Markups::MARKUP_MARKDOWN, "", options: {commonmarker_opts: [:UNSAFE], commonmarker_exts: [:autolink, :table, :strikethrough]}) end end diff --git a/test/markups/README.asciidoc b/test/markups/README.asciidoc index d3c872f7..5f65dd2b 100644 --- a/test/markups/README.asciidoc +++ b/test/markups/README.asciidoc @@ -1,4 +1,6 @@ = Document Title +// sectanchors will be ignored +:sectanchors: == First Section @@ -7,7 +9,7 @@ Refer to <> or <>. -Navigate from {docname}{outfilesuffix} to <>. +Navigate from {docname}{outfilesuffix} to xref:another-document.asciidoc[another document]. == Another Section diff --git a/test/markups/README.hidetitle.asciidoc b/test/markups/README.hidetitle.asciidoc new file mode 100644 index 00000000..38cf76b0 --- /dev/null +++ b/test/markups/README.hidetitle.asciidoc @@ -0,0 +1,4 @@ += Not Shown +:!showtitle: + +This test verifies the author can disable the document title by adding `:!showtitle:` to the document header. diff --git a/test/markups/README.hidetitle.asciidoc.html b/test/markups/README.hidetitle.asciidoc.html new file mode 100644 index 00000000..ae6b258f --- /dev/null +++ b/test/markups/README.hidetitle.asciidoc.html @@ -0,0 +1,3 @@ +
+

This test verifies the author can disable the document title by adding :!showtitle: to the document header.

+
diff --git a/test/markups/README.rst.html b/test/markups/README.rst.html index 7b8e9fca..031c3c4a 100644 --- a/test/markups/README.rst.html +++ b/test/markups/README.rst.html @@ -16,6 +16,47 @@

Header 2

  • Somé UTF-8°

    • The UTF-8 quote character in this table used to cause python to go boom. Now docutils just silently ignores it.

    + +Things that are Awesome (on a scale of 1-11) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    ThingAwesomeness
    Icecream7
    Honey Badgers10.5
    Nickelback-2
    Iron Man10
    Iron Man 23
    Tabular Data5
    Made up ratings11
     A block of code
    diff --git a/test/markups/README.toc.asciidoc b/test/markups/README.toc.asciidoc new file mode 100644 index 00000000..a8e42bdd --- /dev/null +++ b/test/markups/README.toc.asciidoc @@ -0,0 +1,15 @@ += Document Title +:toc: +:toc-title: Contents + +== Section A + +=== Subsection A-1 + +=== Subsection A-2 + +== Section B + +=== Subsection B-1 + +=== Subsection B-2 diff --git a/test/markups/README.toc.asciidoc.html b/test/markups/README.toc.asciidoc.html new file mode 100644 index 00000000..e6f598cf --- /dev/null +++ b/test/markups/README.toc.asciidoc.html @@ -0,0 +1,46 @@ +

    Document Title

    + +
    +

    Section A

    +
    +
    +

    Subsection A-1

    + +
    +
    +

    Subsection A-2

    + +
    +
    +
    +
    +

    Section B

    +
    +
    +

    Subsection B-1

    + +
    +
    +

    Subsection B-2

    + +
    +
    +