From 70ee21beaf459ffe6b041e87a902525f1d86405c Mon Sep 17 00:00:00 2001 From: Kevin Lee Date: Sun, 5 Jul 2015 01:42:25 +1000 Subject: [PATCH 1/3] Added: Tests for Parser.SpecialChar() and extension options (It can detect bugs like #159 and #161). --- src/test/scala/org/pegdown/ParserSpec.scala | 240 ++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 src/test/scala/org/pegdown/ParserSpec.scala diff --git a/src/test/scala/org/pegdown/ParserSpec.scala b/src/test/scala/org/pegdown/ParserSpec.scala new file mode 100644 index 0000000..e534305 --- /dev/null +++ b/src/test/scala/org/pegdown/ParserSpec.scala @@ -0,0 +1,240 @@ +package org.pegdown + +import org.parboiled.Rule +import org.parboiled.matchers.{AnyOfMatcher, CharMatcher, NothingMatcher} +import org.parboiled.support.Characters +import org.pegdown.plugins.PegDownPlugins +import org.specs2.mutable._ + +import scala.annotation.tailrec +import scala.util.Random + +/** + * @author Lee, Seong Hyun (Kevin) + * @since 2015-07-04 + */ +class ParserSpec extends Specification with ParserHelper { + + for (options <- allOptions; option2 <- options) { + val theOptions = (for (option <- option2) yield option.code).foldLeft(0)(_ | _) + s"ParserSpec(${option2.map(_.name).mkString(" | ")} == $theOptions, 1000L, Parser.DefaultParseRunnerProvider, PegDownPlugins.NONE)" should { + val parser = new Parser(theOptions, 1000L, Parser.DefaultParseRunnerProvider, PegDownPlugins.NONE) + val expected = specialCharsBuilder(theOptions, PegDownPlugins.NONE) + s" specialChar() === $expected" in { + val actual = parser.SpecialChar() + val actualChars = extractSpecialChars(actual) + actualChars must not be (None) + actualChars.get === expected + } + } + } + + s"ParserSpec(Extensions.ALL, 1000L, Parser.DefaultParseRunnerProvider, PegDownPlugins.NONE)" should { + val parser = new Parser(Extensions.ALL, 1000L, Parser.DefaultParseRunnerProvider, PegDownPlugins.NONE) + val expected = specialCharsBuilder(Extensions.ALL, PegDownPlugins.NONE) + s" specialChar() === $expected" in { + val actual = parser.SpecialChar() + val actualChars = extractSpecialChars(actual) + actualChars must not be (None) + actualChars.get === expected + } + } + + s"ParserSpec(${options.map(_.name).mkString(" | ")} == ${options.map(_.code).foldLeft(0)((a, b) => a | b)}, 1000L, Parser.DefaultParseRunnerProvider, PegDownPlugins.NONE)" should { + val allKnownOptions = options.map(_.code).foldLeft(0)((a, b) => a | b) + val parser = new Parser(allKnownOptions, 1000L, Parser.DefaultParseRunnerProvider, PegDownPlugins.NONE) + val expected = specialCharsBuilder(allKnownOptions, PegDownPlugins.NONE) + s" specialChar() === $expected" in { + val actual = parser.SpecialChar() + val actualChars = extractSpecialChars(actual) + actualChars must not be (None) + actualChars.get === expected + } + } + + val plugins = PegDownPlugins.builder withSpecialChars('ƒ', 'Ω', '∫', 'ß', 'å') build + + s"ParserSpec(Extensions.NONE, 1000L, Parser.DefaultParseRunnerProvider, PegDownPlugins(specialChars = ${plugins.getSpecialChars.mkString("[", ",", "]")}))" should { + val parser = new Parser(Extensions.NONE, 1000L, Parser.DefaultParseRunnerProvider, plugins) + val expected = specialCharsBuilder(Extensions.NONE, plugins) + s" specialChar() === $expected" in { + val actual = parser.SpecialChar() + val actualChars = extractSpecialChars(actual) + actualChars must not be (None) + actualChars.get === expected + } + } + + s"ParserSpec(Extensions.ALL, 1000L, Parser.DefaultParseRunnerProvider, PegDownPlugins(specialChars = ${plugins.getSpecialChars.mkString("[", ",", "]")}))" should { + val parser = new Parser(Extensions.ALL, 1000L, Parser.DefaultParseRunnerProvider, plugins) + val expected = specialCharsBuilder(Extensions.ALL, plugins) + s" specialChar() === $expected" in { + val actual = parser.SpecialChar() + val actualChars = extractSpecialChars(actual) + actualChars must not be (None) + actualChars.get === expected + } + } + + val (extensionOptions, smartOptions, htmlOptions) = ( + Set( + Extensions.ABBREVIATIONS, + Extensions.ANCHORLINKS, + Extensions.AUTOLINKS, + Extensions.DEFINITIONS, + Extensions.FENCED_CODE_BLOCKS, + Extensions.HARDWRAPS, + Extensions.NONE, + Extensions.STRIKETHROUGH, + Extensions.TABLES, + Extensions.WIKILINKS), + Set(Extensions.QUOTES, + Extensions.SMARTS, + Extensions.SMARTYPANTS), + Set( + Extensions.SUPPRESS_ALL_HTML, + Extensions.SUPPRESS_HTML_BLOCKS, + Extensions.SUPPRESS_INLINE_HTML) + ) + + "Extensions's options" should { + val expected = 0 + val extensionOptionPars = collectPairs(extensionOptions)(extensionOptions ++ smartOptions ++ htmlOptions) + extensionOptionPars.foreach { case (a, b) => + s"not have any common bit ($a, $b)" in { + val actual = a & b + if ((a & b) != 0) println(s"($a, $b) == ${a & b}") + actual === expected + } + } + } + + "Extensions's smartOptions" should { + val expected = 0 + val smartExtensionOptionPairs = collectPairs(smartOptions)(extensionOptions ++ htmlOptions) + smartExtensionOptionPairs.foreach { case (a, b) => + s"not have any common bit ($a, $b)" in { + val actual = a & b + if ((a & b) != 0) println(s"($a, $b) == ${a & b}") + actual === expected + } + } + } + + "Extensions's htmlOptions" should { + val expected = 0 + val htmlExtensionOptionPairs = collectPairs(htmlOptions)(extensionOptions ++ smartOptions) + htmlExtensionOptionPairs.foreach { case (a, b) => + s"not have any common bit ($a, $b)" in { + val actual = a & b + if ((a & b) != 0) println(s"($a, $b) == ${a & b}") + actual === expected + } + } + } + +} + +trait ParserHelper { + def extractSpecialChars(rule: Rule): Option[Characters] = rule match { + case cm: CharMatcher => + Option(Characters.of(cm.character)) + case _: NothingMatcher => + None + case am: AnyOfMatcher => + Option(am.characters) + } + + def specialCharsBuilder(options: Int, plugins: PegDownPlugins): Characters = { + + def ext(extension: Int): Boolean = (options & extension) > 0 + + val chars = new StringBuilder("*_`&[]<>!#\\") + + import Extensions._ + + if (ext(QUOTES)) chars ++= "'\"" + if (ext(SMARTS)) chars ++= ".-" + if (ext(AUTOLINKS)) chars ++= "(){}" + if (ext(DEFINITIONS)) chars ++= ":" + if (ext(TABLES)) chars ++= "|" + if (ext(DEFINITIONS) | ext(FENCED_CODE_BLOCKS)) chars ++= "~" + + for (ch <- plugins.getSpecialChars) { + if (!chars.contains(ch.toString)) { + chars += ch + } + } + Characters.of(chars.toString) + } + + protected case class Extension(name: String, code: Int) + + /* @formatter:off */ + protected val options = Set( + Extension( "ABBREVIATIONS", Extensions.ABBREVIATIONS), + Extension( "ANCHORLINKS", Extensions.ANCHORLINKS), + Extension( "AUTOLINKS", Extensions.AUTOLINKS), + Extension( "DEFINITIONS", Extensions.DEFINITIONS), + Extension( "FENCED_CODE_BLOCKS", Extensions.FENCED_CODE_BLOCKS), + Extension( "HARDWRAPS", Extensions.HARDWRAPS), + Extension( "NONE", Extensions.NONE), + Extension( "QUOTES", Extensions.QUOTES), + Extension( "SMARTS", Extensions.SMARTS), + Extension( "SMARTYPANTS", Extensions.SMARTYPANTS), + Extension( "STRIKETHROUGH", Extensions.STRIKETHROUGH), + Extension( "SUPPRESS_ALL_HTML", Extensions.SUPPRESS_ALL_HTML), + Extension("SUPPRESS_HTML_BLOCKS", Extensions.SUPPRESS_HTML_BLOCKS), + Extension("SUPPRESS_INLINE_HTML", Extensions.SUPPRESS_INLINE_HTML), + Extension( "TABLES", Extensions.TABLES), + Extension( "WIKILINKS", Extensions.WIKILINKS) + ) + /* @formatter:on */ + + /* This has to be split into smaller sets. Otherwise it takes too long to test due to too many possible option combinations. */ + protected val groupedOptions = Random.shuffle(options).grouped(5).toList + + /** + * helper method to create list of option sets. + * + * {{{ + * // examples, + * + * input: Set("a") + * output: List(Set(a)) + * + * input: Set("a", "b") + * output: List(Set(a), Set(b), Set(a, b)) + * + * input: Set("a", "b", "c") + * output: List(Set(b), Set(c), Set(a), Set(a, b), Set(b, c), Set(a, c), Set(a, b, c)) + * }}} + * + * @param options the given options + * @tparam T the element type + * @return List of option Set containing all possible combinations of the given options. + */ + def formOptions[T](options: Set[T]): List[Set[T]] = { + + @tailrec + def eachNumberOfOptions(options: Set[T], howMany: Int, acc: Set[Set[T]]): Set[Set[T]] = howMany match { + case 0 => + acc + case _ => + @tailrec + def collectOptions(options: Set[T], howMany: Int, acc: Set[Set[T]]): Set[Set[T]] = howMany match { + case 0 => + acc + case _ => + collectOptions(options, howMany - 1, acc.flatMap(found => (options diff found).map(found + _))) + } + eachNumberOfOptions(options, howMany - 1, collectOptions(options, howMany, Set(Set())) ++ acc) + } + eachNumberOfOptions(options, options.size, Set()).toList.sortBy(_.size) + } + + protected def collectPairs[T](base: Set[T])(options: Set[T]): Set[(T, T)] = base.flatMap(option => (options - option).map((option, _))) + + protected val allOptions = for (each <- groupedOptions) yield formOptions(each) + +} From aebc710e6de18b6910137e6489168da6d9f202ab Mon Sep 17 00:00:00 2001 From: Kevin Lee Date: Sun, 5 Jul 2015 01:51:26 +1000 Subject: [PATCH 2/3] Added: Tests for Parser.SpecialChar() and extension options (It can detect bugs like #159 and #161). --- src/test/scala/org/pegdown/ParserSpec.scala | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/test/scala/org/pegdown/ParserSpec.scala b/src/test/scala/org/pegdown/ParserSpec.scala index e534305..dd0f92a 100644 --- a/src/test/scala/org/pegdown/ParserSpec.scala +++ b/src/test/scala/org/pegdown/ParserSpec.scala @@ -101,9 +101,8 @@ class ParserSpec extends Specification with ParserHelper { val expected = 0 val extensionOptionPars = collectPairs(extensionOptions)(extensionOptions ++ smartOptions ++ htmlOptions) extensionOptionPars.foreach { case (a, b) => - s"not have any common bit ($a, $b)" in { + s"not have any common bit: ($a, $b) => ($a & $b) === $expected" in { val actual = a & b - if ((a & b) != 0) println(s"($a, $b) == ${a & b}") actual === expected } } @@ -113,9 +112,8 @@ class ParserSpec extends Specification with ParserHelper { val expected = 0 val smartExtensionOptionPairs = collectPairs(smartOptions)(extensionOptions ++ htmlOptions) smartExtensionOptionPairs.foreach { case (a, b) => - s"not have any common bit ($a, $b)" in { + s"not have any common bit: ($a, $b) => ($a & $b) === $expected" in { val actual = a & b - if ((a & b) != 0) println(s"($a, $b) == ${a & b}") actual === expected } } @@ -125,9 +123,8 @@ class ParserSpec extends Specification with ParserHelper { val expected = 0 val htmlExtensionOptionPairs = collectPairs(htmlOptions)(extensionOptions ++ smartOptions) htmlExtensionOptionPairs.foreach { case (a, b) => - s"not have any common bit ($a, $b)" in { + s"not have any common bit: ($a, $b) => ($a & $b) === $expected" in { val actual = a & b - if ((a & b) != 0) println(s"($a, $b) == ${a & b}") actual === expected } } From f4f24afd5c1296d749e3b80362284a7548f97a08 Mon Sep 17 00:00:00 2001 From: Kevin Lee Date: Tue, 14 Jul 2015 00:19:37 +1000 Subject: [PATCH 3/3] #168 - refactored: formOptions is done using Seq.combinations instead of recursive functions as suggested by @Deraen. --- src/test/scala/org/pegdown/ParserSpec.scala | 48 +++++++-------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/src/test/scala/org/pegdown/ParserSpec.scala b/src/test/scala/org/pegdown/ParserSpec.scala index dd0f92a..f449daa 100644 --- a/src/test/scala/org/pegdown/ParserSpec.scala +++ b/src/test/scala/org/pegdown/ParserSpec.scala @@ -6,7 +6,6 @@ import org.parboiled.support.Characters import org.pegdown.plugins.PegDownPlugins import org.specs2.mutable._ -import scala.annotation.tailrec import scala.util.Random /** @@ -23,7 +22,7 @@ class ParserSpec extends Specification with ParserHelper { s" specialChar() === $expected" in { val actual = parser.SpecialChar() val actualChars = extractSpecialChars(actual) - actualChars must not be (None) + actualChars must not be None actualChars.get === expected } } @@ -35,7 +34,7 @@ class ParserSpec extends Specification with ParserHelper { s" specialChar() === $expected" in { val actual = parser.SpecialChar() val actualChars = extractSpecialChars(actual) - actualChars must not be (None) + actualChars must not be None actualChars.get === expected } } @@ -47,7 +46,7 @@ class ParserSpec extends Specification with ParserHelper { s" specialChar() === $expected" in { val actual = parser.SpecialChar() val actualChars = extractSpecialChars(actual) - actualChars must not be (None) + actualChars must not be None actualChars.get === expected } } @@ -60,7 +59,7 @@ class ParserSpec extends Specification with ParserHelper { s" specialChar() === $expected" in { val actual = parser.SpecialChar() val actualChars = extractSpecialChars(actual) - actualChars must not be (None) + actualChars must not be None actualChars.get === expected } } @@ -71,7 +70,7 @@ class ParserSpec extends Specification with ParserHelper { s" specialChar() === $expected" in { val actual = parser.SpecialChar() val actualChars = extractSpecialChars(actual) - actualChars must not be (None) + actualChars must not be None actualChars.get === expected } } @@ -185,50 +184,33 @@ trait ParserHelper { Extension("SUPPRESS_INLINE_HTML", Extensions.SUPPRESS_INLINE_HTML), Extension( "TABLES", Extensions.TABLES), Extension( "WIKILINKS", Extensions.WIKILINKS) - ) + ).toVector /* @formatter:on */ /* This has to be split into smaller sets. Otherwise it takes too long to test due to too many possible option combinations. */ protected val groupedOptions = Random.shuffle(options).grouped(5).toList /** - * helper method to create list of option sets. + * helper method to create Seq of options. * * {{{ * // examples, * - * input: Set("a") - * output: List(Set(a)) + * input: Seq("a") + * output: Seq(Seq(a)) * - * input: Set("a", "b") - * output: List(Set(a), Set(b), Set(a, b)) + * input: Seq("a", "b") + * output: Seq(Seq(a), Seq(b), Seq(a, b)) * - * input: Set("a", "b", "c") - * output: List(Set(b), Set(c), Set(a), Set(a, b), Set(b, c), Set(a, c), Set(a, b, c)) + * input: Seq("a", "b", "c") + * output: Seq(Seq(b), Seq(c), Seq(a), Seq(a, b), Seq(b, c), Seq(a, c), Seq(a, b, c)) * }}} * * @param options the given options * @tparam T the element type - * @return List of option Set containing all possible combinations of the given options. + * @return Seq of option Seq containing all possible combinations of the given options. */ - def formOptions[T](options: Set[T]): List[Set[T]] = { - - @tailrec - def eachNumberOfOptions(options: Set[T], howMany: Int, acc: Set[Set[T]]): Set[Set[T]] = howMany match { - case 0 => - acc - case _ => - @tailrec - def collectOptions(options: Set[T], howMany: Int, acc: Set[Set[T]]): Set[Set[T]] = howMany match { - case 0 => - acc - case _ => - collectOptions(options, howMany - 1, acc.flatMap(found => (options diff found).map(found + _))) - } - eachNumberOfOptions(options, howMany - 1, collectOptions(options, howMany, Set(Set())) ++ acc) - } - eachNumberOfOptions(options, options.size, Set()).toList.sortBy(_.size) - } + def formOptions[T](options: Seq[T]): Seq[Seq[T]] = (1 to options.length).flatMap(options.combinations(_)) protected def collectPairs[T](base: Set[T])(options: Set[T]): Set[(T, T)] = base.flatMap(option => (options - option).map((option, _)))