feat: add ret tokenizer for RegExp

README.md

@@ -69,6 +69,7 @@ The AST explorer provides following code parsers:
  - [regexp-tree][]
  - [regexpp][]
  - [regjsparser][]
+ - [ret][]
 - Protocol Buffers:
  - [pbkit][]
 - Scala
@@ -172,6 +173,7 @@ are included so you can prototype your own plugins:
 [regexp-tree]: https://github.com/DmitrySoshnikov/regexp-tree
 [regexpp]: https://github.com/mysticatea/regexpp
 [regjsparser]: https://github.com/jviereck/regjsparser
+[ret]: https://github.com/fent/ret.js
 [php-parser]: https://github.com/glayzzle/php-parser
 [pug]: https://github.com/pugjs/pug
 [glimmer]: https://github.com/glimmerjs/glimmer-vm

website/package.json

@@ -141,6 +141,7 @@
  "remark-frontmatter": "^3.0.0",
  "remark-gfm": "^1.0.0",
  "remark-math": "^4.0.0",
+ "ret": "^0.4.0",
  "san": "^3.9.4",
  "scalameta-parsers": "^4.4.17",
  "seafox": "^1.6.9",

website/src/parsers/regexp/ret.js

@@ -0,0 +1,92 @@
+import defaultParserInterface from '../utils/defaultParserInterface';
+import pkg from 'ret/package.json';
+
+const ID = 'ret';
+
+export default {
+ ...defaultParserInterface,
+
+ id: ID,
+ displayName: ID,
+ version: pkg.version,
+ homepage: pkg.homepage,
+
+ loadParser(callback) {
+ require(['ret'], (ret) => {
+ callback(ret);
+ });
+ },
+
+ parse(ret, code, options={}) {
+ this.types = ret.types;
+
+ // Enforce /regexp/ syntax to match other regexp parsers.
+ const firstSlash = code.indexOf('/');
+ const lastSlash = code.lastIndexOf('/');
+ if (firstSlash !== 0 || lastSlash < 1) {
+ throw new Error('Please wrap the regex pattern by slash `/`, i.e. /foo/');
+ }
+
+ // ret does not do anything with the flags, so just strip them.
+ const pattern = code.slice(firstSlash + 1, lastSlash);
+ return ret(pattern);
+ },
+
+ *forEachProperty(node) {
+ if (node && typeof node === 'object') {
+ for (let prop in node) {
+ if (prop === 'type') {
+ continue;
+ }
+
+ yield {
+ value: node[prop],
+ key: prop,
+ computed: false,
+ };
+ }
+
+ if (node.type === this.types.CHAR) {
+ yield {
+ value: String.fromCharCode(node.value),
+ key: 'char',
+ computed: true,
+ };
+ }
+ }
+ },
+
+ getNodeName(node) {
+ switch (node.type) {
+ case this.types.ROOT:
+ return "ROOT";
+ case this.types.GROUP:
+ return "GROUP";
+ case this.types.POSITION:
+ return "POSITION";
+ case this.types.SET:
+ return "SET";
+ case this.types.RANGE:
+ return "RANGE";
+ case this.types.REPETITION:
+ return "REPETITION";
+ case this.types.REFERENCE:
+ return "REFERENCE";
+ case this.types.CHAR:
+ return "CHAR";
+ }
+ },
+
+ opensByDefault(node, key) {
+ return (
+ key === "options" ||
+ key === "stack" ||
+ node.type === this.types.CHAR ||
+ Array.isArray(node)
+ );
+ },
+
+ getDefaultOptions() {
+ return {};
+ },
+};

website/webpack.config.js

@@ -193,6 +193,7 @@ module.exports = Object.assign({
  path.join(__dirname, 'node_modules', 'regexp-tree'),
  path.join(__dirname, 'node_modules', 'regjsparser'),
  path.join(__dirname, 'node_modules', 'regexpp'),
+ path.join(__dirname, 'node_modules', 'ret'),
  path.join(__dirname, 'node_modules', 'simple-html-tokenizer'),
  path.join(__dirname, 'node_modules', 'symbol-observable', 'es'),
  path.join(__dirname, 'node_modules', 'typescript-eslint-parser'),

website/yarn.lock

@@ -9429,6 +9429,11 @@ restore-cursor@^3.1.0:
  onetime "^5.1.0"
  signal-exit "^3.0.2"
 
+ret@^0.4.0:
+ version "0.4.0"
+ resolved "https://registry.yarnpkg.com/ret/-/ret-0.4.0.tgz#b040b0e9c2e1ba2df3e9af39fc8159857f50ea74"
+ integrity sha512-ZAUfxFXaRPGDk8P2qxoDT/xz4DF+Iaq+pIH07ZMGhIGWo/5zFZRg/mPvKkzxRWTqfqy9M2Hw7Uqv3z3XwE6cpw==
+
 ret@~0.1.10:
  version "0.1.15"
  resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc"