Skip to content

Commit

Permalink
perf: implement optimized tree-sitter CST access
Browse files Browse the repository at this point in the history
Refs #691
  • Loading branch information
char0n committed Nov 29, 2021
1 parent e811f7b commit f672afd
Show file tree
Hide file tree
Showing 19 changed files with 473 additions and 34 deletions.
2 changes: 1 addition & 1 deletion packages/apidom-ast/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,4 +69,4 @@ export {
visit,
getNodeType,
isNode,
} from './visitor';
} from './traversal/visitor';
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ export const mergeAll = (

/* eslint-disable no-continue, no-nested-ternary, no-param-reassign */
/**
* visit() will walk through an AST using a depth first traversal, calling
* visit() will walk through an AST using a preorder depth first traversal, calling
* the visitor's enter function at each node in the traversal, and calling the
* leave function after visiting that node and all of its child nodes.
*
Expand Down
2 changes: 1 addition & 1 deletion packages/apidom-ast/test/visitor.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import sinon from 'sinon';
import { assert } from 'chai';

import { visit } from '../src/visitor';
import { visit } from '../src/traversal/visitor';

describe('visitor', function () {
context('given structure with cycle', function () {
Expand Down
2 changes: 1 addition & 1 deletion packages/apidom-parser-adapter-json/.mocharc.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@
module.exports = {
recursive: true,
spec: 'test/**/*.ts',
require: ['test/mocha-bootstrap.js'],
file: ['test/mocha-bootstrap.js'],
};
1 change: 1 addition & 0 deletions packages/apidom-parser-adapter-json/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
"typescript:declaration": "tsc -p declaration.tsconfig.json && rollup -c config/rollup/types.dist.js",
"test": "cross-env BABEL_ENV=cjs mocha",
"perf": "cross-env BABEL_ENV=cjs node ./test/perf/index.js",
"perf:lexical-analysis": "cross-env BABEL_ENV=cjs node ./test/perf/lexical-analysis.js",
"perf:parse-syntactic-analysis-direct": "cross-env BABEL_ENV=cjs node ./test/perf/parse-syntactic-analysis-direct.js",
"perf:parse-syntactic-analysis-indirect": "cross-env BABEL_ENV=cjs node ./test/perf/parse-syntactic-analysis-indirect.js",
"prepack": "copyfiles -u 3 ../../LICENSES/* LICENSES && copyfiles -u 2 ../../NOTICE .",
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import { TreeCursor as NodeTreeCursor, Point as NodePoint } from 'tree-sitter';
import { TreeCursor as WebTreeCursor, Point as WebPoint } from 'web-tree-sitter';

type TreeCursor = NodeTreeCursor | WebTreeCursor;
type Point = NodePoint | WebPoint;

export interface SyntaxNodeSurrogate {
type: string;
startPosition: Point;
endPosition: Point;
children: SyntaxNodeSurrogate[];
[key: string]: unknown;
}

class PreOrderCursorChildrenIterator {
protected cursor;

constructor(cursor: TreeCursor) {
this.cursor = cursor;
}

protected createNode(): SyntaxNodeSurrogate {
return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
children: [],
};
}

public *[Symbol.iterator]() {
// @ts-ignore
const method = this[this.cursor.nodeType];
const currentNode = (method || this.createNode).call(this) as SyntaxNodeSurrogate;
const constructor = this.constructor as any;

if (this.cursor.gotoFirstChild()) {
currentNode.children.push(...[...new constructor(this.cursor)]);

while (this.cursor.gotoNextSibling()) {
currentNode.children.push(...[...new constructor(this.cursor)]);
}

this.cursor.gotoParent();
}

yield currentNode;
}
}

export default PreOrderCursorChildrenIterator;
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import { TreeCursor as NodeTreeCursor, Point as NodePoint } from 'tree-sitter';
import { TreeCursor as WebTreeCursor, Point as WebPoint } from 'web-tree-sitter';

type TreeCursor = NodeTreeCursor | WebTreeCursor;
type Point = NodePoint | WebPoint;

interface SyntaxNodeSurrogate {
type: string;
startPosition: Point;
endPosition: Point;
[key: string]: unknown;
}

class PreOrderCursorIterator {
protected cursor;

constructor(cursor: TreeCursor) {
this.cursor = cursor;
}

protected createNode(): SyntaxNodeSurrogate {
return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
};
}

public *[Symbol.iterator]() {
let reachedRoot = false;

while (!reachedRoot) {
// @ts-ignore
const method = this[this.cursor.nodeType];

yield (method || this.createNode).call(this) as SyntaxNodeSurrogate;

if (this.cursor.gotoFirstChild()) {
continue; // eslint-disable-line no-continue
}

if (this.cursor.gotoNextSibling()) {
continue; // eslint-disable-line no-continue
}

let retracting = true;
while (retracting) {
if (!this.cursor.gotoParent()) {
retracting = false;
reachedRoot = true;
}

if (this.cursor.gotoNextSibling()) {
retracting = false;
}
}
}
}
}

export default PreOrderCursorIterator;
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
import PreOrderCursorChildrenIterator, {
SyntaxNodeSurrogate,
} from '../PreOrderCursorChildrenIterator';

class CursorIterator extends PreOrderCursorChildrenIterator {
document(): SyntaxNodeSurrogate {
return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
children: [],
};
}

object(): SyntaxNodeSurrogate {
return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
fieldName: this.cursor.currentFieldName,
children: [],
};
}

array(): SyntaxNodeSurrogate {
return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
fieldName: this.cursor.currentFieldName,
children: [],
};
}

pair(): SyntaxNodeSurrogate {
return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
get keyNode() {
return this.children.find((node: any) => node.fieldName === 'key');
},
get valueNode() {
return this.children.find((node: any) => node.fieldName === 'value');
},
children: [],
};
}

string(): SyntaxNodeSurrogate {
return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
text: this.cursor.nodeText,
fieldName: this.cursor.currentFieldName,
children: [],
};
}

number(): SyntaxNodeSurrogate {
return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
text: this.cursor.nodeText,
fieldName: this.cursor.currentFieldName,
children: [],
};
}

null(): SyntaxNodeSurrogate {
return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
fieldName: this.cursor.currentFieldName,
children: [],
};
}

true(): SyntaxNodeSurrogate {
return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
fieldName: this.cursor.currentFieldName,
children: [],
};
}

false(): SyntaxNodeSurrogate {
return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
fieldName: this.cursor.currentFieldName,
children: [],
};
}

ERROR(): SyntaxNodeSurrogate {
const { currentNode } = this.cursor;

return {
type: this.cursor.nodeType,
startPosition: this.cursor.startPosition,
endPosition: this.cursor.endPosition,
// @ts-ignore
hasError: () => currentNode.hasError(),
children: [],
};
}
}

export default CursorIterator;
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import stampit from 'stampit';
import { Tree as NodeTree, SyntaxNode as NodeSyntaxNode } from 'tree-sitter';
import { Tree as WebTree, SyntaxNode as WebSyntaxNode } from 'web-tree-sitter';
import { SyntaxNode as NodeSyntaxNode } from 'tree-sitter';
import { SyntaxNode as WebSyntaxNode } from 'web-tree-sitter';
import { visit, getNodeType as getCSTNodeType, isNode as isCSTNode } from '@swagger-api/apidom-ast';
import {
BooleanElement,
Expand All @@ -23,7 +23,6 @@ import {

/* eslint-disable no-underscore-dangle */

type Tree = WebTree | NodeTree;
type SyntaxNode = WebSyntaxNode | NodeSyntaxNode;

const keyMap = {
Expand Down Expand Up @@ -261,7 +260,7 @@ const Visitor = stampit({
* This version of syntactic analysis translates TreeSitter CTS into ApiDOM.
* Single traversal pass is needed to get from CST to ApiDOM.
*/
const analyze = (cst: Tree, { sourceMap = false } = {}): ParseResultElement => {
const analyze = (cst: { rootNode: unknown }, { sourceMap = false } = {}): ParseResultElement => {
const visitor = Visitor();

return visit(cst.rootNode, visitor, {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`adapter-browser should parse 1`] = `
(ParseResultElement
(ObjectElement
(MemberElement
(StringElement)
(StringElement))
(MemberElement
(StringElement)
(BooleanElement))
(MemberElement
(StringElement)
(StringElement))))
`;
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`adapter-node should parse 1`] = `
(ParseResultElement
(ObjectElement
(MemberElement
(StringElement)
(StringElement))
(MemberElement
(StringElement)
(BooleanElement))
(MemberElement
(StringElement)
(StringElement))))
`;
Loading

0 comments on commit f672afd

Please sign in to comment.