Skip to content

Commit

Permalink
refactor: delay creation of operatorsTrie and hide this implementation
Browse files Browse the repository at this point in the history
BREAKING CHANGE: use `operators` instead of `operatorsTrie` as Tokenizer constructor argument, #500
  • Loading branch information
harttle committed Nov 27, 2022
1 parent ff112a4 commit bb58d3e
Show file tree
Hide file tree
Showing 21 changed files with 86 additions and 96 deletions.
1 change: 1 addition & 0 deletions rollup.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ const treeshake = {
propertyReadSideEffects: false
}
const tsconfig = (target) => ({
check: true,
tsconfigOverride: {
include: [ 'src' ],
exclude: [ 'test', 'benchmark' ],
Expand Down
9 changes: 1 addition & 8 deletions src/liquid-options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import { LRU } from './cache/lru'
import { FS } from './fs/fs'
import * as fs from './fs/node'
import { defaultOperators, Operators } from './render/operator'
import { createTrie, Trie } from './util/operator-trie'
import { filters } from './filters'
import { assert } from './types'

Expand Down Expand Up @@ -99,7 +98,6 @@ interface NormalizedOptions extends LiquidOptions {
layouts?: string[];
cache?: LiquidCache;
outputEscape?: OutputEscape;
operatorsTrie?: Trie;
}

export interface NormalizedFullOptions extends NormalizedOptions {
Expand Down Expand Up @@ -130,7 +128,6 @@ export interface NormalizedFullOptions extends NormalizedOptions {
globals: object;
keepOutputType: boolean;
operators: Operators;
operatorsTrie: Trie;
}

export const defaultOptions: NormalizedFullOptions = {
Expand Down Expand Up @@ -160,14 +157,10 @@ export const defaultOptions: NormalizedFullOptions = {
lenientIf: false,
globals: {},
keepOutputType: false,
operators: defaultOperators,
operatorsTrie: createTrie(defaultOperators)
operators: defaultOperators
}

export function normalize (options: LiquidOptions): NormalizedFullOptions {
if (options.hasOwnProperty('operators')) {
(options as NormalizedOptions).operatorsTrie = createTrie(options.operators!)
}
if (options.hasOwnProperty('root')) {
if (!options.hasOwnProperty('partials')) options.partials = options.root
if (!options.hasOwnProperty('layouts')) options.layouts = options.root
Expand Down
2 changes: 1 addition & 1 deletion src/parser/parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ export default class Parser {
this.loader = new Loader(this.liquid.options)
}
public parse (html: string, filepath?: string): Template[] {
const tokenizer = new Tokenizer(html, this.liquid.options.operatorsTrie, filepath)
const tokenizer = new Tokenizer(html, this.liquid.options.operators, filepath)
const tokens = tokenizer.readTopLevelTokens(this.liquid.options)
return this.parseTokens(tokens)
}
Expand Down
9 changes: 6 additions & 3 deletions src/parser/tokenizer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,21 +22,24 @@ import { TokenizationError } from '../util/error'
import { NormalizedFullOptions, defaultOptions } from '../liquid-options'
import { TYPES, QUOTE, BLANK, IDENTIFIER } from '../util/character'
import { matchOperator } from './match-operator'
import { Trie } from '../util/operator-trie'
import { Trie, createTrie } from '../util/operator-trie'
import { Expression } from '../render/expression'
import { Operators } from '../render/operator'
import { LiquidTagToken } from '../tokens/liquid-tag-token'

export class Tokenizer {
p = 0
N: number
private rawBeginAt = -1
private opTrie: Trie

constructor (
public input: string,
private trie: Trie = defaultOptions.operatorsTrie,
operators: Operators = defaultOptions.operators,
public file: string = ''
) {
this.N = input.length
this.opTrie = createTrie(operators)
}

readExpression () {
Expand All @@ -62,7 +65,7 @@ export class Tokenizer {
}
readOperator (): OperatorToken | undefined {
this.skipBlank()
const end = matchOperator(this.input, this.p, this.trie)
const end = matchOperator(this.input, this.p, this.opTrie)
if (end === -1) return
return new OperatorToken(this.input, this.p, (this.p = end), this.file)
}
Expand Down
2 changes: 1 addition & 1 deletion src/tags/assign.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Value, Tokenizer, assert, TagImplOptions, TagToken, Context } from '../

export default {
parse: function (token: TagToken) {
const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
this.key = tokenizer.readIdentifier().content
tokenizer.skipBlank()
assert(tokenizer.peek() === '=', () => `illegal token ${token.getText()}`)
Expand Down
2 changes: 1 addition & 1 deletion src/tags/capture.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { evalQuotedToken } from '../render/expression'

export default {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operators)
this.variable = readVariableName(tokenizer)
assert(this.variable, () => `${tagToken.args} not valid identifier`)

Expand Down
2 changes: 1 addition & 1 deletion src/tags/case.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export default {
.on('tag:when', (token: TagToken) => {
p = []

const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)

while (!tokenizer.end()) {
const value = tokenizer.readValue()
Expand Down
2 changes: 1 addition & 1 deletion src/tags/cycle.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { Tokenizer } from '../parser/tokenizer'

export default {
parse: function (tagToken: TagToken) {
const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operators)
const group = tokenizer.readValue()
tokenizer.skipBlank()

Expand Down
2 changes: 1 addition & 1 deletion src/tags/decrement.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { isNumber, stringify } from '../util/underscore'

export default {
parse: function (token: TagToken) {
const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
this.variable = tokenizer.readIdentifier().content
},
render: function (context: Context, emitter: Emitter) {
Expand Down
2 changes: 1 addition & 1 deletion src/tags/for.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ type valueof<T> = T[keyof T]
export default {
type: 'block',
parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)

const variable = tokenizer.readIdentifier()
const inStr = tokenizer.readIdentifier()
Expand Down
2 changes: 1 addition & 1 deletion src/tags/include.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ export default {
renderFilePath,
parse: function (token: TagToken) {
const args = token.args
const tokenizer = new Tokenizer(args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(args, this.liquid.options.operators)
this['file'] = this.parseFilePath(tokenizer, this.liquid)
this['currentFile'] = token.file

Expand Down
2 changes: 1 addition & 1 deletion src/tags/increment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { Tokenizer, Emitter, TagToken, Context, TagImplOptions } from '../types'

export default {
parse: function (token: TagToken) {
const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
this.variable = tokenizer.readIdentifier().content
},
render: function (context: Context, emitter: Emitter) {
Expand Down
2 changes: 1 addition & 1 deletion src/tags/layout.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ export default {
parseFilePath,
renderFilePath,
parse: function (token: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
this['file'] = this.parseFilePath(tokenizer, this.liquid)
this['currentFile'] = token.file
this.hash = new Hash(tokenizer.remaining())
Expand Down
2 changes: 1 addition & 1 deletion src/tags/liquid.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { Tokenizer } from '../parser/tokenizer'

export default {
parse: function (token: TagToken) {
const tokenizer = new Tokenizer(token.args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(token.args, this.liquid.options.operators)
const tokens = tokenizer.readLiquidTagTokens(this.liquid.options)
this.tpls = this.liquid.parser.parseTokens(tokens)
},
Expand Down
2 changes: 1 addition & 1 deletion src/tags/render.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ export default {
renderFilePath,
parse: function (token: TagToken) {
const args = token.args
const tokenizer = new Tokenizer(args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(args, this.liquid.options.operators)
this['file'] = this.parseFilePath(tokenizer, this.liquid)
this['currentFile'] = token.file
while (!tokenizer.end()) {
Expand Down
2 changes: 1 addition & 1 deletion src/tags/tablerow.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { Tokenizer } from '../parser/tokenizer'

export default {
parse: function (tagToken: TagToken, remainTokens: TopLevelToken[]) {
const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(tagToken.args, this.liquid.options.operators)

const variable = tokenizer.readIdentifier()
tokenizer.skipBlank()
Expand Down
2 changes: 1 addition & 1 deletion src/template/value.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export class Value {
* @param str the value to be valuated, eg.: "foobar" | truncate: 3
*/
public constructor (str: string, liquid: Liquid) {
const tokenizer = new Tokenizer(str, liquid.options.operatorsTrie)
const tokenizer = new Tokenizer(str, liquid.options.operators)
this.initial = tokenizer.readExpression()
this.filters = tokenizer.readFilters().map(({ name, args }) => new Filter(name, liquid.filters.get(name), args, liquid))
}
Expand Down
2 changes: 1 addition & 1 deletion src/tokens/liquid-tag-token.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export class LiquidTagToken extends DelimitedToken {
this.name = ''
this.args = ''
} else {
const tokenizer = new Tokenizer(this.content, options.operatorsTrie)
const tokenizer = new Tokenizer(this.content, options.operators)
this.name = tokenizer.readTagName()
if (!this.name) throw new TokenizationError(`illegal liquid tag syntax`, this)

Expand Down
2 changes: 1 addition & 1 deletion src/tokens/tag-token.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export class TagToken extends DelimitedToken {
const value = input.slice(begin + tagDelimiterLeft.length, end - tagDelimiterRight.length)
super(TokenKind.Tag, value, input, begin, end, trimTagLeft, trimTagRight, file)

const tokenizer = new Tokenizer(this.content, options.operatorsTrie)
const tokenizer = new Tokenizer(this.content, options.operators)
this.name = tokenizer.readTagName()
if (!this.name) throw new TokenizationError(`illegal tag syntax`, this)

Expand Down
Loading

0 comments on commit bb58d3e

Please sign in to comment.