Skip to content

Commit

Permalink
docs: better syntax highlighting (#652)
Browse files Browse the repository at this point in the history
### Summary of Changes

Add rules for [all applicable builtin tokens of
Pygments](https://pygments.org/docs/tokens/) to the lexer to improve
syntax highlighting in the grammar. In particular, syntactic elements
now get highlighted similarly to their Python equivalent, where
possible.

---------

Co-authored-by: megalinter-bot <129584137+megalinter-bot@users.noreply.github.com>
  • Loading branch information
lars-reimann and megalinter-bot committed Oct 20, 2023
1 parent b5615c0 commit c59856c
Show file tree
Hide file tree
Showing 2 changed files with 86 additions and 21 deletions.
105 changes: 85 additions & 20 deletions docs/lexer/safe_ds_lexer/_safe_ds_lexer.py
Original file line number Diff line number Diff line change
@@ -1,41 +1,51 @@
from pygments.lexer import RegexLexer, words
from pygments.token import Comment, Keyword, Name, Number, Operator, String, Whitespace

constants = (
keywords_annotation = ("annotation",)

keywords_class = (
"class",
"enum",
)

keywords_constant = (
"attr",
"val",
)

keywords_function = (
"fun",
"pipeline",
"segment",
)

keywords_literal = (
"false",
"null",
"true",
)

keywords = (
"annotation",
keywords_namespace = (
"from",
"package",
)

keywords_generic = (
"as",
"attr",
"class",
"const",
"enum",
"fun",
"import",
"in",
"internal",
"literal",
"out",
"pipeline",
"private",
"schema",
"segment",
"static",
"union",
"val",
"where",
"yield",
)

namespace = (
"from",
"import",
"package",
)

operators = (
"and",
"not",
Expand All @@ -44,6 +54,21 @@
"super",
)

builtins = (
"Any",
"Nothing",
"Boolean",
"Number",
"Int",
"Float",
"ListMap",
"String",
)

identifier_fragment = r"[_a-zA-Z][_a-zA-Z0-9]*"
identifier_regex = rf"{identifier_fragment}|`{identifier_fragment}`"
qualified_name_regex = rf"({identifier_regex})(\.({identifier_regex}))*"


class SafeDsLexer(RegexLexer):
name = "safe-ds"
Expand All @@ -59,17 +84,57 @@ class SafeDsLexer(RegexLexer):

tokens = {
"root": [
# Literals
(r"\b([0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?)\b", Number),
(r'"|}}', String, "string"),
(words(constants, prefix=r"\b", suffix=r"\b"), Keyword.Constant),
(words(keywords, prefix=r"\b", suffix=r"\b"), Keyword),
(words(namespace, prefix=r"\b", suffix=r"\b"), Keyword.Namespace),
# Keywords
(
words(keywords_annotation, prefix=r"\b", suffix=r"\b"),
Keyword,
"annotation",
),
(words(keywords_class, prefix=r"\b", suffix=r"\b"), Keyword, "class"),
(
words(keywords_constant, prefix=r"\b", suffix=r"\b"),
Keyword.Declaration,
"placeholder",
),
(words(keywords_function, prefix=r"\b", suffix=r"\b"), Keyword, "function"),
(words(keywords_literal, prefix=r"\b", suffix=r"\b"), Keyword.Constant),
(
words(keywords_namespace, prefix=r"\b", suffix=r"\b"),
Keyword.Namespace,
"namespace",
),
(words(keywords_generic, prefix=r"\b", suffix=r"\b"), Keyword),
# Operators
(words(operators, prefix=r"\b", suffix=r"\b"), Operator.Word),
(r"`[_a-zA-Z][_a-zA-Z0-9]*`", Name),
# Builtins
(words(builtins, prefix=r"\b", suffix=r"\b"), Name.Builtin),
# Identifiers
(rf"@{identifier_regex}", Name.Decorator),
(identifier_regex, Name),
# Comments
(r"//.+?$", Comment.Single),
(r"/\*[\s\S]*?\*/", Comment.Multiline),
# Whitespace
(r"\s+", Whitespace),
],
"annotation": [
(identifier_regex, Name.Decorator, "#pop"),
],
"class": [
(identifier_regex, Name.Class, "#pop"),
],
"function": [
(identifier_regex, Name.Function, "#pop"),
],
"namespace": [
(qualified_name_regex, Name.Namespace, "#pop"),
],
"placeholder": [
(identifier_regex, Name.Constant, "#pop"),
],
"string": [
(r'([^"{]|\{(?!\{))+', String),
(r'\{\{|"', String, "#pop"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package tests.validation.other.argumentLists.missingRequiredParameter
// $TEST$ no error r"The parameters? .* must be set here\."

@MyAnnotation
segment mySegment2(
segment mySegment3(
myCallableType: (a: Int, b: Int, c: Int = 0) -> ()
) {
val myBlockLambda = (a: Int, b: Int, c: Int = 0) {};
Expand Down

0 comments on commit c59856c

Please sign in to comment.