Module textual.css.tokenize
Expand source code
from __future__ import annotations
import re
from pathlib import PurePath
from typing import Iterable
from textual.css.tokenizer import Expect, Tokenizer, Token
PERCENT = r"-?\d+\.?\d*%"
DECIMAL = r"-?\d+\.?\d*"
COMMA = r"\s*,\s*"
OPEN_BRACE = r"\(\s*"
CLOSE_BRACE = r"\s*\)"
HEX_COLOR = r"\#[0-9a-fA-F]{8}|\#[0-9a-fA-F]{6}|\#[0-9a-fA-F]{4}|\#[0-9a-fA-F]{3}"
RGB_COLOR = rf"rgb{OPEN_BRACE}{DECIMAL}{COMMA}{DECIMAL}{COMMA}{DECIMAL}{CLOSE_BRACE}|rgba{OPEN_BRACE}{DECIMAL}{COMMA}{DECIMAL}{COMMA}{DECIMAL}{COMMA}{DECIMAL}{CLOSE_BRACE}"
HSL_COLOR = rf"hsl{OPEN_BRACE}{DECIMAL}{COMMA}{PERCENT}{COMMA}{PERCENT}{CLOSE_BRACE}|hsla{OPEN_BRACE}{DECIMAL}{COMMA}{PERCENT}{COMMA}{PERCENT}{COMMA}{DECIMAL}{CLOSE_BRACE}"
COMMENT_START = r"\/\*"
SCALAR = rf"{DECIMAL}(?:fr|%|w|h|vw|vh)"
DURATION = r"\d+\.?\d*(?:ms|s)"
NUMBER = r"\-?\d+\.?\d*"
COLOR = rf"{HEX_COLOR}|{RGB_COLOR}|{HSL_COLOR}"
KEY_VALUE = r"[a-zA-Z_-][a-zA-Z0-9_-]*=[0-9a-zA-Z_\-\/]+"
TOKEN = "[a-zA-Z][a-zA-Z0-9_-]*"
STRING = r"\".*?\""
VARIABLE_REF = r"\$[a-zA-Z0-9_\-]+"
IDENTIFIER = r"[a-zA-Z_\-][a-zA-Z0-9_\-]*"
# Values permitted in variable and rule declarations.
DECLARATION_VALUES = {
"scalar": SCALAR,
"duration": DURATION,
"number": NUMBER,
"color": COLOR,
"key_value": KEY_VALUE,
"token": TOKEN,
"string": STRING,
"variable_ref": VARIABLE_REF,
}
# The tokenizers "expectation" while at the root/highest level of scope
# in the CSS file. At this level we might expect to see selectors, comments,
# variable definitions etc.
expect_root_scope = Expect(
whitespace=r"\s+",
comment_start=COMMENT_START,
selector_start_id=r"\#" + IDENTIFIER,
selector_start_class=r"\." + IDENTIFIER,
selector_start_universal=r"\*",
selector_start=r"[a-zA-Z_\-]+",
variable_name=rf"{VARIABLE_REF}:",
).expect_eof(True)
# After a variable declaration e.g. "$warning-text: TOKENS;"
# for tokenizing variable value ------^~~~~~~^
expect_variable_name_continue = Expect(
variable_value_end=r"\n|;",
whitespace=r"\s+",
comment_start=COMMENT_START,
**DECLARATION_VALUES,
).expect_eof(True)
expect_comment_end = Expect(
comment_end=re.escape("*/"),
)
# After we come across a selector in CSS e.g. ".my-class", we may
# find other selectors, pseudo-classes... e.g. ".my-class :hover"
expect_selector_continue = Expect(
whitespace=r"\s+",
comment_start=COMMENT_START,
pseudo_class=r"\:[a-zA-Z_-]+",
selector_id=r"\#[a-zA-Z_\-][a-zA-Z0-9_\-]*",
selector_class=r"\.[a-zA-Z_\-][a-zA-Z0-9_\-]*",
selector_universal=r"\*",
selector=r"[a-zA-Z_\-]+",
combinator_child=">",
new_selector=r",",
declaration_set_start=r"\{",
)
# A rule declaration e.g. "text: red;"
# ^---^
expect_declaration = Expect(
whitespace=r"\s+",
comment_start=COMMENT_START,
declaration_name=r"[a-zA-Z_\-]+\:",
declaration_set_end=r"\}",
)
expect_declaration_solo = Expect(
whitespace=r"\s+",
comment_start=COMMENT_START,
declaration_name=r"[a-zA-Z_\-]+\:",
declaration_set_end=r"\}",
).expect_eof(True)
# The value(s)/content from a rule declaration e.g. "text: red;"
# ^---^
expect_declaration_content = Expect(
declaration_end=r";",
whitespace=r"\s+",
comment_start=COMMENT_START,
**DECLARATION_VALUES,
important=r"\!important",
comma=",",
declaration_set_end=r"\}",
)
expect_declaration_content_solo = Expect(
declaration_end=r";",
whitespace=r"\s+",
comment_start=COMMENT_START,
**DECLARATION_VALUES,
important=r"\!important",
comma=",",
declaration_set_end=r"\}",
).expect_eof(True)
class TokenizerState:
"""State machine for the tokenizer.
Attributes:
EXPECT: The initial expectation of the tokenizer. Since we start tokenizing
at the root scope, we might expect to see either a variable or selector, for example.
STATE_MAP: Maps token names to Expects, defines the sets of valid tokens
that we'd expect to see next, given the current token. For example, if
we've just processed a variable declaration name, we next expect to see
the value of that variable.
"""
EXPECT = expect_root_scope
STATE_MAP = {
"variable_name": expect_variable_name_continue,
"variable_value_end": expect_root_scope,
"selector_start": expect_selector_continue,
"selector_start_id": expect_selector_continue,
"selector_start_class": expect_selector_continue,
"selector_start_universal": expect_selector_continue,
"selector_id": expect_selector_continue,
"selector_class": expect_selector_continue,
"selector_universal": expect_selector_continue,
"declaration_set_start": expect_declaration,
"declaration_name": expect_declaration_content,
"declaration_end": expect_declaration,
"declaration_set_end": expect_root_scope,
}
def __call__(self, code: str, path: str | PurePath) -> Iterable[Token]:
tokenizer = Tokenizer(code, path=path)
expect = self.EXPECT
get_token = tokenizer.get_token
get_state = self.STATE_MAP.get
while True:
token = get_token(expect)
name = token.name
if name == "comment_start":
tokenizer.skip_to(expect_comment_end)
continue
elif name == "eof":
break
expect = get_state(name, expect)
yield token
class DeclarationTokenizerState(TokenizerState):
EXPECT = expect_declaration_solo
STATE_MAP = {
"declaration_name": expect_declaration_content,
"declaration_end": expect_declaration_solo,
}
class ValueTokenizerState(TokenizerState):
EXPECT = expect_declaration_content_solo
tokenize = TokenizerState()
tokenize_declarations = DeclarationTokenizerState()
tokenize_value = ValueTokenizerState()
def tokenize_values(values: dict[str, str]) -> dict[str, list[Token]]:
"""Tokens the values in a dict of strings.
Args:
values (dict[str, str]): A mapping of CSS variable name on to a value, to be
added to the CSS context.
Returns:
dict[str, list[Token]]: A mapping of name on to a list of tokens,
"""
value_tokens = {
name: list(tokenize_value(value, "__name__")) for name, value in values.items()
}
return value_tokens
if __name__ == "__main__":
from rich import print
css = """#something {
color: rgb(10,12,23)
}
"""
# transition: offset 500 in_out_cubic;
tokens = tokenize(css, __name__)
print(list(tokens))
print(tokenize_values({"primary": "rgb(10,20,30)", "secondary": "#ff00ff"}))
Functions
def tokenize_values(values: dict[str, str]) ‑> dict[str, list[Token]]
-
Tokens the values in a dict of strings.
Args
values
:dict[str, str]
- A mapping of CSS variable name on to a value, to be added to the CSS context.
Returns
dict[str, list[Token]]
- A mapping of name on to a list of tokens,
Expand source code
def tokenize_values(values: dict[str, str]) -> dict[str, list[Token]]: """Tokens the values in a dict of strings. Args: values (dict[str, str]): A mapping of CSS variable name on to a value, to be added to the CSS context. Returns: dict[str, list[Token]]: A mapping of name on to a list of tokens, """ value_tokens = { name: list(tokenize_value(value, "__name__")) for name, value in values.items() } return value_tokens
Classes
class DeclarationTokenizerState
-
State machine for the tokenizer.
Attributes
EXPECT
- The initial expectation of the tokenizer. Since we start tokenizing at the root scope, we might expect to see either a variable or selector, for example.
STATE_MAP
- Maps token names to Expects, defines the sets of valid tokens that we'd expect to see next, given the current token. For example, if we've just processed a variable declaration name, we next expect to see the value of that variable.
Expand source code
class DeclarationTokenizerState(TokenizerState): EXPECT = expect_declaration_solo STATE_MAP = { "declaration_name": expect_declaration_content, "declaration_end": expect_declaration_solo, }
Ancestors
Class variables
var EXPECT
var STATE_MAP
class TokenizerState
-
State machine for the tokenizer.
Attributes
EXPECT
- The initial expectation of the tokenizer. Since we start tokenizing at the root scope, we might expect to see either a variable or selector, for example.
STATE_MAP
- Maps token names to Expects, defines the sets of valid tokens that we'd expect to see next, given the current token. For example, if we've just processed a variable declaration name, we next expect to see the value of that variable.
Expand source code
class TokenizerState: """State machine for the tokenizer. Attributes: EXPECT: The initial expectation of the tokenizer. Since we start tokenizing at the root scope, we might expect to see either a variable or selector, for example. STATE_MAP: Maps token names to Expects, defines the sets of valid tokens that we'd expect to see next, given the current token. For example, if we've just processed a variable declaration name, we next expect to see the value of that variable. """ EXPECT = expect_root_scope STATE_MAP = { "variable_name": expect_variable_name_continue, "variable_value_end": expect_root_scope, "selector_start": expect_selector_continue, "selector_start_id": expect_selector_continue, "selector_start_class": expect_selector_continue, "selector_start_universal": expect_selector_continue, "selector_id": expect_selector_continue, "selector_class": expect_selector_continue, "selector_universal": expect_selector_continue, "declaration_set_start": expect_declaration, "declaration_name": expect_declaration_content, "declaration_end": expect_declaration, "declaration_set_end": expect_root_scope, } def __call__(self, code: str, path: str | PurePath) -> Iterable[Token]: tokenizer = Tokenizer(code, path=path) expect = self.EXPECT get_token = tokenizer.get_token get_state = self.STATE_MAP.get while True: token = get_token(expect) name = token.name if name == "comment_start": tokenizer.skip_to(expect_comment_end) continue elif name == "eof": break expect = get_state(name, expect) yield token
Subclasses
Class variables
var EXPECT
var STATE_MAP
class ValueTokenizerState
-
State machine for the tokenizer.
Attributes
EXPECT
- The initial expectation of the tokenizer. Since we start tokenizing at the root scope, we might expect to see either a variable or selector, for example.
STATE_MAP
- Maps token names to Expects, defines the sets of valid tokens that we'd expect to see next, given the current token. For example, if we've just processed a variable declaration name, we next expect to see the value of that variable.
Expand source code
class ValueTokenizerState(TokenizerState): EXPECT = expect_declaration_content_solo
Ancestors
Class variables
var EXPECT