chatterino2
Code Issues

refactor(eventsub): generate entire files (#5897)

Browse Source
This commit is contained in:
nerix
2025-02-05 16:53:25 +01:00
committed by GitHub
parent 5e3412c3bb
commit 449aefc6bc
54 changed files with 972 additions and 922 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/twitch-eventsub-ws/ast/lib/__init__.py
+3 -5
View File
@@ -1,15 +1,13 @@
from .generate import generate
from .helpers import get_clang_builtin_include_dirs, init_clang_cindex, temporary_file
from .logging import init_logging
from .replace import definition_markers, implementation_markers, replace_in_file
from .parse import parse
__all__ = [
"generate",
"get_clang_builtin_include_dirs",
"init_clang_cindex",
"temporary_file",
"init_logging",
"definition_markers",
"implementation_markers",
"replace_in_file",
"parse",
"temporary_file",
]
lib/twitch-eventsub-ws/ast/lib/build.py
+2 -38
View File
@@ -9,50 +9,14 @@ from .helpers import get_clang_builtin_include_dirs
from .struct import Struct
from .enum import Enum
from .walker import Walker
from .parse import parse
log = logging.getLogger(__name__)
def build_structs(filename: str, additional_includes: list[str] = []) -> List[Struct | Enum]:
if not os.path.isfile(filename):
raise ValueError(f"Path {filename} is not a file. cwd: {os.getcwd()}")
parse_args = [
"-std=c++17",
# Uncomment this if you need to debug where it tries to find headers
# "-H",
]
parse_options = (
clang.cindex.TranslationUnit.PARSE_INCOMPLETE | clang.cindex.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES
)
extra_includes, system_includes = get_clang_builtin_include_dirs()
for include_dir in system_includes:
parse_args.append(f"-isystem{include_dir}")
for include_dir in extra_includes:
parse_args.append(f"-I{include_dir}")
for dir in additional_includes:
parse_args.append("-I")
parse_args.append(dir)
# Append default dirs
# - Append dir of file
file_dir = os.path.dirname(os.path.realpath(filename))
parse_args.append(f"-I{file_dir}")
# - Append project include dir
file_subdir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(filename)), "../.."))
parse_args.append(f"-I{file_subdir}")
# TODO: Use build_commands if available
index = clang.cindex.Index.create()
log.debug("Parsing translation unit")
tu = index.parse(filename, args=parse_args, options=parse_options)
tu = parse(filename, additional_includes)
root = tu.cursor
for diag in tu.diagnostics:
lib/twitch-eventsub-ws/ast/lib/enum.py
+2 -1
View File
@@ -13,12 +13,13 @@ log = logging.getLogger(__name__)
class Enum:
def __init__(self, name: str) -> None:
def __init__(self, name: str, namespace: tuple[str, ...]) -> None:
self.name = name
self.constants: List[EnumConstant] = []
self.parent: str = ""
self.comment_commands: CommentCommands = []
self.inner_root: str = ""
self.namespace = namespace
@property
def full_name(self) -> str:
lib/twitch-eventsub-ws/ast/lib/generate.py
+38 -3
View File
@@ -1,22 +1,57 @@
from typing import Tuple
import logging
from pathlib import Path
from .struct import Struct
from .enum import Enum
from .build import build_structs
from .format import format_code
from .helpers import init_clang_cindex, temporary_file
from .jinja_env import env
from .logging import init_logging
log = logging.getLogger(__name__)
def _generate_implementation(header_path: str, items: list[Struct | Enum]) -> str:
current_ns = ""
doc = f"""
// WARNING: This file is automatically generated. Any changes will be lost.
#include "{header_path}"
#include "twitch-eventsub-ws/chrono.hpp" // IWYU pragma: keep
#include "twitch-eventsub-ws/errors.hpp"
#include <boost/json.hpp>
"""
for item in items:
next_ns = "::".join(item.namespace)
if next_ns != current_ns:
if current_ns:
doc += "} // namespace " + current_ns + "\n"
if next_ns:
doc += "namespace " + next_ns + "{\n\n"
current_ns = next_ns
doc += item.try_value_to_implementation(env) + "\n\n"
if current_ns:
doc += "\n} // namespace " + current_ns + "\n\n"
return doc
def _get_relative_header_path(header_path: str) -> str:
root_path = Path(__file__).parent.parent.parent / "include"
return Path(header_path).relative_to(root_path).as_posix()
def generate(header_path: str, additional_includes: list[str] = []) -> Tuple[str, str]:
structs = build_structs(header_path, additional_includes)
rel_header_path = _get_relative_header_path(header_path)
log.debug("Generate & format definitions")
definitions = format_code("\n\n".join([struct.try_value_to_definition(env) for struct in structs]))
log.debug("Generate & format implementations")
implementations = format_code("\n\n".join([struct.try_value_to_implementation(env) for struct in structs]))
implementations = format_code(_generate_implementation(rel_header_path, structs))
return (definitions, implementations)
lib/twitch-eventsub-ws/ast/lib/logging.py
+3 -3
View File
@@ -4,12 +4,12 @@ import sys
from colorama import Fore, Style
def init_logging() -> None:
def init_logging(level=logging.DEBUG) -> None:
root = logging.getLogger()
root.setLevel(logging.DEBUG)
root.setLevel(level)
handler = logging.StreamHandler(sys.stderr)
handler.setLevel(logging.DEBUG)
handler.setLevel(level)
colors = {
"WARNING": Fore.YELLOW,
lib/twitch-eventsub-ws/ast/lib/member.py
+2 -2
View File
@@ -13,7 +13,7 @@ from .membertype import MemberType
log = logging.getLogger(__name__)
def get_type_name(type: clang.cindex.Type, namespace: List[str]) -> str:
def get_type_name(type: clang.cindex.Type, namespace: tuple[str, ...]) -> str:
if namespace:
namespace_str = f"{'::'.join(namespace)}::"
else:
@@ -68,7 +68,7 @@ class Member:
log.warning(f"Unknown comment command found: {other} with value {value}")
@staticmethod
def from_field(node: clang.cindex.Cursor, namespace: List[str]) -> Member:
def from_field(node: clang.cindex.Cursor, namespace: tuple[str, ...]) -> Member:
assert node.type is not None
name = node.spelling
lib/twitch-eventsub-ws/ast/lib/parse.py
+45
View File
@@ -0,0 +1,45 @@
import os
import clang.cindex
from .helpers import get_clang_builtin_include_dirs
def parse(filename: str, additional_includes: list[str] = []) -> clang.cindex.TranslationUnit:
if not os.path.isfile(filename):
raise ValueError(f"Path {filename} is not a file. cwd: {os.getcwd()}")
parse_args = [
"-std=c++17",
# Uncomment this if you need to debug where it tries to find headers
# "-H",
]
parse_options = (
clang.cindex.TranslationUnit.PARSE_INCOMPLETE | clang.cindex.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES
)
extra_includes, system_includes = get_clang_builtin_include_dirs()
for include_dir in system_includes:
parse_args.append(f"-isystem{include_dir}")
for include_dir in extra_includes:
parse_args.append(f"-I{include_dir}")
for dir in additional_includes:
parse_args.append("-I")
parse_args.append(dir)
# Append default dirs
# - Append dir of file
file_dir = os.path.dirname(os.path.realpath(filename))
parse_args.append(f"-I{file_dir}")
# - Append project include dir
file_subdir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(filename)), "../.."))
parse_args.append(f"-I{file_subdir}")
# TODO: Use build_commands if available
index = clang.cindex.Index.create()
return index.parse(filename, args=parse_args, options=parse_options)
lib/twitch-eventsub-ws/ast/lib/replace.py
-41
View File
@@ -1,41 +0,0 @@
from typing import Tuple
import logging
log = logging.getLogger(__name__)
definition_markers: Tuple[str, str] = (
"// DESERIALIZATION DEFINITION START",
"// DESERIALIZATION DEFINITION END",
)
implementation_markers: Tuple[str, str] = (
"// DESERIALIZATION IMPLEMENTATION START",
"// DESERIALIZATION IMPLEMENTATION END",
)
def replace_in_file(
path: str,
markers: Tuple[str, str],
replacement: str,
) -> None:
with open(path, "r+") as fh:
# Read file into a list of strings
lines = [line.rstrip() for line in fh.readlines()]
start = lines.index(markers[0])
end = lines.index(markers[1], start)
log.debug(f"Markers: {start}-{end}")
lines[start + 1 : end] = [line.rstrip() for line in replacement.splitlines()]
# Clear file
fh.truncate(0)
fh.seek(0)
# Build new file data
data = "\n".join(lines)
# Add a newline at the end of the file
data += "\n"
# Write data to file
fh.write(data)
lib/twitch-eventsub-ws/ast/lib/struct.py
+2 -1
View File
@@ -13,12 +13,13 @@ log = logging.getLogger(__name__)
class Struct:
def __init__(self, name: str) -> None:
def __init__(self, name: str, namespace: tuple[str, ...]) -> None:
self.name = name
self.members: List[Member] = []
self.parent: str = ""
self.comment_commands: CommentCommands = []
self.inner_root: str = ""
self.namespace = namespace
@property
def full_name(self) -> str:
lib/twitch-eventsub-ws/ast/lib/walker.py
+8 -6
View File
@@ -21,12 +21,12 @@ class Walker:
self.real_filepath = os.path.realpath(self.filename)
self.structs: List[Struct] = []
self.enums: List[Enum] = []
self.namespace: List[str] = []
self.namespace: tuple[str, ...] = ()
def handle_node(self, node: clang.cindex.Cursor, struct: Optional[Struct], enum: Optional[Enum]) -> bool:
match node.kind:
case CursorKind.STRUCT_DECL:
new_struct = Struct(node.spelling)
new_struct = Struct(node.spelling, self.namespace)
if node.raw_comment is not None:
new_struct.comment_commands = parse_comment_commands(node.raw_comment)
new_struct.apply_comment_commands(new_struct.comment_commands)
@@ -41,7 +41,7 @@ class Walker:
return True
case CursorKind.ENUM_DECL:
new_enum = Enum(node.spelling)
new_enum = Enum(node.spelling, self.namespace)
if node.raw_comment is not None:
new_enum.comment_commands = parse_comment_commands(node.raw_comment)
new_enum.apply_comment_commands(new_enum.comment_commands)
@@ -77,9 +77,11 @@ class Walker:
struct.members.append(member)
case CursorKind.NAMESPACE:
self.namespace.append(node.spelling)
# Ignore namespaces
pass
self.namespace += (node.spelling,)
for child in node.get_children():
self.walk(child)
self.namespace = self.namespace[:-1]
return True
case CursorKind.FUNCTION_DECL:
# Ignore function declarations