chatterino2
Code Issues

refactor(eventsub): generate entire files (#5897)

Browse Source
This commit is contained in:
nerix
2025-02-05 16:53:25 +01:00
committed by GitHub
parent 5e3412c3bb
commit 449aefc6bc
54 changed files with 972 additions and 922 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/twitch-eventsub-ws/ast/README.md
+4 -13
View File
@@ -1,21 +1,12 @@
## Available scripts
- `generate-and-replace-dir.py`
Usage: `./generate-and-replace-dir.py <path-to-dir-containing-header-files>`
Will find all header files in that directory (it won't search recursively), and if they also have a matching source file, generate json deserialize definition & implementations for them, and replacing the markers in the given file.
- `get-builtin-include-dirs.py`
Usage: `./get-builtin-include-dirs.py`
- `get-builtin-include-dirs.py`
Usage: `./get-builtin-include-dirs.py`
Prints what builtin include dirs will be used for any of the other scripts.
- `generate.py`
Usage: `./generate.py <path-to-header-file>`
Generates definitions & implementations for the given header file and write them to temporary files.
- `replace-in-file.py`
Usage: `./replace-in-file.py <path-to-header-file> <path-to-source-file>`
Reads sdin for two file paths containing the definition & implementations.
Can be used in tandem with `generate.py`, but realistically it's only there because I used it at some point.
Usage: `./generate.py [--includes INCLUDES] [--timestamp path] header_path`
Generates definitions & implementations for the given header file and writes them to the appropriate files.
## Environment variables
lib/twitch-eventsub-ws/ast/check-clang.hpp
+12
View File
@@ -0,0 +1,12 @@
// This is a header used for testing if the library can parse a file.
#include <boost/json.hpp>
#include <QString>
#include <chrono>
boost::json::value doSomething(QString s,
std::chrono::system_clock::time_point tp);
static_assert(sizeof(QString) > sizeof(int));
static_assert(sizeof(boost::json::value) > sizeof(int));
lib/twitch-eventsub-ws/ast/check-clang.py
+20
View File
@@ -0,0 +1,20 @@
import clang.cindex
import sys
from pathlib import Path
from lib import init_clang_cindex, parse
init_clang_cindex()
assert len(sys.argv) == 2
additional_includes = sys.argv[1].split(";")
print("includes:", additional_includes)
tu = parse(Path(__file__).parent / "check-clang.hpp", additional_includes)
if len(tu.diagnostics) > 0:
for diag in tu.diagnostics:
print(diag.location)
print(diag.spelling)
print(diag.option)
assert False, "TU had warnings/errors when parsing"
lib/twitch-eventsub-ws/ast/generate-and-replace-dir.py
-62
View File
@@ -1,62 +0,0 @@
#!/usr/bin/env python3
import logging
import os
import re
import sys
from os.path import realpath
from lib import (
definition_markers,
generate,
implementation_markers,
init_clang_cindex,
init_logging,
replace_in_file,
)
log = logging.getLogger("generate-and-replace-dir")
def main():
init_logging()
init_clang_cindex()
if len(sys.argv) < 2:
log.error(f"usage: {sys.argv[0]} <path-to-dir>")
sys.exit(1)
additional_includes = []
if len(sys.argv) >= 4 and sys.argv[2] == "--includes":
additional_includes = sys.argv[3].split(";")
log.debug(f"additional includes: {additional_includes}")
dir = realpath(sys.argv[1])
log.debug(f"Searching for header files in {dir}")
for path in os.scandir(dir):
if not path.is_file():
continue
if not path.name.endswith(".hpp"):
continue
header_path = path.path
source_path = re.sub(r"\.hpp$", ".cpp", header_path)
source_path = re.sub(r"include[/\\]twitch-eventsub-ws[/\\]", "src/", source_path)
if not os.path.isfile(source_path):
log.warning(f"Header file {header_path} did not have a matching source file at {source_path}")
continue
log.debug(f"Found header & source {header_path} / {source_path}")
(definition, implementation) = generate(header_path, additional_includes)
replace_in_file(header_path, definition_markers, definition)
replace_in_file(source_path, implementation_markers, implementation)
if __name__ == "__main__":
main()
lib/twitch-eventsub-ws/ast/generate.py
+74 -14
View File
@@ -1,35 +1,95 @@
#!/usr/bin/env python3
import logging
import os
import re
import sys
import argparse
from datetime import datetime
from pathlib import Path
from lib import generate, init_clang_cindex, init_logging, temporary_file
from lib import (
generate,
init_clang_cindex,
init_logging,
)
log = logging.getLogger("generate")
def _logging_level():
def to_bool(v: str | None) -> bool:
return bool(v) and v.lower() not in ("false", "0", "off", "no")
return (
logging.DEBUG
if to_bool(os.environ.get("CI")) or to_bool(os.environ.get("DEBUG_EVENTSUB_GEN"))
else logging.WARNING
)
def _validate_header_path(value: str) -> str:
if not value.endswith(".hpp"):
raise argparse.ArgumentTypeError("Header path must end with '.hpp'")
if not os.path.isfile(value):
raise argparse.ArgumentTypeError(f"File '{value}' does not exist")
return value
def main():
init_logging()
init_logging(_logging_level())
init_clang_cindex()
if len(sys.argv) < 2:
log.error(f"Missing header file argument. Usage: {sys.argv[0]} <path-to-header-file>")
log.error(f"usage: {sys.argv[0]} <path-to-header>")
sys.exit(1)
(definitions, implementations) = generate(sys.argv[1])
parser = argparse.ArgumentParser()
parser.add_argument(
"header_path", type=_validate_header_path, help="Path to an existing header file ending with .hpp"
)
parser.add_argument(
"--includes",
type=lambda value: value.split(";") if value else [],
default=[],
help="Semicolon-separated list of include paths",
)
parser.add_argument("--timestamp", metavar="path", help="Path to the timestamp file to be generated")
with temporary_file() as (f, path):
log.debug(f"Write definitions to {path}")
f.write(definitions)
f.write("\n")
print(path)
args = parser.parse_args()
with temporary_file() as (f, path):
log.debug(f"Write implementations to {path}")
f.write(implementations)
f.write("\n")
print(path)
path = Path(args.header_path).resolve()
if not path.is_file():
log.error(f"{path} does not exist")
sys.exit(1)
if not path.name.endswith(".hpp"):
log.error(f"{path} is not a header file")
header_path = str(path)
def_path = re.sub(r"\.hpp$", ".inc", header_path)
source_path = re.sub(r"\.hpp$", ".cpp", header_path)
source_path = re.sub(r"include[/\\]twitch-eventsub-ws[/\\]", "src/generated/", source_path)
log.debug(f"{header_path}: definition={def_path}, implementation={source_path}")
(definition, implementation) = generate(header_path, args.includes)
# ensure directories are created
Path(def_path).parent.mkdir(parents=True, exist_ok=True)
Path(source_path).parent.mkdir(parents=True, exist_ok=True)
with open(def_path, "w") as f:
f.write(definition)
with open(source_path, "w") as f:
f.write(implementation)
if args.timestamp:
path = Path(args.timestamp)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(str(datetime.now().timestamp()))
if __name__ == "__main__":
lib/twitch-eventsub-ws/ast/lib/__init__.py
+3 -5
View File
@@ -1,15 +1,13 @@
from .generate import generate
from .helpers import get_clang_builtin_include_dirs, init_clang_cindex, temporary_file
from .logging import init_logging
from .replace import definition_markers, implementation_markers, replace_in_file
from .parse import parse
__all__ = [
"generate",
"get_clang_builtin_include_dirs",
"init_clang_cindex",
"temporary_file",
"init_logging",
"definition_markers",
"implementation_markers",
"replace_in_file",
"parse",
"temporary_file",
]
lib/twitch-eventsub-ws/ast/lib/build.py
+2 -38
View File
@@ -9,50 +9,14 @@ from .helpers import get_clang_builtin_include_dirs
from .struct import Struct
from .enum import Enum
from .walker import Walker
from .parse import parse
log = logging.getLogger(__name__)
def build_structs(filename: str, additional_includes: list[str] = []) -> List[Struct | Enum]:
if not os.path.isfile(filename):
raise ValueError(f"Path {filename} is not a file. cwd: {os.getcwd()}")
parse_args = [
"-std=c++17",
# Uncomment this if you need to debug where it tries to find headers
# "-H",
]
parse_options = (
clang.cindex.TranslationUnit.PARSE_INCOMPLETE | clang.cindex.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES
)
extra_includes, system_includes = get_clang_builtin_include_dirs()
for include_dir in system_includes:
parse_args.append(f"-isystem{include_dir}")
for include_dir in extra_includes:
parse_args.append(f"-I{include_dir}")
for dir in additional_includes:
parse_args.append("-I")
parse_args.append(dir)
# Append default dirs
# - Append dir of file
file_dir = os.path.dirname(os.path.realpath(filename))
parse_args.append(f"-I{file_dir}")
# - Append project include dir
file_subdir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(filename)), "../.."))
parse_args.append(f"-I{file_subdir}")
# TODO: Use build_commands if available
index = clang.cindex.Index.create()
log.debug("Parsing translation unit")
tu = index.parse(filename, args=parse_args, options=parse_options)
tu = parse(filename, additional_includes)
root = tu.cursor
for diag in tu.diagnostics:
lib/twitch-eventsub-ws/ast/lib/enum.py
+2 -1
View File
@@ -13,12 +13,13 @@ log = logging.getLogger(__name__)
class Enum:
def __init__(self, name: str) -> None:
def __init__(self, name: str, namespace: tuple[str, ...]) -> None:
self.name = name
self.constants: List[EnumConstant] = []
self.parent: str = ""
self.comment_commands: CommentCommands = []
self.inner_root: str = ""
self.namespace = namespace
@property
def full_name(self) -> str:
lib/twitch-eventsub-ws/ast/lib/generate.py
+38 -3
View File
@@ -1,22 +1,57 @@
from typing import Tuple
import logging
from pathlib import Path
from .struct import Struct
from .enum import Enum
from .build import build_structs
from .format import format_code
from .helpers import init_clang_cindex, temporary_file
from .jinja_env import env
from .logging import init_logging
log = logging.getLogger(__name__)
def _generate_implementation(header_path: str, items: list[Struct | Enum]) -> str:
current_ns = ""
doc = f"""
// WARNING: This file is automatically generated. Any changes will be lost.
#include "{header_path}"
#include "twitch-eventsub-ws/chrono.hpp" // IWYU pragma: keep
#include "twitch-eventsub-ws/errors.hpp"
#include <boost/json.hpp>
"""
for item in items:
next_ns = "::".join(item.namespace)
if next_ns != current_ns:
if current_ns:
doc += "} // namespace " + current_ns + "\n"
if next_ns:
doc += "namespace " + next_ns + "{\n\n"
current_ns = next_ns
doc += item.try_value_to_implementation(env) + "\n\n"
if current_ns:
doc += "\n} // namespace " + current_ns + "\n\n"
return doc
def _get_relative_header_path(header_path: str) -> str:
root_path = Path(__file__).parent.parent.parent / "include"
return Path(header_path).relative_to(root_path).as_posix()
def generate(header_path: str, additional_includes: list[str] = []) -> Tuple[str, str]:
structs = build_structs(header_path, additional_includes)
rel_header_path = _get_relative_header_path(header_path)
log.debug("Generate & format definitions")
definitions = format_code("\n\n".join([struct.try_value_to_definition(env) for struct in structs]))
log.debug("Generate & format implementations")
implementations = format_code("\n\n".join([struct.try_value_to_implementation(env) for struct in structs]))
implementations = format_code(_generate_implementation(rel_header_path, structs))
return (definitions, implementations)
lib/twitch-eventsub-ws/ast/lib/logging.py
+3 -3
View File
@@ -4,12 +4,12 @@ import sys
from colorama import Fore, Style
def init_logging() -> None:
def init_logging(level=logging.DEBUG) -> None:
root = logging.getLogger()
root.setLevel(logging.DEBUG)
root.setLevel(level)
handler = logging.StreamHandler(sys.stderr)
handler.setLevel(logging.DEBUG)
handler.setLevel(level)
colors = {
"WARNING": Fore.YELLOW,
lib/twitch-eventsub-ws/ast/lib/member.py
+2 -2
View File
@@ -13,7 +13,7 @@ from .membertype import MemberType
log = logging.getLogger(__name__)
def get_type_name(type: clang.cindex.Type, namespace: List[str]) -> str:
def get_type_name(type: clang.cindex.Type, namespace: tuple[str, ...]) -> str:
if namespace:
namespace_str = f"{'::'.join(namespace)}::"
else:
@@ -68,7 +68,7 @@ class Member:
log.warning(f"Unknown comment command found: {other} with value {value}")
@staticmethod
def from_field(node: clang.cindex.Cursor, namespace: List[str]) -> Member:
def from_field(node: clang.cindex.Cursor, namespace: tuple[str, ...]) -> Member:
assert node.type is not None
name = node.spelling
lib/twitch-eventsub-ws/ast/lib/parse.py
+45
View File
@@ -0,0 +1,45 @@
import os
import clang.cindex
from .helpers import get_clang_builtin_include_dirs
def parse(filename: str, additional_includes: list[str] = []) -> clang.cindex.TranslationUnit:
if not os.path.isfile(filename):
raise ValueError(f"Path {filename} is not a file. cwd: {os.getcwd()}")
parse_args = [
"-std=c++17",
# Uncomment this if you need to debug where it tries to find headers
# "-H",
]
parse_options = (
clang.cindex.TranslationUnit.PARSE_INCOMPLETE | clang.cindex.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES
)
extra_includes, system_includes = get_clang_builtin_include_dirs()
for include_dir in system_includes:
parse_args.append(f"-isystem{include_dir}")
for include_dir in extra_includes:
parse_args.append(f"-I{include_dir}")
for dir in additional_includes:
parse_args.append("-I")
parse_args.append(dir)
# Append default dirs
# - Append dir of file
file_dir = os.path.dirname(os.path.realpath(filename))
parse_args.append(f"-I{file_dir}")
# - Append project include dir
file_subdir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(filename)), "../.."))
parse_args.append(f"-I{file_subdir}")
# TODO: Use build_commands if available
index = clang.cindex.Index.create()
return index.parse(filename, args=parse_args, options=parse_options)
lib/twitch-eventsub-ws/ast/lib/replace.py
-41
View File
@@ -1,41 +0,0 @@
from typing import Tuple
import logging
log = logging.getLogger(__name__)
definition_markers: Tuple[str, str] = (
"// DESERIALIZATION DEFINITION START",
"// DESERIALIZATION DEFINITION END",
)
implementation_markers: Tuple[str, str] = (
"// DESERIALIZATION IMPLEMENTATION START",
"// DESERIALIZATION IMPLEMENTATION END",
)
def replace_in_file(
path: str,
markers: Tuple[str, str],
replacement: str,
) -> None:
with open(path, "r+") as fh:
# Read file into a list of strings
lines = [line.rstrip() for line in fh.readlines()]
start = lines.index(markers[0])
end = lines.index(markers[1], start)
log.debug(f"Markers: {start}-{end}")
lines[start + 1 : end] = [line.rstrip() for line in replacement.splitlines()]
# Clear file
fh.truncate(0)
fh.seek(0)
# Build new file data
data = "\n".join(lines)
# Add a newline at the end of the file
data += "\n"
# Write data to file
fh.write(data)
lib/twitch-eventsub-ws/ast/lib/struct.py
+2 -1
View File
@@ -13,12 +13,13 @@ log = logging.getLogger(__name__)
class Struct:
def __init__(self, name: str) -> None:
def __init__(self, name: str, namespace: tuple[str, ...]) -> None:
self.name = name
self.members: List[Member] = []
self.parent: str = ""
self.comment_commands: CommentCommands = []
self.inner_root: str = ""
self.namespace = namespace
@property
def full_name(self) -> str:
lib/twitch-eventsub-ws/ast/lib/walker.py
+8 -6
View File
@@ -21,12 +21,12 @@ class Walker:
self.real_filepath = os.path.realpath(self.filename)
self.structs: List[Struct] = []
self.enums: List[Enum] = []
self.namespace: List[str] = []
self.namespace: tuple[str, ...] = ()
def handle_node(self, node: clang.cindex.Cursor, struct: Optional[Struct], enum: Optional[Enum]) -> bool:
match node.kind:
case CursorKind.STRUCT_DECL:
new_struct = Struct(node.spelling)
new_struct = Struct(node.spelling, self.namespace)
if node.raw_comment is not None:
new_struct.comment_commands = parse_comment_commands(node.raw_comment)
new_struct.apply_comment_commands(new_struct.comment_commands)
@@ -41,7 +41,7 @@ class Walker:
return True
case CursorKind.ENUM_DECL:
new_enum = Enum(node.spelling)
new_enum = Enum(node.spelling, self.namespace)
if node.raw_comment is not None:
new_enum.comment_commands = parse_comment_commands(node.raw_comment)
new_enum.apply_comment_commands(new_enum.comment_commands)
@@ -77,9 +77,11 @@ class Walker:
struct.members.append(member)
case CursorKind.NAMESPACE:
self.namespace.append(node.spelling)
# Ignore namespaces
pass
self.namespace += (node.spelling,)
for child in node.get_children():
self.walk(child)
self.namespace = self.namespace[:-1]
return True
case CursorKind.FUNCTION_DECL:
# Ignore function declarations
lib/twitch-eventsub-ws/ast/replace-in-file.py
-42
View File
@@ -1,42 +0,0 @@
#!/usr/bin/env python3
import logging
import sys
from os.path import realpath
from lib import definition_markers, implementation_markers, init_logging, replace_in_file
log = logging.getLogger("replace-in-file")
def main():
init_logging()
if len(sys.argv) < 3:
log.error(f"usage: {sys.argv[0]} <path-to-header-file> <path-to-source-file>")
sys.exit(1)
# The files where we will do the replacements
header_path = realpath(sys.argv[1])
source_path = realpath(sys.argv[2])
# The files where we read the generated definitions & implementations from
# TODO: pass these along in argv too?
stdin_lines = sys.stdin.readlines()
generated_definition_path = realpath(stdin_lines[0].strip())
generated_implementation_path = realpath(stdin_lines[1].strip())
log.debug(f"Generated files: {generated_definition_path} / {generated_implementation_path}")
log.debug(f"Real files: {header_path} / {source_path}")
with open(generated_definition_path, "r") as generated_fh:
replacement = generated_fh.read()
replace_in_file(source_path, definition_markers, replacement)
with open(generated_implementation_path, "r") as generated_fh:
replacement = generated_fh.read()
replace_in_file(source_path, implementation_markers, replacement)
if __name__ == "__main__":
main()