feat: add initial experimental Twitch Eventsub support (#5837)

Co-authored-by: nerix <nerixdev@outlook.de>
This commit is contained in:
pajlada
2025-02-02 17:03:24 +01:00
committed by GitHub
parent f9f7d47b43
commit 0f8a29fdb9
130 changed files with 19656 additions and 8 deletions
+2
View File
@@ -0,0 +1,2 @@
/venv
/__pycache__
+36
View File
@@ -0,0 +1,36 @@
## Available scripts
- `generate-and-replace-dir.py`
Usage: `./generate-and-replace-dir.py <path-to-dir-containing-header-files>`
Will find all header files in that directory (it won't search recursively), and if they also have a matching source file, generate json deserialize definition & implementations for them, and replacing the markers in the given file.
- `get-builtin-include-dirs.py`
Usage: `./get-builtin-include-dirs.py`
Prints what builtin include dirs will be used for any of the other scripts.
- `generate.py`
Usage: `./generate.py <path-to-header-file>`
Generates definitions & implementations for the given header file and write them to temporary files.
- `replace-in-file.py`
Usage: `./replace-in-file.py <path-to-header-file> <path-to-source-file>`
Reads sdin for two file paths containing the definition & implementations.
Can be used in tandem with `generate.py`, but realistically it's only there because I used it at some point.
## Environment variables
The following environment variables can be configured to change the behaviours of this project
- `LLVM_PATH`
Will be used to change where to find the `clang++` executable file.
Example: `LLVM_PATH=/opt/llvm` ./get-builtin-include-dirs.py will use `/opt/llvm/bin/clang++`
- `LIBCLANG_LIBRARY_FILE`
Will be used to change where clang cindex can find the dynamic library.
Must be an absolute path to the file.
Example: `LIBCLANG_LIBRARY_FILE=/opt/llvm/lib/libclang.so.15.0.7`
- `LIBCLANG_LIBRARY_PATH`
Will be used to change where clang cindex can look for the dynamic library.
Must be an absolute path to the directory.
Example: `LIBCLANG_LIBRARY_PATH=/opt/llvm/lib`
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
import logging
import os
import re
import sys
from os.path import realpath
from lib import (
definition_markers,
generate,
implementation_markers,
init_clang_cindex,
init_logging,
replace_in_file,
)
log = logging.getLogger("generate-and-replace-dir")
def main():
init_logging()
init_clang_cindex()
if len(sys.argv) < 2:
log.error(f"usage: {sys.argv[0]} <path-to-dir>")
sys.exit(1)
additional_includes = []
if len(sys.argv) >= 4 and sys.argv[2] == "--includes":
additional_includes = sys.argv[3].split(";")
log.debug(f"additional includes: {additional_includes}")
dir = realpath(sys.argv[1])
log.debug(f"Searching for header files in {dir}")
for path in os.scandir(dir):
if not path.is_file():
continue
if not path.name.endswith(".hpp"):
continue
header_path = path.path
source_path = re.sub(r"\.hpp$", ".cpp", header_path)
source_path = re.sub(r"include[/\\]twitch-eventsub-ws[/\\]", "src/", source_path)
if not os.path.isfile(source_path):
log.warning(f"Header file {header_path} did not have a matching source file at {source_path}")
continue
log.debug(f"Found header & source {header_path} / {source_path}")
(definition, implementation) = generate(header_path, additional_includes)
replace_in_file(header_path, definition_markers, definition)
replace_in_file(source_path, implementation_markers, implementation)
if __name__ == "__main__":
main()
+36
View File
@@ -0,0 +1,36 @@
#!/usr/bin/env python3
import logging
import sys
from lib import generate, init_clang_cindex, init_logging, temporary_file
log = logging.getLogger("generate")
def main():
init_logging()
init_clang_cindex()
if len(sys.argv) < 2:
log.error(f"Missing header file argument. Usage: {sys.argv[0]} <path-to-header-file>")
sys.exit(1)
(definitions, implementations) = generate(sys.argv[1])
with temporary_file() as (f, path):
log.debug(f"Write definitions to {path}")
f.write(definitions)
f.write("\n")
print(path)
with temporary_file() as (f, path):
log.debug(f"Write implementations to {path}")
f.write(implementations)
f.write("\n")
print(path)
if __name__ == "__main__":
main()
+25
View File
@@ -0,0 +1,25 @@
#!/usr/bin/env python3
import logging
from lib import get_clang_builtin_include_dirs, init_logging
log = logging.getLogger(__name__)
def main():
init_logging()
quote_includes, angle_includes = get_clang_builtin_include_dirs()
print("Quote includes:")
print("\n".join(quote_includes))
print()
print("Angle includes:")
print("\n".join(angle_includes))
print()
if __name__ == "__main__":
main()
@@ -0,0 +1,15 @@
from .generate import generate
from .helpers import get_clang_builtin_include_dirs, init_clang_cindex, temporary_file
from .logging import init_logging
from .replace import definition_markers, implementation_markers, replace_in_file
__all__ = [
"generate",
"get_clang_builtin_include_dirs",
"init_clang_cindex",
"temporary_file",
"init_logging",
"definition_markers",
"implementation_markers",
"replace_in_file",
]
+67
View File
@@ -0,0 +1,67 @@
from typing import List
import logging
import os
import clang.cindex
from .helpers import get_clang_builtin_include_dirs
from .struct import Struct
from .enum import Enum
from .walker import Walker
log = logging.getLogger(__name__)
def build_structs(filename: str, additional_includes: list[str] = []) -> List[Struct | Enum]:
if not os.path.isfile(filename):
raise ValueError(f"Path {filename} is not a file. cwd: {os.getcwd()}")
parse_args = [
"-std=c++17",
# Uncomment this if you need to debug where it tries to find headers
# "-H",
]
parse_options = (
clang.cindex.TranslationUnit.PARSE_INCOMPLETE | clang.cindex.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES
)
extra_includes, system_includes = get_clang_builtin_include_dirs()
for include_dir in system_includes:
parse_args.append(f"-isystem{include_dir}")
for include_dir in extra_includes:
parse_args.append(f"-I{include_dir}")
for dir in additional_includes:
parse_args.append("-I")
parse_args.append(dir)
# Append default dirs
# - Append dir of file
file_dir = os.path.dirname(os.path.realpath(filename))
parse_args.append(f"-I{file_dir}")
# - Append project include dir
file_subdir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(filename)), "../.."))
parse_args.append(f"-I{file_subdir}")
# TODO: Use build_commands if available
index = clang.cindex.Index.create()
log.debug("Parsing translation unit")
tu = index.parse(filename, args=parse_args, options=parse_options)
root = tu.cursor
for diag in tu.diagnostics:
log.warning(diag.location)
log.warning(diag.spelling)
log.warning(diag.option)
log.debug("Walking over AST")
walker = Walker(filename)
walker.walk(root)
return walker.enums + walker.structs
@@ -0,0 +1,37 @@
from typing import List, Tuple
import logging
import re
CommentCommands = List[Tuple[str, str]]
log = logging.getLogger(__name__)
def parse_comment_commands(raw_comment: str) -> CommentCommands:
comment_commands: CommentCommands = []
def clean_comment_line(line: str) -> str:
return line.replace("/", "").replace("*", "").strip()
comment_lines = [line for line in map(clean_comment_line, raw_comment.splitlines()) if line != ""]
for comment in comment_lines:
parts = comment.split("=", 2)
if len(parts) != 2:
continue
command = parts[0].strip()
value = parts[1].strip()
comment_commands.append((command, value))
return comment_commands
def json_transform(input_str: str, transformation: str) -> str:
match transformation:
case "snake_case":
return re.sub(r"(?<![A-Z])\B[A-Z]", r"_\g<0>", input_str).lower()
case other:
log.warning(f"Unknown transformation '{other}', ignoring")
return input_str
+63
View File
@@ -0,0 +1,63 @@
from __future__ import annotations
from typing import List
import logging
from jinja2 import Environment
from .comment_commands import CommentCommands
from .enum_constant import EnumConstant
log = logging.getLogger(__name__)
class Enum:
def __init__(self, name: str) -> None:
self.name = name
self.constants: List[EnumConstant] = []
self.parent: str = ""
self.comment_commands: CommentCommands = []
self.inner_root: str = ""
@property
def full_name(self) -> str:
if self.parent:
return f"{self.parent}::{self.name}"
else:
return self.name
def __eq__(self, other: object) -> bool:
if isinstance(other, self.__class__):
if self.name != other.name:
return False
return self.constants == other.constants
return False
def __str__(self) -> str:
pretty_constants = "\n ".join(map(str, self.constants))
return f"enum class {self.name} {{\n {pretty_constants}\n}}"
def try_value_to_implementation(self, env: Environment) -> str:
return env.get_template("enum-implementation.tmpl").render(enum=self)
def try_value_to_definition(self, env: Environment) -> str:
return env.get_template("enum-definition.tmpl").render(enum=self)
def apply_comment_commands(self, comment_commands: CommentCommands) -> None:
for command, value in comment_commands:
match command:
case "json_rename":
# Do nothing on enums
pass
case "json_dont_fail_on_deserialization":
# Do nothing on enums
pass
case "json_transform":
# Do nothing on enums
pass
case "json_inner":
self.inner_root = value
case other:
log.warning(f"Unknown comment command found: {other} with value {value}")
@@ -0,0 +1,73 @@
from __future__ import annotations
from typing import Optional
import logging
import clang.cindex
from .comment_commands import CommentCommands, json_transform, parse_comment_commands
log = logging.getLogger(__name__)
class EnumConstant:
def __init__(
self,
name: str,
) -> None:
self.name = name
self.json_name = name
self.tag: Optional[str] = None
self.dont_fail_on_deserialization: bool = False
def apply_comment_commands(self, comment_commands: CommentCommands) -> None:
for command, value in comment_commands:
match command:
case "json_rename":
# Rename the key that this field will use in json terms
log.debug(f"Rename json key from {self.json_name} to {value}")
self.json_name = value
case "json_dont_fail_on_deserialization":
# Don't fail when an optional object exists and its data is bad
log.debug(f"Don't fail on deserialization for {self.name}")
self.dont_fail_on_deserialization = bool(value.lower() == "true")
case "json_transform":
# Transform the key from whatever-case to case specified by `value`
self.json_name = json_transform(self.json_name, value)
case "json_inner":
# Do nothing on members
pass
case "json_tag":
# Rename the key that this field will use in json terms
log.debug(f"Applied json tag on {self.json_name}: {value}")
self.tag = value
case other:
log.warning(f"Unknown comment command found: {other} with value {value}")
@staticmethod
def from_node(node: clang.cindex.Cursor) -> EnumConstant:
assert node.type is not None
name = node.spelling
enum = EnumConstant(name)
if node.raw_comment is not None:
comment_commands = parse_comment_commands(node.raw_comment)
enum.apply_comment_commands(comment_commands)
return enum
def __eq__(self, other: object) -> bool:
if not isinstance(other, self.__class__):
return False
if self.name != other.name:
return False
return True
def __repr__(self) -> str:
return f"{self.name}"
+13
View File
@@ -0,0 +1,13 @@
import logging
import subprocess
log = logging.getLogger(__name__)
def format_code(code: str) -> str:
proc = subprocess.Popen(["clang-format", "-"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
outs, errs = proc.communicate(input=code.encode(), timeout=2)
if errs is not None:
log.warning(f"Error formatting code: {errs.decode()}")
return outs.decode()
@@ -0,0 +1,22 @@
from typing import Tuple
import logging
from .build import build_structs
from .format import format_code
from .helpers import init_clang_cindex, temporary_file
from .jinja_env import env
from .logging import init_logging
log = logging.getLogger(__name__)
def generate(header_path: str, additional_includes: list[str] = []) -> Tuple[str, str]:
structs = build_structs(header_path, additional_includes)
log.debug("Generate & format definitions")
definitions = format_code("\n\n".join([struct.try_value_to_definition(env) for struct in structs]))
log.debug("Generate & format implementations")
implementations = format_code("\n\n".join([struct.try_value_to_implementation(env) for struct in structs]))
return (definitions, implementations)
+119
View File
@@ -0,0 +1,119 @@
from typing import Generator, List, Tuple
import contextlib
import logging
import os
import subprocess
from io import TextIOWrapper
from tempfile import mkstemp
import clang.cindex
log = logging.getLogger(__name__)
def init_clang_cindex() -> None:
# 1. Has the LIBCLANG_LIBRARY_FILE environment variable been set? Use it
env_library_file = os.environ.get("LIBCLANG_LIBRARY_FILE")
if env_library_file is not None:
log.debug(f"Setting clang library file from LIBCLANG_LIBRARY_PATH environment variable: {env_library_file}")
clang.cindex.Config.set_library_file(env_library_file)
return
# 2. Has the LIBCLANG_LIBRARY_PATH environment variable been set? Use it
env_library_path = os.environ.get("LIBCLANG_LIBRARY_PATH")
if env_library_path is not None:
log.debug(
f"Setting clang library search path from LIBCLANG_LIBRARY_PATH environment variable: {env_library_path}"
)
clang.cindex.Config.set_library_path(env_library_path)
return
import ctypes.util
autodetected_path = ctypes.util.find_library("libclang" if os.name == "nt" else "clang")
if autodetected_path is not None:
log.debug(f"Autodetected clang library file: {autodetected_path}")
clang.cindex.Config.set_library_file(autodetected_path)
return
@contextlib.contextmanager
def temporary_file() -> Generator[Tuple[TextIOWrapper, str], None, None]:
fd, path = mkstemp()
log.debug(f"Opening file {path}")
f = os.fdopen(fd, "w")
yield (f, path)
f.flush()
f.close()
log.debug(f"Closed file {path}")
def get_clang_builtin_include_dirs() -> Tuple[List[str], List[str]]:
quote_includes: List[str] = []
angle_includes: List[str] = []
if os.name == "nt":
return [], []
cmd = [
"clang++",
"-E",
"-x",
"c++",
"-v",
"-",
]
path_entries = os.environ.get("PATH", "").split(os.pathsep)
llvm_path = os.environ.get("LLVM_PATH")
if llvm_path is not None:
path_entries.insert(0, os.path.join(llvm_path, "bin"))
path_str = os.pathsep.join(path_entries)
full_env = os.environ
full_env["PATH"] = path_str
proc = subprocess.Popen(
cmd,
stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
env=full_env,
)
_, errs = proc.communicate(input=None, timeout=10)
doing_quote_includes = False
doing_angle_includes = False
for line in errs.decode("utf8").splitlines():
line = line.strip()
if line == r'#include "..." search starts here:':
doing_angle_includes = False
doing_quote_includes = True
continue
if line == r"#include <...> search starts here:":
doing_quote_includes = False
doing_angle_includes = True
continue
if line == r"End of search list.":
doing_quote_includes = False
doing_angle_includes = False
continue
if doing_quote_includes or doing_angle_includes:
if " " in line:
continue
line = line.split(" ", 1)[0]
p = os.path.realpath(line)
if doing_quote_includes:
quote_includes.append(p)
if doing_angle_includes:
angle_includes.append(p)
return (quote_includes, angle_includes)
@@ -0,0 +1,16 @@
import os
from jinja2 import Environment, FileSystemLoader
from jinja2_workarounds import MultiLineInclude
from .membertype import MemberType
template_paths = os.path.join(os.path.dirname(os.path.realpath(__file__)), "templates")
# print(f"Loading jinja templates from {template_paths}")
env = Environment(
loader=FileSystemLoader(template_paths),
extensions=[MultiLineInclude],
)
env.globals["MemberType"] = MemberType
+32
View File
@@ -0,0 +1,32 @@
import logging
import sys
from colorama import Fore, Style
def init_logging() -> None:
root = logging.getLogger()
root.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stderr)
handler.setLevel(logging.DEBUG)
colors = {
"WARNING": Fore.YELLOW,
"INFO": Fore.WHITE,
"DEBUG": Fore.BLUE,
"CRITICAL": Fore.YELLOW,
"ERROR": Fore.RED,
}
class ColoredFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
levelname = record.levelname
if levelname in colors:
levelname_color = Style.BRIGHT + colors[levelname] + levelname + Style.RESET_ALL
record.levelname = levelname_color
return logging.Formatter.format(self, record)
colored_formatter = ColoredFormatter("%(asctime)s %(levelname)-16s %(name)s: %(message)s")
handler.setFormatter(colored_formatter)
root.addHandler(handler)
+160
View File
@@ -0,0 +1,160 @@
from __future__ import annotations
from typing import Optional, List
import logging
import clang.cindex
from clang.cindex import CursorKind
from .comment_commands import CommentCommands, json_transform, parse_comment_commands
from .membertype import MemberType
log = logging.getLogger(__name__)
def get_type_name(type: clang.cindex.Type, namespace: List[str]) -> str:
if namespace:
namespace_str = f"{'::'.join(namespace)}::"
else:
namespace_str = ""
type_name = type.spelling
if type.is_const_qualified():
type_name = type_name.replace("const", "").strip()
type_name = type_name.removeprefix(namespace_str)
return type_name
class Member:
def __init__(
self,
name: str,
member_type: MemberType = MemberType.BASIC,
type_name: str = "?",
) -> None:
self.name = name
self.json_name = name
self.member_type = member_type
self.type_name = type_name
self.tag: Optional[str] = None
self.dont_fail_on_deserialization: bool = False
def apply_comment_commands(self, comment_commands: CommentCommands) -> None:
for command, value in comment_commands:
match command:
case "json_rename":
# Rename the key that this field will use in json terms
log.debug(f"Rename json key from {self.json_name} to {value}")
self.json_name = value
case "json_dont_fail_on_deserialization":
# Don't fail when an optional object exists and its data is bad
log.debug(f"Don't fail on deserialization for {self.name}")
self.dont_fail_on_deserialization = bool(value.lower() == "true")
case "json_transform":
# Transform the key from whatever-case to case specified by `value`
self.json_name = json_transform(self.json_name, value)
case "json_inner":
# Do nothing on members
pass
case "json_tag":
# Rename the key that this field will use in json terms
log.debug(f"Applied json tag on {self.json_name}: {value}")
self.tag = value
case other:
log.warning(f"Unknown comment command found: {other} with value {value}")
@staticmethod
def from_field(node: clang.cindex.Cursor, namespace: List[str]) -> Member:
assert node.type is not None
name = node.spelling
member_type = MemberType.BASIC
type_name = get_type_name(node.type, namespace)
log.debug(f"{node.spelling} - {type_name} - {node.type.is_const_qualified()}")
ntargs = node.type.get_num_template_arguments()
if ntargs > 0:
overwrite_member_type: Optional[MemberType] = None
# log.debug(node.type.get_template_argument_type(0).kind)
# log.debug(node.type.get_template_argument_type(0).spelling)
# log.debug(node.type.get_template_argument_type(0).get_named_type().spelling)
# log.debug(node.type.get_template_argument_type(0).get_class_type().spelling)
type_name = get_type_name(node.type.get_template_argument_type(0), namespace)
for xd in node.get_children():
match xd.kind:
case CursorKind.NAMESPACE_REF:
# Ignore namespaces
pass
case CursorKind.TEMPLATE_REF:
match xd.spelling:
case "optional":
match overwrite_member_type:
case None:
overwrite_member_type = MemberType.OPTIONAL
case other:
log.warning(f"Optional cannot be added on top of other member type: {other}")
case "vector":
match overwrite_member_type:
case None:
overwrite_member_type = MemberType.VECTOR
case MemberType.OPTIONAL:
overwrite_member_type = MemberType.OPTIONAL_VECTOR
case other:
log.warning(f"Vector cannot be added on top of other member type: {other}")
case other:
log.warning(f"Unhandled template type: {other}")
case CursorKind.TYPE_REF:
type_name = get_type_name(xd.type, namespace)
case other:
log.debug(f"Unhandled child kind type: {other}")
if overwrite_member_type is not None:
member_type = overwrite_member_type
member = Member(name, member_type, type_name)
if node.raw_comment is not None:
comment_commands = parse_comment_commands(node.raw_comment)
member.apply_comment_commands(comment_commands)
return member
def __eq__(self, other: object) -> bool:
if not isinstance(other, self.__class__):
return False
if self.name != other.name:
return False
if self.member_type != other.member_type:
return False
if self.type_name != other.type_name:
return False
return True
def __repr__(self) -> str:
match self.member_type:
case MemberType.BASIC:
return f"{self.type_name} {self.name}"
case MemberType.VECTOR:
return f"std::vector<{self.type_name}> {self.name}"
case MemberType.OPTIONAL:
return f"std::optional<{self.type_name}> {self.name}"
case MemberType.OPTIONAL_VECTOR:
return f"std::optional<std::vector<{self.type_name}>> {self.name}"
@@ -0,0 +1,8 @@
from enum import Enum
class MemberType(Enum):
BASIC = 1
VECTOR = 2
OPTIONAL = 3
OPTIONAL_VECTOR = 4
+41
View File
@@ -0,0 +1,41 @@
from typing import Tuple
import logging
log = logging.getLogger(__name__)
definition_markers: Tuple[str, str] = (
"// DESERIALIZATION DEFINITION START",
"// DESERIALIZATION DEFINITION END",
)
implementation_markers: Tuple[str, str] = (
"// DESERIALIZATION IMPLEMENTATION START",
"// DESERIALIZATION IMPLEMENTATION END",
)
def replace_in_file(
path: str,
markers: Tuple[str, str],
replacement: str,
) -> None:
with open(path, "r+") as fh:
# Read file into a list of strings
lines = [line.rstrip() for line in fh.readlines()]
start = lines.index(markers[0])
end = lines.index(markers[1], start)
log.debug(f"Markers: {start}-{end}")
lines[start + 1 : end] = [line.rstrip() for line in replacement.splitlines()]
# Clear file
fh.truncate(0)
fh.seek(0)
# Build new file data
data = "\n".join(lines)
# Add a newline at the end of the file
data += "\n"
# Write data to file
fh.write(data)
+63
View File
@@ -0,0 +1,63 @@
from __future__ import annotations
from typing import List
import logging
from jinja2 import Environment
from .comment_commands import CommentCommands
from .member import Member
log = logging.getLogger(__name__)
class Struct:
def __init__(self, name: str) -> None:
self.name = name
self.members: List[Member] = []
self.parent: str = ""
self.comment_commands: CommentCommands = []
self.inner_root: str = ""
@property
def full_name(self) -> str:
if self.parent:
return f"{self.parent}::{self.name}"
else:
return self.name
def __eq__(self, other: object) -> bool:
if isinstance(other, self.__class__):
if self.name != other.name:
return False
return self.members == other.members
return False
def __str__(self) -> str:
pretty_members = "\n ".join(map(str, self.members))
return f"struct {self.name} {{\n {pretty_members}\n}}"
def try_value_to_implementation(self, env: Environment) -> str:
return env.get_template("struct-implementation.tmpl").render(struct=self)
def try_value_to_definition(self, env: Environment) -> str:
return env.get_template("struct-definition.tmpl").render(struct=self)
def apply_comment_commands(self, comment_commands: CommentCommands) -> None:
for command, value in comment_commands:
match command:
case "json_rename":
# Do nothing on structs
pass
case "json_dont_fail_on_deserialization":
# Do nothing on structs
pass
case "json_transform":
# Do nothing on structs
pass
case "json_inner":
self.inner_root = value
case other:
log.warning(f"Unknown comment command found: {other} with value {value}")
@@ -0,0 +1,2 @@
boost::json::result_for<{{enum.full_name}}, boost::json::value>::type tag_invoke(
boost::json::try_value_to_tag<{{enum.full_name}}>, const boost::json::value &jvRoot);
@@ -0,0 +1,21 @@
boost::json::result_for<{{enum.full_name}}, boost::json::value>::type tag_invoke(
boost::json::try_value_to_tag<{{enum.full_name}}>, const boost::json::value &jvRoot)
{
if (!jvRoot.is_string())
{
static const error::ApplicationErrorCategory errorMustBeString{"{{enum.full_name}} must be a string"};
return boost::system::error_code{129, errorMustBeString};
}
std::string_view eString(jvRoot.get_string());
using namespace std::string_view_literals;
{%- for constant in enum.constants %}
if (eString == "{{constant.json_name}}"sv) {
return {{enum.full_name}}::{{constant.name}};
}
{%- endfor %}
static const error::ApplicationErrorCategory errorEnumNameDidNotExist{"{{enum.full_name}} did not have a constant that matched this string"};
return boost::system::error_code{129, errorEnumNameDidNotExist};
}
@@ -0,0 +1 @@
return {{field.name}}.error();
@@ -0,0 +1,2 @@
static const error::ApplicationErrorCategory error_missing_field_{{field.name}}{"Missing required key {{field.json_name}}"};
return boost::system::error_code{129, error_missing_field_{{field.name}}};
@@ -0,0 +1,15 @@
const auto *jv{{field.name}} = root.if_contains("{{field.json_name}}");
if (jv{{field.name}} == nullptr)
{
{% include 'error-missing-field.tmpl' indent content %}
}
{% if field.tag %}
auto {{field.name}} = boost::json::try_value_to<{{field.type_name}}>(*jv{{field.name}}, {{field.tag}}());
{% else %}
auto {{field.name}} = boost::json::try_value_to<{{field.type_name}}>(*jv{{field.name}});
{% endif %}
if ({{field.name}}.has_error())
{
{% include 'error-failed-to-deserialize.tmpl' indent content %}
}
@@ -0,0 +1 @@
assert(false && "OPTIONAL VECTOR FIELD UNIMPLEMENTED");
@@ -0,0 +1,27 @@
std::optional<{{field.type_name}}> {{field.name}} = std::nullopt;
const auto *jv{{field.name}} = root.if_contains("{{field.json_name}}");
if (jv{{field.name}} != nullptr && !jv{{field.name}}->is_null())
{
{% if field.tag %}
auto t{{field.name}} = boost::json::try_value_to<{{field.type_name}}>(*jv{{field.name}}, {{field.tag}}());
{% else %}
auto t{{field.name}} = boost::json::try_value_to<{{field.type_name}}>(*jv{{field.name}});
{% endif %}
{% if field.dont_fail_on_deserialization %}
if (t{{field.name}}.has_error())
{
// TODO: report error in some way?
}
else
{
{{field.name}} = t{{field.name}}.value();
}
{% else %}
if (t{{field.name}}.has_error())
{
return t{{field.name}}.error();
}
{{field.name}} = std::move(t{{field.name}}.value());
{% endif %}
}
@@ -0,0 +1,14 @@
{% if field.tag -%}
static_assert(false && "JSON tag support is not implemented for vectors");
{%- endif %}
const auto *jv{{field.name}} = root.if_contains("{{field.json_name}}");
if (jv{{field.name}} == nullptr)
{
{% include 'error-missing-field.tmpl' indent content %}
}
const auto {{field.name}} = boost::json::try_value_to<std::vector<{{field.type_name}}>>(*jv{{field.name}});
if ({{field.name}}.has_error())
{
{% include 'error-failed-to-deserialize.tmpl' indent content %}
}
@@ -0,0 +1 @@
.{{field.name}} = std::move({{field.name}}.value()),
@@ -0,0 +1 @@
assert(false && "OPTIONAL VECTOR INITIALIZER UNIMPLEMENTED");
@@ -0,0 +1 @@
.{{field.name}} = std::move({{field.name}}),
@@ -0,0 +1 @@
.{{field.name}} = {{field.name}}.value(),
@@ -0,0 +1,2 @@
boost::json::result_for<{{struct.full_name}}, boost::json::value>::type tag_invoke(
boost::json::try_value_to_tag<{{struct.full_name}}>, const boost::json::value &jvRoot);
@@ -0,0 +1,58 @@
boost::json::result_for<{{struct.full_name}}, boost::json::value>::type tag_invoke(
boost::json::try_value_to_tag<{{struct.full_name}}>, const boost::json::value &jvRoot)
{
{% if struct.inner_root %}
if (!jvRoot.is_object())
{
static const error::ApplicationErrorCategory errorMustBeObject{"{{struct.full_name}} must be an object"};
return boost::system::error_code{129, errorMustBeObject};
}
const auto &outerRoot = jvRoot.get_object();
const auto *jvInnerRoot = outerRoot.if_contains("{{struct.inner_root}}");
if (jvInnerRoot == nullptr)
{
static const error::ApplicationErrorCategory errorMissing{"{{struct.full_name}}'s key {{struct.inner_root}} is missing"};
return boost::system::error_code{129, errorMissing};
}
if (!jvInnerRoot->is_object())
{
static const error::ApplicationErrorCategory errorMustBeObject{"{{struct.full_name}}'s {{struct.inner_root}} must be an object"};
return boost::system::error_code{129, errorMustBeObject};
}
const auto &root = jvInnerRoot->get_object();
{% else %}
if (!jvRoot.is_object())
{
static const error::ApplicationErrorCategory errorMustBeObject{"{{struct.full_name}} must be an object"};
return boost::system::error_code{129, errorMustBeObject};
}
const auto &root = jvRoot.get_object();
{% endif %}
{% for field in struct.members %}
{% if field.member_type == MemberType.BASIC -%}
{% include 'field-basic.tmpl' indent content %}
{%- elif field.member_type == MemberType.VECTOR -%}
{% include 'field-vector.tmpl' indent content %}
{%- elif field.member_type == MemberType.OPTIONAL -%}
{% include 'field-optional.tmpl' indent content %}
{%- elif field.member_type == MemberType.OPTIONAL_VECTOR -%}
{% include 'field-optional-vector.tmpl' indent content %}
{%- endif -%}
{% endfor %}
return {{struct.full_name}}{
{%- for field in struct.members %}
{% if field.member_type == MemberType.BASIC -%}
{% include 'initializer-basic.tmpl' indent content %}
{%- elif field.member_type == MemberType.VECTOR -%}
{% include 'initializer-vector.tmpl' indent content %}
{%- elif field.member_type == MemberType.OPTIONAL -%}
{% include 'initializer-optional.tmpl' indent content %}
{%- elif field.member_type == MemberType.OPTIONAL_VECTOR -%}
{% include 'initializer-optional-vector.tmpl' indent content %}
{%- endif -%}
{% endfor %}
};
}
@@ -0,0 +1,14 @@
#include <optional>
#include <vector>
struct Pod {
};
struct Const {
const int a;
const bool b;
const char c;
const Pod d;
const std::vector<bool> e;
const std::optional<bool> f;
};
@@ -0,0 +1,5 @@
#include <optional>
struct Optional {
std::optional<bool> a;
};
@@ -0,0 +1,5 @@
struct Simple {
int a;
bool b;
char c;
};
@@ -0,0 +1,10 @@
#include <optional>
#include <string>
#include <vector>
struct String {
std::string a;
const std::string b;
std::vector<std::string> c;
std::optional<std::string> d;
};
@@ -0,0 +1,27 @@
#include <optional>
#include <vector>
struct Foo {
struct InnerFoo {
int asd;
};
int test;
};
struct S {
Foo a;
std::vector<Foo> as;
/**
* 4
* 5
* json_rename=vec_of_ints
**/
std::vector<int> vecOfInts;
std::optional<Foo> ab;
/**
* json_dont_fail_on_deserialization=True
**/
std::optional<Foo> ac;
int b;
};
@@ -0,0 +1,11 @@
#include <vector>
struct Pod {
int a;
bool b;
char c;
};
struct VectorPod {
std::vector<Pod> a;
};
@@ -0,0 +1,6 @@
#include <vector>
struct Vector {
std::vector<bool> a;
std::vector<std::vector<bool>> b;
};
@@ -0,0 +1,175 @@
from lib.build import build_structs
from lib.helpers import init_clang_cindex
from lib.membertype import MemberType
import pytest
def init_clang():
init_clang_cindex()
def test_simple():
structs = build_structs("lib/tests/resources/simple.hpp")
assert len(structs) == 1
s = structs[0]
assert s.name == "Simple"
assert len(s.members) == 3
assert s.members[0].name == "a"
assert s.members[0].member_type == MemberType.BASIC
assert s.members[0].type_name == "int"
assert s.members[1].name == "b"
assert s.members[1].member_type == MemberType.BASIC
assert s.members[1].type_name == "bool"
assert s.members[2].name == "c"
assert s.members[2].member_type == MemberType.BASIC
assert s.members[2].type_name == "char"
def test_vector():
import clang.cindex
print(clang.cindex.conf.get_filename())
structs = build_structs("lib/tests/resources/vector.hpp")
assert len(structs) == 1
s = structs[0]
assert s.name == "Vector"
assert len(s.members) == 2
assert s.members[0].name == "a"
assert s.members[0].member_type == MemberType.VECTOR
assert s.members[0].type_name == "bool"
assert s.members[1].name == "b"
assert s.members[1].member_type == MemberType.VECTOR
assert s.members[1].type_name == "std::vector<bool>"
def test_optional():
import clang.cindex
print(clang.cindex.conf.get_filename())
structs = build_structs("lib/tests/resources/optional.hpp")
assert len(structs) == 1
s = structs[0]
assert s.name == "Optional"
assert len(s.members) == 1
assert s.members[0].name == "a"
assert s.members[0].member_type == MemberType.OPTIONAL
assert s.members[0].type_name == "bool"
def test_vector_pod():
import clang.cindex
print(clang.cindex.conf.get_filename())
structs = build_structs("lib/tests/resources/vector-pod.hpp")
assert len(structs) == 2
s = structs[0]
assert s.name == "Pod"
assert len(s.members) == 3
assert s.members[0].name == "a"
assert s.members[0].member_type == MemberType.BASIC
assert s.members[0].type_name == "int"
assert s.members[1].name == "b"
assert s.members[1].member_type == MemberType.BASIC
assert s.members[1].type_name == "bool"
assert s.members[2].name == "c"
assert s.members[2].member_type == MemberType.BASIC
assert s.members[2].type_name == "char"
s = structs[1]
assert s.name == "VectorPod"
assert len(s.members) == 1
assert s.members[0].name == "a"
assert s.members[0].member_type == MemberType.VECTOR
assert s.members[0].type_name == "Pod"
def test_const():
import clang.cindex
print(clang.cindex.conf.get_filename())
structs = build_structs("lib/tests/resources/const.hpp")
assert len(structs) == 2
s = structs[0]
assert s.name == "Pod"
assert len(s.members) == 0
s = structs[1]
assert s.name == "Const"
assert len(s.members) == 6
assert s.members[0].name == "a"
assert s.members[0].member_type == MemberType.BASIC
assert s.members[0].type_name == "int"
assert s.members[1].name == "b"
assert s.members[1].member_type == MemberType.BASIC
assert s.members[1].type_name == "bool"
assert s.members[2].name == "c"
assert s.members[2].member_type == MemberType.BASIC
assert s.members[2].type_name == "char"
assert s.members[3].name == "d"
assert s.members[3].member_type == MemberType.BASIC
assert s.members[3].type_name == "Pod"
assert s.members[4].name == "e"
assert s.members[4].member_type == MemberType.VECTOR
assert s.members[4].type_name == "bool"
assert s.members[5].name == "f"
assert s.members[5].member_type == MemberType.OPTIONAL
assert s.members[5].type_name == "bool"
def test_string():
import clang.cindex
print(clang.cindex.conf.get_filename())
structs = build_structs("lib/tests/resources/string.hpp")
assert len(structs) == 1
s = structs[0]
assert s.name == "String"
assert len(s.members) == 4
assert s.members[0].name == "a"
assert s.members[0].member_type == MemberType.BASIC
assert s.members[0].type_name == "std::string"
assert s.members[1].name == "b"
assert s.members[1].member_type == MemberType.BASIC
assert s.members[1].type_name == "std::string"
assert s.members[2].name == "c"
assert s.members[2].member_type == MemberType.VECTOR
assert s.members[2].type_name == "std::string"
assert s.members[3].name == "d"
assert s.members[3].member_type == MemberType.OPTIONAL
assert s.members[3].type_name == "std::string"
init_clang()
+102
View File
@@ -0,0 +1,102 @@
from typing import List, Optional
import logging
import os
import clang.cindex
from clang.cindex import CursorKind
from .comment_commands import parse_comment_commands
from .member import Member
from .enum_constant import EnumConstant
from .struct import Struct
from .enum import Enum
log = logging.getLogger(__name__)
class Walker:
def __init__(self, filename: str) -> None:
self.filename = filename
self.real_filepath = os.path.realpath(self.filename)
self.structs: List[Struct] = []
self.enums: List[Enum] = []
self.namespace: List[str] = []
def handle_node(self, node: clang.cindex.Cursor, struct: Optional[Struct], enum: Optional[Enum]) -> bool:
match node.kind:
case CursorKind.STRUCT_DECL:
new_struct = Struct(node.spelling)
if node.raw_comment is not None:
new_struct.comment_commands = parse_comment_commands(node.raw_comment)
new_struct.apply_comment_commands(new_struct.comment_commands)
if struct is not None:
new_struct.parent = struct.full_name
for child in node.get_children():
self.handle_node(child, new_struct, None)
self.structs.append(new_struct)
return True
case CursorKind.ENUM_DECL:
new_enum = Enum(node.spelling)
if node.raw_comment is not None:
new_enum.comment_commands = parse_comment_commands(node.raw_comment)
new_enum.apply_comment_commands(new_enum.comment_commands)
if struct is not None:
new_enum.parent = struct.full_name
for child in node.get_children():
self.handle_node(child, None, new_enum)
self.enums.append(new_enum)
return True
case CursorKind.ENUM_CONSTANT_DECL:
if enum:
# log.warning(
# f"enum constant decl {node.spelling} - enum comments: {enum.comment_commands} - node comments: {node.raw_comment}"
# )
constant = EnumConstant.from_node(node)
constant.apply_comment_commands(enum.comment_commands)
enum.constants.append(constant)
case CursorKind.FIELD_DECL:
type = node.type
if type is None:
# Skip nodes without a type
return False
# log.debug(f"{struct}: {type.spelling} {node.spelling} ({type.kind})")
if struct:
member = Member.from_field(node, self.namespace)
member.apply_comment_commands(struct.comment_commands)
struct.members.append(member)
case CursorKind.NAMESPACE:
self.namespace.append(node.spelling)
# Ignore namespaces
pass
case CursorKind.FUNCTION_DECL:
# Ignore function declarations
pass
case _:
# log.warning(f"unhandled kind: {node.kind}")
pass
return False
def walk(self, node: clang.cindex.Cursor) -> None:
if node.location.file and not clang.cindex.Config().lib.clang_Location_isFromMainFile(node.location):
return
handled = self.handle_node(node, None, None)
if not handled:
for child in node.get_children():
self.walk(child)
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python3
import logging
import sys
from os.path import realpath
from lib import definition_markers, implementation_markers, init_logging, replace_in_file
log = logging.getLogger("replace-in-file")
def main():
init_logging()
if len(sys.argv) < 3:
log.error(f"usage: {sys.argv[0]} <path-to-header-file> <path-to-source-file>")
sys.exit(1)
# The files where we will do the replacements
header_path = realpath(sys.argv[1])
source_path = realpath(sys.argv[2])
# The files where we read the generated definitions & implementations from
# TODO: pass these along in argv too?
stdin_lines = sys.stdin.readlines()
generated_definition_path = realpath(stdin_lines[0].strip())
generated_implementation_path = realpath(stdin_lines[1].strip())
log.debug(f"Generated files: {generated_definition_path} / {generated_implementation_path}")
log.debug(f"Real files: {header_path} / {source_path}")
with open(generated_definition_path, "r") as generated_fh:
replacement = generated_fh.read()
replace_in_file(source_path, definition_markers, replacement)
with open(generated_implementation_path, "r") as generated_fh:
replacement = generated_fh.read()
replace_in_file(source_path, implementation_markers, replacement)
if __name__ == "__main__":
main()
@@ -0,0 +1,4 @@
black==23.3.0
flake8==6.0.0
mypy==1.3.0
isort==5.12.0
@@ -0,0 +1,6 @@
clang==15.0.7
Jinja2==3.1.2
jinja2-workarounds==0.1.0
clang-format==16.0.4
colorama==0.4.6
pytest==7.3.1