feat(plugins): add JSON parsing/serialization (#6420)

Reviewed-by: Mm2PL <mm2pl+gh@kotmisia.pl>
Reviewed-by: pajlada <rasmus.karlsson@pajlada.com>
This commit is contained in:
nerix
2025-11-21 17:43:28 +01:00
committed by GitHub
parent 115428d7f4
commit b63739e792
17 changed files with 1509 additions and 4 deletions
+6
View File
@@ -252,6 +252,12 @@ set(SOURCE_FILES
controllers/plugins/api/HTTPResponse.hpp
controllers/plugins/api/IOWrapper.cpp
controllers/plugins/api/IOWrapper.hpp
controllers/plugins/api/JSON.cpp
controllers/plugins/api/JSON.hpp
controllers/plugins/api/JSONParse.cpp
controllers/plugins/api/JSONParse.hpp
controllers/plugins/api/JSONStringify.cpp
controllers/plugins/api/JSONStringify.hpp
controllers/plugins/api/Message.cpp
controllers/plugins/api/Message.hpp
controllers/plugins/api/WebSocket.cpp
+11
View File
@@ -126,6 +126,17 @@ sol::table createEnumTable(sol::state_view &lua)
return out;
}
/// luaL_error but with [[noreturn]]
[[noreturn]]
void fail(lua_State *L, const char *fmt, auto &&...args)
{
luaL_error(L, fmt, std::forward<decltype(args)>(args)...);
// luaL_error is not annotated with [[noreturn]] for backwards
// compatibility, but it will never return. If we ever get here, something
// is seriously wrong, so abort.
std::terminate();
}
} // namespace chatterino::lua
#endif
+11 -1
View File
@@ -11,6 +11,7 @@
# include "controllers/plugins/api/HTTPRequest.hpp"
# include "controllers/plugins/api/HTTPResponse.hpp"
# include "controllers/plugins/api/IOWrapper.hpp"
# include "controllers/plugins/api/JSON.hpp"
# include "controllers/plugins/api/Message.hpp"
# include "controllers/plugins/api/WebSocket.hpp"
# include "controllers/plugins/LuaAPI.hpp"
@@ -40,6 +41,7 @@ namespace chatterino {
PluginController::PluginController(const Paths &paths_)
: paths(paths_)
{
this->loaders_.emplace_back("chatterino.json", &lua::api::loadJson);
}
void PluginController::initialize(Settings &settings)
@@ -264,6 +266,14 @@ void PluginController::initSol(sol::state_view &lua, Plugin *plugin)
sol::table package = g["package"];
package.set_function("loadlib", &lua::api::package_loadlib);
for (const auto &[name, fn] : this->loaders_)
{
package["preload"][name] = [fn](sol::this_main_state state) {
sol::state_view sv(state);
return fn(sv);
};
}
}
void PluginController::load(const QFileInfo &index, const QDir &pluginDir,
@@ -282,7 +292,7 @@ void PluginController::load(const QFileInfo &index, const QDir &pluginDir,
<< " because safe mode is enabled.";
return;
}
PluginController::openLibrariesFor(temp);
this->openLibrariesFor(temp);
if (!PluginController::isPluginEnabled(pluginName) ||
!getSettings()->pluginsEnabled)
+6 -2
View File
@@ -72,15 +72,19 @@ private:
const PluginMeta &meta);
// This function adds lua standard libraries into the state
static void openLibrariesFor(Plugin *plugin);
void openLibrariesFor(Plugin *plugin);
static void initSol(sol::state_view &lua, Plugin *plugin);
void initSol(sol::state_view &lua, Plugin *plugin);
static void loadChatterinoLib(lua_State *l);
bool tryLoadFromDir(const QDir &pluginDir);
std::map<QString, std::unique_ptr<Plugin>> plugins_;
WebSocketPool webSocketPool_;
std::vector<
std::pair<std::string, std::function<sol::object(sol::state_view)>>>
loaders_;
// This is for tests, pay no attention
friend class PluginControllerAccess;
};
+92
View File
@@ -0,0 +1,92 @@
#include "controllers/plugins/api/JSON.hpp"
#ifdef CHATTERINO_HAVE_PLUGINS
# include "controllers/plugins/api/JSONParse.hpp"
# include "controllers/plugins/api/JSONStringify.hpp"
# include "controllers/plugins/LuaUtilities.hpp"
# include <sol/sol.hpp>
namespace {
// NOLINTBEGIN(cppcoreguidelines-pro-type-vararg)
/// `__tostring` implementation for lightuserdata
///
/// If the value is `nullptr`, returns "null", otherwise the pointer value in
/// hex (like in '%p').
int stringifyLightuserdata(lua_State *L)
{
luaL_checktype(L, 1, LUA_TLIGHTUSERDATA);
void *ptr = lua_touserdata(L, 1);
if (ptr == nullptr)
{
lua_pushstring(L, "null");
}
else
{
std::array<char, 2 + 16 + 1> buf{};
// FIXME: use std::format once we require GCC 13/libstdc++ 13
int written = std::snprintf(buf.data(), buf.size(), "%p", ptr);
if (written < 0 || std::ranges::find(buf, '\0') == buf.end())
{
luaL_error(L, "Failed to format pointer");
}
lua_pushstring(L, buf.data());
}
return 1;
}
// NOLINTEND(cppcoreguidelines-pro-type-vararg)
/// Sets the global metatable for lightuserdata
///
/// `nullptr` is used as a sentinel for "null" values in both parsing and
/// stringifying. To allow users to get a proper string representation of
/// this value, we add a `__tostring` method on lightuserdata. As individual
/// lightuserdata values don't have metatables, this is done on the global
/// one.
void setLightuserdataMetatable(lua_State *L)
{
chatterino::lua::StackGuard g(L);
lua_checkstack(L, 4);
lua_pushlightuserdata(L, nullptr);
lua_createtable(L, 0, 1);
lua_pushstring(L, "__tostring");
lua_pushcfunction(L, &stringifyLightuserdata);
// [-4] lightuserdata (nullptr)
// [-3] table {}
// [-2] "__tostring"
// [-1] &stringifyLightuserdata
lua_rawset(L, -3); // tbl.__tostring = fn
// [-2] lightuserdata (nullptr)
// [-1] table { __tostring = fn }
lua_setmetatable(L, -2); // setmeta(lightuserdata) = tbl
// [-1] lightuserdata (nullptr)
lua_pop(L, 1);
}
} // namespace
namespace chatterino::lua::api {
sol::object loadJson(sol::state_view lua)
{
setLightuserdataMetatable(lua.lua_state());
return lua.create_table_with( //
"parse", jsonParse, //
"stringify", jsonStringify, //
// pushed as lightuserdata
"null", static_cast<void *>(nullptr) //
);
}
} // namespace chatterino::lua::api
#endif
+16
View File
@@ -0,0 +1,16 @@
#pragma once
#ifdef CHATTERINO_HAVE_PLUGINS
# include <sol/forward.hpp>
namespace chatterino::lua::api {
/// Loads the 'json' module as a table.
///
/// Because `nullptr` is used as a sentinel for "null", this also adds a
/// `__tostring` method on lightuserdata.
sol::object loadJson(sol::state_view lua);
} // namespace chatterino::lua::api
#endif
+416
View File
@@ -0,0 +1,416 @@
#include "controllers/plugins/api/JSONParse.hpp"
#ifdef CHATTERINO_HAVE_PLUGINS
# include "controllers/plugins/LuaUtilities.hpp"
# include <boost/json/basic_parser.hpp>
# include <boost/json/serialize.hpp>
# include <QVarLengthArray>
# include <sol/sol.hpp>
# include <limits>
namespace {
using namespace chatterino::lua;
// NOLINTNEXTLINE(performance-enum-size) -- we want `int` because that's what boost::system::error_code takes
enum class ParseError : int {
MoreThanOneTopLevel = 1,
TooMuchNesting,
NoElements,
StackExhausted,
};
class ParseErrorCategory final : public boost::system::error_category
{
public:
consteval ParseErrorCategory() = default;
const char *name() const noexcept override
{
return "JSON parse error";
}
std::string message(int ev) const override
{
switch (static_cast<ParseError>(ev))
{
case ParseError::MoreThanOneTopLevel:
return "There was more than one top level element";
case ParseError::TooMuchNesting:
return "Too much nesting";
case ParseError::NoElements:
return "Invalid state: no elements on stack";
case ParseError::StackExhausted:
return "Exhausted Lua stack (stack overflow)";
}
return "Unknown error code";
}
};
constexpr ParseErrorCategory CATEGORY;
boost::system::error_code makeCode(ParseError kind)
{
return {static_cast<int>(kind), CATEGORY};
}
/// Reserve at least `size` slots on the Lua stack.
[[nodiscard]] bool reserveStack(lua_State *state, int size,
boost::system::error_code &ec)
{
if (lua_checkstack(state, size) == 0)
{
ec = makeCode(ParseError::StackExhausted);
return false;
}
return true;
}
// NOLINTBEGIN(cppcoreguidelines-pro-type-vararg)
// NOLINTBEGIN(readability-identifier-naming)
// NOLINTBEGIN(readability-convert-member-functions-to-static)
struct SaxHandler {
constexpr static std::size_t max_object_size = 1 << 20; // about 1 million
constexpr static std::size_t max_array_size = 1 << 28; // about 200 million
constexpr static std::size_t max_key_size = 1 << 29; // 512 MiB
constexpr static std::size_t max_string_size = 1 << 30; // 1 GiB
constexpr static uint16_t MAX_NESTING = 256;
SaxHandler(lua_State *state)
: state(state)
{
}
bool on_document_begin(boost::system::error_code & /* ec */)
{
return true;
}
bool on_document_end(boost::system::error_code & /* ec */)
{
return true;
}
bool on_object_begin(boost::system::error_code &ec)
{
this->elements.append(Element{
.index = 0,
.isObject = 1,
});
if (this->elements.size() > MAX_NESTING)
{
ec = makeCode(ParseError::TooMuchNesting);
return false;
}
// grow stack by 3 (table, key, value)
if (!reserveStack(this->state, 3, ec))
{
return false;
}
lua_newtable(this->state);
return true;
}
bool on_object_end(size_t /* nElements */, boost::system::error_code &ec)
{
if (this->elements.empty())
{
ec = makeCode(ParseError::NoElements);
assert(false && "No information about current on object on stack");
return false;
}
assert(this->elements.back().isObject);
this->elements.pop_back();
return this->appendValue(ec);
}
bool on_array_begin(boost::system::error_code &ec)
{
this->elements.append(Element{
.index = 0,
.isObject = 0,
});
if (this->elements.size() > MAX_NESTING)
{
ec = makeCode(ParseError::TooMuchNesting);
return false;
}
// grow stack by 2 (table, value)
if (!reserveStack(this->state, 2, ec))
{
return false;
}
lua_newtable(this->state);
return true;
}
bool on_array_end(size_t /* nElements */, boost::system::error_code &ec)
{
if (this->elements.empty())
{
ec = makeCode(ParseError::NoElements);
assert(false && "No information about current array on stack");
return false;
}
assert(!this->elements.back().isObject);
this->elements.pop_back();
return this->appendValue(ec);
}
bool on_key_part(boost::json::string_view sv, std::size_t /* n */,
boost::system::error_code & /* ec */)
{
this->buffer.append(sv);
return true; // no push - we'll get a on_key call
}
bool on_key(boost::json::string_view sv, size_t /* n */,
boost::system::error_code & /* ec */)
{
if (this->buffer.empty())
{
lua_pushlstring(this->state, sv.data(), sv.size());
}
else
{
this->buffer.append(sv);
lua_pushlstring(this->state, this->buffer.c_str(),
this->buffer.size());
this->buffer.clear();
}
// no appendValue (wait for the value)
return true;
}
// scalar values
bool on_string_part(boost::json::string_view sv, size_t /* n */,
boost::system::error_code & /* ec */)
{
this->buffer.append(sv);
return true; // no appendValue - we'll get a on_string call
}
bool on_string(boost::json::string_view sv, size_t /* n */,
boost::system::error_code &ec)
{
if (this->buffer.empty())
{
lua_pushlstring(this->state, sv.data(), sv.size());
}
else
{
this->buffer.append(sv);
lua_pushlstring(this->state, this->buffer.c_str(),
this->buffer.size());
this->buffer.clear();
}
return this->appendValue(ec);
}
bool on_number_part(boost::json::string_view /* part */,
boost::system::error_code & /* ec */)
{
// unhandled
return true;
}
bool on_int64(int64_t i, boost::json::string_view /* source */,
boost::system::error_code &ec)
{
lua_pushinteger(this->state, static_cast<lua_Integer>(i));
return this->appendValue(ec);
}
bool on_uint64(uint64_t i, boost::json::string_view /* source */,
boost::system::error_code &ec)
{
if (i <= static_cast<uint64_t>(std::numeric_limits<lua_Integer>::max()))
{
lua_pushinteger(this->state, static_cast<lua_Integer>(i));
}
else
{
lua_pushnumber(this->state, static_cast<lua_Number>(i));
}
return this->appendValue(ec);
}
bool on_double(double d, boost::json::string_view /* source */,
boost::system::error_code &ec)
{
lua_pushnumber(this->state, d);
return this->appendValue(ec);
}
bool on_bool(bool b, boost::system::error_code &ec)
{
lua_pushboolean(this->state, static_cast<int>(b));
return this->appendValue(ec);
}
bool on_null(boost::system::error_code &ec)
{
lua_pushlightuserdata(this->state, nullptr); // json.null
return this->appendValue(ec);
}
bool on_comment_part(boost::json::string_view /* part */,
boost::system::error_code & /* ec */)
{
// unhandled
return true;
}
bool on_comment(boost::json::string_view /* comment */,
boost::system::error_code & /* ec */)
{
// unhandled
return true;
}
private:
/// Append the value to the current container.
/// If we only got a scalar, do nothing but remember that we got some value
/// already.
bool appendValue(boost::system::error_code &ec)
{
if (this->elements.empty())
{
if (this->hadElement)
{
ec = makeCode(ParseError::MoreThanOneTopLevel);
return false;
}
// The element is already at the top of the stack
this->hadElement = true;
return true;
}
if (this->elements.back().isObject)
{
// [-3] table
// [-2] key
// [-1] value
lua_rawset(this->state, -3); // table[key] = value
}
else /* array */
{
// [-2] table
// [-1] value
lua_rawseti(this->state, -2, ++this->elements.back().index);
}
return true;
}
struct Element {
uint16_t index : 15; // index of the last array element
uint16_t isObject : 1;
};
lua_State *state = nullptr;
/// The stack of elements we're in.
///
/// We need to keep track of this to issue the right call to Lua when we see
/// an element.
/// Usually this shouldn't be too deep, so allocate some inline space.
QVarLengthArray<Element, 16> elements;
/// Did we see any top-level element yet?
///
/// This is only for safety. The JSON parser should guarantee that we don't
/// get two top-level elements.
bool hadElement = false;
/// A buffer for partial strings
std::string buffer;
friend class SaxParser;
};
class SaxParser
{
public:
SaxParser(const boost::json::parse_options &opts, lua_State *state)
: parser(opts, state)
{
}
void parse(std::string_view sv)
{
boost::system::error_code ec;
auto nRead = this->parser.write_some(false, sv.data(), sv.length(), ec);
if (ec)
{
fail(this->parser.handler().state,
"Failed to parse JSON: %s (at %s) pos: %d",
ec.message().c_str(), ec.location().to_string().c_str(),
static_cast<int>(nRead));
}
if (nRead != sv.size())
{
fail(this->parser.handler().state,
"Failed to parse JSON: trailing junk", ec.message().c_str());
}
}
private:
boost::json::basic_parser<SaxHandler> parser;
};
// NOLINTEND(readability-convert-member-functions-to-static)
// NOLINTEND(readability-identifier-naming)
// NOLINTEND(cppcoreguidelines-pro-type-vararg)
} // namespace
namespace chatterino::lua::api {
int jsonParse(lua_State *L)
{
auto nArgs = lua_gettop(L);
luaL_argcheck(L, nArgs >= 1, 1, "expected at least one argument");
boost::json::parse_options opts;
opts.max_depth = SaxHandler::MAX_NESTING;
opts.allow_invalid_utf8 = true;
# if BOOST_VERSION >= 108700 // 1.87
opts.allow_invalid_utf16 = true;
# endif
if (nArgs >= 2)
{
auto tbl = sol::stack::check_get<sol::table>(L, 2);
if (tbl)
{
opts.allow_comments = tbl->get_or("allow_comments", false);
opts.allow_trailing_commas =
tbl->get_or("allow_trailing_commas", false);
}
}
size_t size = 0;
const char *str = luaL_checklstring(L, 1, &size);
std::string_view sv(str, size);
SaxParser parser(opts, L);
parser.parse(sv);
return 1;
}
} // namespace chatterino::lua::api
# include <boost/json/src.hpp>
#endif // CHATTERINO_HAVE_PLUGINS
+17
View File
@@ -0,0 +1,17 @@
#pragma once
#ifdef CHATTERINO_HAVE_PLUGINS
# include <sol/forward.hpp>
namespace chatterino::lua::api {
/// Parse JSON from a string into a Lua value
///
/// Exposed in the 'json' package as `parse(string[, options]): any`.
/// It uses the C-Function signature, because we mostly use the raw Lua API in
/// the implementation.
int jsonParse(lua_State *L);
} // namespace chatterino::lua::api
#endif
@@ -0,0 +1,284 @@
#include "controllers/plugins/api/JSONStringify.hpp"
#ifdef CHATTERINO_HAVE_PLUGINS
# include "controllers/plugins/LuaUtilities.hpp"
# include <rapidjson/prettywriter.h>
# include <rapidjson/writer.h>
# include <sol/sol.hpp>
// NOLINTBEGIN(cppcoreguidelines-pro-type-vararg) -- luaL_error is a vararg function
namespace {
using namespace chatterino::lua;
constexpr size_t ENCODE_MAX_TABLE_LENGTH = 1 << 20; // about 1 mil
constexpr uint16_t ENCODE_MAX_DEPTH = 256;
/// Reserve at least `size` slots on the Lua stack.
void reserveStack(lua_State *L, int size)
{
if (lua_checkstack(L, size) == 0)
{
fail(L, "Failed to reserve %d more stack slots (stack overflow)", size);
}
}
/// Get the size of an "array" (table at index -1)
///
/// Lua doesn't have a notion of arrays - everything is a table. We consider a
/// table an array if it only has positive non-zero integer keys. One notable
/// exception is an empty array. An empty table is considered an empty object.
/// If the user wants to specify an empty array, they specify a table with
/// `json.null` at item `0` (i.e. `{ [0] = json.null }`). `json.null` is a
/// `nullptr` lightuserdata.
///
/// Arrays might have holes in them (e.g. `{1, [10]=2}` or `{1, nil, 2}`), so we
/// keep track of the maximum index we saw.
///
/// @returns The size of the array if it is one - otherwise it's an object.
std::optional<size_t> inferArraySize(lua_State *L)
{
reserveStack(L, 3); // key + value + potential error
size_t max = 0;
bool hasNull = false;
lua_pushnil(L); // first key
while (lua_next(L, -2) != 0)
{
// [-3] table
// [-2] key
// [-1] value
if (lua_isinteger(L, -2) == 0)
{
lua_pop(L, 2);
return std::nullopt; // not an array
}
auto key = lua_tointeger(L, -2);
if (key <= 0)
{
// special empty table indicator if element at 0 is our nullptr lightuserdata
if (key == 0 && lua_islightuserdata(L, -1) &&
lua_touserdata(L, -1) == nullptr)
{
hasNull = true;
}
else
{
fail(L, "Table keys can't be negative integers");
}
}
else
{
max = std::max(max, static_cast<size_t>(key));
}
lua_pop(L, 1); // value
}
if (max > ENCODE_MAX_TABLE_LENGTH)
{
fail(L, "Table is too big");
}
if (max == 0 && !hasNull)
{
return std::nullopt; // empty tables are objects
}
return max;
}
void stringifyValue(lua_State *L, auto &writer, uint16_t depth);
void stringifyArray(lua_State *L, auto &writer, uint16_t depth, size_t length)
{
reserveStack(L, 2); // value + potential error
if (!writer.StartArray())
{
fail(L, "Failed to write array start");
}
for (size_t i = 0; i < length; i++)
{
lua_rawgeti(L, -1, static_cast<lua_Integer>(i) + 1);
stringifyValue(L, writer, depth);
lua_pop(L, 1); // pop value
}
if (!writer.EndArray())
{
fail(L, "Failed to write array end");
}
}
void stringifyObject(lua_State *L, auto &writer, uint16_t depth)
{
reserveStack(L, 3); // key + value + potential error
if (!writer.StartObject())
{
fail(L, "Failed to write object start");
}
size_t items = 0;
lua_pushnil(L);
while (lua_next(L, -2) != 0)
{
items++;
if (items > ENCODE_MAX_TABLE_LENGTH)
{
fail(L, "Too many items in table");
}
// Stack:
// [-3] table
// [-2] key
// [-1] value
auto keyType = lua_type(L, -2);
if (keyType != LUA_TSTRING)
{
fail(L, "Object key was not a string, got %s",
lua_typename(L, keyType));
}
size_t len = 0;
const char *str = lua_tolstring(L, -2, &len);
if (!writer.Key(str, len, /*copy=*/true))
{
fail(L, "Failed to write object key");
}
stringifyValue(L, writer, depth);
lua_pop(L, 1); // pop value
}
if (!writer.EndObject())
{
fail(L, "Failed to write object end");
}
}
void stringifyValue(lua_State *L, auto &writer, uint16_t depth)
{
auto typeId = lua_type(L, -1);
bool ok = true;
switch (typeId)
{
case LUA_TNIL:
ok = writer.Null();
break;
case LUA_TBOOLEAN:
ok = writer.Bool(lua_toboolean(L, -1) != 0);
break;
case LUA_TNUMBER: {
if (lua_isinteger(L, -1))
{
ok = writer.Int64(lua_tointeger(L, -1));
}
else
{
ok = writer.Double(lua_tonumber(L, -1));
}
}
break;
case LUA_TSTRING: {
size_t len = 0;
const char *str = lua_tolstring(L, -1, &len);
ok = writer.String(str, len, /*copy=*/true);
}
break;
case LUA_TTABLE: {
if (depth >= ENCODE_MAX_DEPTH)
{
fail(L, "Too deep");
}
auto arraySize = inferArraySize(L);
if (arraySize)
{
stringifyArray(L, writer, depth + 1, *arraySize);
}
else
{
stringifyObject(L, writer, depth + 1);
}
}
break;
case LUA_TLIGHTUSERDATA: {
auto *ptr = lua_touserdata(L, -1);
if (ptr == nullptr)
{
ok = writer.Null();
}
else
{
fail(L, "Unsupported type: lightuserdata");
}
}
break;
case LUA_TFUNCTION:
case LUA_TUSERDATA:
case LUA_TTHREAD:
default:
fail(L, "Unsupported type: %s", lua_typename(L, typeId));
}
if (!ok)
{
fail(L, "Failed to format %s", lua_typename(L, typeId));
}
}
} // namespace
namespace chatterino::lua::api {
int jsonStringify(lua_State *L)
{
auto nArgs = lua_gettop(L);
luaL_argcheck(L, nArgs >= 1, 1, "expected at at least one argument");
bool pretty = false;
char indentChar = ' ';
unsigned int indentSize = 4;
if (nArgs >= 2)
{
auto tbl = sol::stack::check_get<sol::table>(L);
if (tbl)
{
pretty = tbl->get_or("pretty", pretty);
indentChar = tbl->get_or<char>("indent_char", indentChar);
indentSize = tbl->get_or("indent_size", indentSize);
}
// discard everything but the input
lua_pop(L, nArgs - 1);
}
rapidjson::StringBuffer sb;
if (pretty)
{
rapidjson::PrettyWriter pw(
sb, static_cast<rapidjson::CrtAllocator *>(nullptr), 32);
pw.SetIndent(indentChar, indentSize);
stringifyValue(L, pw, 0);
}
else
{
rapidjson::Writer w(sb);
stringifyValue(L, w, 0);
}
lua_pushlstring(L, sb.GetString(), sb.GetSize());
return 1;
}
} // namespace chatterino::lua::api
// NOLINTEND(cppcoreguidelines-pro-type-vararg)
#endif // CHATTERINO_HAVE_PLUGINS
@@ -0,0 +1,17 @@
#pragma once
#ifdef CHATTERINO_HAVE_PLUGINS
# include <sol/forward.hpp>
namespace chatterino::lua::api {
/// Serializes a Lua value to JSON.
///
/// Exposed in the 'json' package as `stringify(value[, options]): string`.
/// It uses the C-Function signature, because we mostly use the raw Lua API in
/// the implementation.
int jsonStringify(lua_State *L);
} // namespace chatterino::lua::api
#endif