From 321242b0d9bf737dff931be8cedd6a1590a3be5a Mon Sep 17 00:00:00 2001 From: marios8543 Date: Tue, 17 Oct 2023 23:52:18 +0300 Subject: Experimental support for async method calls --- backend/src/plugin.py | 163 ------------------------------ backend/src/plugin/method_call_request.py | 29 ++++++ backend/src/plugin/plugin.py | 55 ++++++++++ backend/src/plugin/sandboxed_plugin.py | 136 +++++++++++++++++++++++++ 4 files changed, 220 insertions(+), 163 deletions(-) delete mode 100644 backend/src/plugin.py create mode 100644 backend/src/plugin/method_call_request.py create mode 100644 backend/src/plugin/plugin.py create mode 100644 backend/src/plugin/sandboxed_plugin.py (limited to 'backend') diff --git a/backend/src/plugin.py b/backend/src/plugin.py deleted file mode 100644 index ab33c3eb..00000000 --- a/backend/src/plugin.py +++ /dev/null @@ -1,163 +0,0 @@ -import multiprocessing -from asyncio import (Lock, get_event_loop, new_event_loop, - set_event_loop, sleep) -from importlib.util import module_from_spec, spec_from_file_location -from json import dumps, load, loads -from logging import getLogger -from traceback import format_exc -from os import path, environ -from signal import SIGINT, signal -from sys import exit, path as syspath, modules as sysmodules -from typing import Any, Dict -from .localsocket import LocalSocket -from .localplatform import setgid, setuid, get_username, get_home_path -from .customtypes import UserType -from . import helpers - -class PluginWrapper: - def __init__(self, file: str, plugin_directory: str, plugin_path: str) -> None: - self.file = file - self.plugin_path = plugin_path - self.plugin_directory = plugin_directory - self.method_call_lock = Lock() - self.socket: LocalSocket = LocalSocket(self._on_new_message) - - self.version = None - - json = load(open(path.join(plugin_path, plugin_directory, "plugin.json"), "r", encoding="utf-8")) - if path.isfile(path.join(plugin_path, plugin_directory, "package.json")): - package_json = load(open(path.join(plugin_path, plugin_directory, "package.json"), "r", encoding="utf-8")) - self.version = package_json["version"] - - self.name = json["name"] - self.author = json["author"] - self.flags = json["flags"] - - self.log = getLogger("plugin") - - self.passive = not path.isfile(self.file) - - def __str__(self) -> str: - return self.name - - def _init(self): - try: - signal(SIGINT, lambda s, f: exit(0)) - - set_event_loop(new_event_loop()) - if self.passive: - return - setgid(UserType.ROOT if "root" in self.flags else UserType.HOST_USER) - setuid(UserType.ROOT if "root" in self.flags else UserType.HOST_USER) - # export a bunch of environment variables to help plugin developers - environ["HOME"] = get_home_path(UserType.ROOT if "root" in self.flags else UserType.HOST_USER) - environ["USER"] = "root" if "root" in self.flags else get_username() - environ["DECKY_VERSION"] = helpers.get_loader_version() - environ["DECKY_USER"] = get_username() - environ["DECKY_USER_HOME"] = helpers.get_home_path() - environ["DECKY_HOME"] = helpers.get_homebrew_path() - environ["DECKY_PLUGIN_SETTINGS_DIR"] = path.join(environ["DECKY_HOME"], "settings", self.plugin_directory) - helpers.mkdir_as_user(path.join(environ["DECKY_HOME"], "settings")) - helpers.mkdir_as_user(environ["DECKY_PLUGIN_SETTINGS_DIR"]) - environ["DECKY_PLUGIN_RUNTIME_DIR"] = path.join(environ["DECKY_HOME"], "data", self.plugin_directory) - helpers.mkdir_as_user(path.join(environ["DECKY_HOME"], "data")) - helpers.mkdir_as_user(environ["DECKY_PLUGIN_RUNTIME_DIR"]) - environ["DECKY_PLUGIN_LOG_DIR"] = path.join(environ["DECKY_HOME"], "logs", self.plugin_directory) - helpers.mkdir_as_user(path.join(environ["DECKY_HOME"], "logs")) - helpers.mkdir_as_user(environ["DECKY_PLUGIN_LOG_DIR"]) - environ["DECKY_PLUGIN_DIR"] = path.join(self.plugin_path, self.plugin_directory) - environ["DECKY_PLUGIN_NAME"] = self.name - if self.version: - environ["DECKY_PLUGIN_VERSION"] = self.version - environ["DECKY_PLUGIN_AUTHOR"] = self.author - - # append the plugin's `py_modules` to the recognized python paths - syspath.append(path.join(environ["DECKY_PLUGIN_DIR"], "py_modules")) - - #TODO: FIX IN A LESS CURSED WAY - keys = [key.replace("src.", "") for key in sysmodules if key.startswith("src.")] - for key in keys: - sysmodules[key] = sysmodules["src"].__dict__[key] - - spec = spec_from_file_location("_", self.file) - assert spec is not None - module = module_from_spec(spec) - assert spec.loader is not None - spec.loader.exec_module(module) - self.Plugin = module.Plugin - - if hasattr(self.Plugin, "_migration"): - get_event_loop().run_until_complete(self.Plugin._migration(self.Plugin)) - if hasattr(self.Plugin, "_main"): - get_event_loop().create_task(self.Plugin._main(self.Plugin)) - get_event_loop().create_task(self.socket.setup_server()) - get_event_loop().run_forever() - except: - self.log.error("Failed to start " + self.name + "!\n" + format_exc()) - exit(0) - - async def _unload(self): - try: - self.log.info("Attempting to unload with plugin " + self.name + "'s \"_unload\" function.\n") - if hasattr(self.Plugin, "_unload"): - await self.Plugin._unload(self.Plugin) - self.log.info("Unloaded " + self.name + "\n") - else: - self.log.info("Could not find \"_unload\" in " + self.name + "'s main.py" + "\n") - except: - self.log.error("Failed to unload " + self.name + "!\n" + format_exc()) - exit(0) - - async def _on_new_message(self, message : str) -> str|None: - data = loads(message) - - if "stop" in data: - self.log.info("Calling Loader unload function.") - await self._unload() - get_event_loop().stop() - while get_event_loop().is_running(): - await sleep(0) - get_event_loop().close() - raise Exception("Closing message listener") - - # TODO there is definitely a better way to type this - d: Dict[str, Any] = {"res": None, "success": True} - try: - d["res"] = await getattr(self.Plugin, data["method"])(self.Plugin, **data["args"]) - except Exception as e: - d["res"] = str(e) - d["success"] = False - finally: - return dumps(d, ensure_ascii=False) - - def start(self): - if self.passive: - return self - multiprocessing.Process(target=self._init).start() - return self - - def stop(self): - if self.passive: - return - - async def _(self: PluginWrapper): - await self.socket.write_single_line(dumps({ "stop": True }, ensure_ascii=False)) - await self.socket.close_socket_connection() - - get_event_loop().create_task(_(self)) - - async def execute_method(self, method_name: str, kwargs: Dict[Any, Any]): - if self.passive: - raise RuntimeError("This plugin is passive (aka does not implement main.py)") - async with self.method_call_lock: - # reader, writer = - await self.socket.get_socket_connection() - - await self.socket.write_single_line(dumps({ "method": method_name, "args": kwargs }, ensure_ascii=False)) - - line = await self.socket.read_single_line() - if line != None: - res = loads(line) - if not res["success"]: - raise Exception(res["res"]) - return res["res"] \ No newline at end of file diff --git a/backend/src/plugin/method_call_request.py b/backend/src/plugin/method_call_request.py new file mode 100644 index 00000000..8d93a6cc --- /dev/null +++ b/backend/src/plugin/method_call_request.py @@ -0,0 +1,29 @@ +from typing import Any, TypedDict +from uuid import uuid4 +from asyncio import Event + +class SocketResponseDict(TypedDict): + id: str + success: bool + res: Any + +class MethodCallResponse: + def __init__(self, success: bool, result: Any) -> None: + self.success = success + self.result = result + +class MethodCallRequest: + def __init__(self) -> None: + self.id = str(uuid4()) + self.event = Event() + self.response: MethodCallResponse + + def set_result(self, dc: SocketResponseDict): + self.response = MethodCallResponse(dc["success"], dc["res"]) + self.event.set() + + async def wait_for_result(self): + await self.event.wait() + if not self.response.success: + raise Exception(self.response.result) + return self.response \ No newline at end of file diff --git a/backend/src/plugin/plugin.py b/backend/src/plugin/plugin.py new file mode 100644 index 00000000..b7d0a44a --- /dev/null +++ b/backend/src/plugin/plugin.py @@ -0,0 +1,55 @@ +from json import dumps, load, loads +from logging import getLogger +from os import path + +from .sandboxed_plugin import SandboxedPlugin +from .method_call_request import MethodCallRequest +from ..localplatform.localsocket import LocalSocket + +from typing import Any, Dict + +class PluginWrapper: + def __init__(self, file: str, plugin_directory: str, plugin_path: str) -> None: + self.file = file + self.plugin_path = plugin_path + self.plugin_directory = plugin_directory + + self.version = None + + json = load(open(path.join(plugin_path, plugin_directory, "plugin.json"), "r", encoding="utf-8")) + if path.isfile(path.join(plugin_path, plugin_directory, "package.json")): + package_json = load(open(path.join(plugin_path, plugin_directory, "package.json"), "r", encoding="utf-8")) + self.version = package_json["version"] + + self.name = json["name"] + self.author = json["author"] + self.flags = json["flags"] + self.passive = not path.isfile(self.file) + + self.log = getLogger("plugin") + self.method_call_requests: Dict[str, MethodCallRequest] = {} + self.sandboxed_plugin = SandboxedPlugin(self.name, self.passive, self.flags, self.file, self.plugin_directory, self.plugin_path, self.version, self.author) + #TODO: Maybe somehow make LocalSocket not require on_new_message to make this more clear + self.socket = LocalSocket(self.sandboxed_plugin.on_new_message) + self.sandboxed_plugin.start(self.socket) + + def __str__(self) -> str: + return self.name + + async def response_listener(self): + while True: + line = await self.socket.read_single_line() + if line != None: + res = loads(line) + self.method_call_requests.pop(res["id"]).set_result(res) + + async def execute_method(self, method_name: str, kwargs: Dict[Any, Any]): + if self.passive: + raise RuntimeError("This plugin is passive (aka does not implement main.py)") + + request = MethodCallRequest() + await self.socket.get_socket_connection() + await self.socket.write_single_line(dumps({ "method": method_name, "args": kwargs, "id": request.id }, ensure_ascii=False)) + self.method_call_requests[request.id] = request + + return await request.wait_for_result() \ No newline at end of file diff --git a/backend/src/plugin/sandboxed_plugin.py b/backend/src/plugin/sandboxed_plugin.py new file mode 100644 index 00000000..fea02ac2 --- /dev/null +++ b/backend/src/plugin/sandboxed_plugin.py @@ -0,0 +1,136 @@ +from os import path, environ +from signal import SIGINT, signal +from importlib.util import module_from_spec, spec_from_file_location +from json import dumps, loads +from logging import getLogger +import multiprocessing +from sys import exit, path as syspath +from traceback import format_exc +from asyncio import (get_event_loop, new_event_loop, + set_event_loop, sleep) + +from .method_call_request import SocketResponseDict +from ..localplatform.localsocket import LocalSocket +from ..localplatform.localplatform import setgid, setuid, get_username, get_home_path +from ..customtypes import UserType +from .. import helpers + +from typing import List + +class SandboxedPlugin: + def __init__(self, + name: str, + passive: bool, + flags: List[str], + file: str, + plugin_directory: str, + plugin_path: str, + version: str|None, + author: str) -> None: + self.name = name + self.passive = passive + self.flags = flags + self.file = file + self.plugin_path = plugin_path + self.plugin_directory = plugin_directory + self.version = version + self.author = author + + self.log = getLogger("plugin") + + def _init(self, socket: LocalSocket): + try: + signal(SIGINT, lambda s, f: exit(0)) + + set_event_loop(new_event_loop()) + if self.passive: + return + setgid(UserType.ROOT if "root" in self.flags else UserType.HOST_USER) + setuid(UserType.ROOT if "root" in self.flags else UserType.HOST_USER) + # export a bunch of environment variables to help plugin developers + environ["HOME"] = get_home_path(UserType.ROOT if "root" in self.flags else UserType.HOST_USER) + environ["USER"] = "root" if "root" in self.flags else get_username() + environ["DECKY_VERSION"] = helpers.get_loader_version() + environ["DECKY_USER"] = get_username() + environ["DECKY_USER_HOME"] = helpers.get_home_path() + environ["DECKY_HOME"] = helpers.get_homebrew_path() + environ["DECKY_PLUGIN_SETTINGS_DIR"] = path.join(environ["DECKY_HOME"], "settings", self.plugin_directory) + helpers.mkdir_as_user(path.join(environ["DECKY_HOME"], "settings")) + helpers.mkdir_as_user(environ["DECKY_PLUGIN_SETTINGS_DIR"]) + environ["DECKY_PLUGIN_RUNTIME_DIR"] = path.join(environ["DECKY_HOME"], "data", self.plugin_directory) + helpers.mkdir_as_user(path.join(environ["DECKY_HOME"], "data")) + helpers.mkdir_as_user(environ["DECKY_PLUGIN_RUNTIME_DIR"]) + environ["DECKY_PLUGIN_LOG_DIR"] = path.join(environ["DECKY_HOME"], "logs", self.plugin_directory) + helpers.mkdir_as_user(path.join(environ["DECKY_HOME"], "logs")) + helpers.mkdir_as_user(environ["DECKY_PLUGIN_LOG_DIR"]) + environ["DECKY_PLUGIN_DIR"] = path.join(self.plugin_path, self.plugin_directory) + environ["DECKY_PLUGIN_NAME"] = self.name + if self.version: + environ["DECKY_PLUGIN_VERSION"] = self.version + environ["DECKY_PLUGIN_AUTHOR"] = self.author + + # append the plugin's `py_modules` to the recognized python paths + syspath.append(path.join(environ["DECKY_PLUGIN_DIR"], "py_modules")) + + #TODO: FIX IN A LESS CURSED WAY + keys = [key.replace("src.", "") for key in sysmodules if key.startswith("src.")] + for key in keys: + sysmodules[key] = sysmodules["src"].__dict__[key] + + spec = spec_from_file_location("_", self.file) + assert spec is not None + module = module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + self.Plugin = module.Plugin + + if hasattr(self.Plugin, "_migration"): + get_event_loop().run_until_complete(self.Plugin._migration(self.Plugin)) + if hasattr(self.Plugin, "_main"): + get_event_loop().create_task(self.Plugin._main(self.Plugin)) + get_event_loop().create_task(socket.setup_server()) + get_event_loop().run_forever() + except: + self.log.error("Failed to start " + self.name + "!\n" + format_exc()) + exit(0) + + async def _unload(self): + try: + self.log.info("Attempting to unload with plugin " + self.name + "'s \"_unload\" function.\n") + if hasattr(self.Plugin, "_unload"): + await self.Plugin._unload(self.Plugin) + self.log.info("Unloaded " + self.name + "\n") + else: + self.log.info("Could not find \"_unload\" in " + self.name + "'s main.py" + "\n") + except: + self.log.error("Failed to unload " + self.name + "!\n" + format_exc()) + exit(0) + + async def on_new_message(self, message : str) -> str|None: + data = loads(message) + + if "stop" in data: + self.log.info("Calling Loader unload function.") + await self._unload() + get_event_loop().stop() + while get_event_loop().is_running(): + await sleep(0) + get_event_loop().close() + raise Exception("Closing message listener") + + # TODO there is definitely a better way to type this + d: SocketResponseDict = {"res": None, "success": True, "id": data["id"]} + try: + d["res"] = await getattr(self.Plugin, data["method"])(self.Plugin, **data["args"]) + except Exception as e: + d["res"] = str(e) + d["success"] = False + finally: + return dumps(d, ensure_ascii=False) + + + def start(self, socket: LocalSocket): + if self.passive: + return self + multiprocessing.Process(target=self._init, args=[socket]).start() + return self \ No newline at end of file -- cgit v1.2.3