Implement legacy & modern plugin method calls over WS

This version builds fine and runs all of the 14 plugins I have installed perfectly, so we're really close to having this done.

10 files changed, 106 insertions, 60 deletions

diff --git a/backend/decky_loader/helpers.py b/backend/decky_loader/helpers.py
index 0cd6518b..76c3db0a 100644
--- a/backend/decky_loader/helpers.py
+++ b/backend/decky_loader/helpers.py
@@ -34,7 +34,7 @@ def get_csrf_token():
 
 @middleware
 async def csrf_middleware(request: Request, handler: Handler):
-    if str(request.method) == "OPTIONS" or request.headers.get('Authentication') == csrf_token or str(request.rel_url) == "/auth/token" or str(request.rel_url).startswith("/plugins/load_main/") or str(request.rel_url).startswith("/static/") or str(request.rel_url).startswith("/steam_resource/") or str(request.rel_url).startswith("/frontend/") or assets_regex.match(str(request.rel_url)) or frontend_regex.match(str(request.rel_url)):
+    if str(request.method) == "OPTIONS" or request.headers.get('Authentication') == csrf_token or str(request.rel_url) == "/auth/token" or str(request.rel_url).startswith("/plugins/load_main/") or str(request.rel_url).startswith("/static/") or str(request.rel_url).startswith("/steam_resource/") or str(request.rel_url).startswith("/frontend/") or str(request.rel_url.path) == "/ws" or assets_regex.match(str(request.rel_url)) or frontend_regex.match(str(request.rel_url)):
         return await handler(request)
     return Response(text='Forbidden', status=403)
 
diff --git a/backend/decky_loader/loader.py b/backend/decky_loader/loader.py
index e5a5cc7b..13e54289 100644
--- a/backend/decky_loader/loader.py
+++ b/backend/decky_loader/loader.py
@@ -4,15 +4,15 @@ from json.decoder import JSONDecodeError
 from logging import getLogger
 from os import listdir, path
 from pathlib import Path
-from traceback import print_exc
-from typing import Any, Tuple
+from traceback import format_exc, print_exc
+from typing import Any, Tuple, Dict
 
 from aiohttp import web
 from os.path import exists
 from watchdog.events import RegexMatchingEventHandler, DirCreatedEvent, DirModifiedEvent, FileCreatedEvent, FileModifiedEvent # type: ignore
 from watchdog.observers import Observer # type: ignore
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, List
 if TYPE_CHECKING:
     from .main import PluginManager
 
@@ -92,13 +92,15 @@ class Loader:
         server_instance.web_app.add_routes([
             web.get("/frontend/{path:.*}", self.handle_frontend_assets),
             web.get("/locales/{path:.*}", self.handle_frontend_locales),
-            web.get("/plugins", self.get_plugins),
             web.get("/plugins/{plugin_name}/frontend_bundle", self.handle_frontend_bundle),
-            web.post("/plugins/{plugin_name}/methods/{method_name}", self.handle_plugin_method_call),
             web.get("/plugins/{plugin_name}/assets/{path:.*}", self.handle_plugin_frontend_assets),
-            web.post("/plugins/{plugin_name}/reload", self.handle_backend_reload_request)
         ])
 
+        server_instance.ws.add_route("loader/get_plugins", self.get_plugins)
+        server_instance.ws.add_route("loader/reload_plugin", self.handle_plugin_backend_reload)
+        server_instance.ws.add_route("loader/call_plugin_method", self.handle_plugin_method_call)
+        server_instance.ws.add_route("loader/call_legacy_plugin_method", self.handle_plugin_method_call_legacy)
+
     async def enable_reload_wait(self):
         if self.live_reload:
             await sleep(10)
@@ -119,9 +121,9 @@ class Loader:
             self.logger.info(f"Language {req_lang} not available, returning an empty dictionary")
             return web.json_response(data={}, headers={"Cache-Control": "no-cache"})
 
-    async def get_plugins(self, request: web.Request):
+    async def get_plugins(self):
         plugins = list(self.plugins.values())
-        return web.json_response([{"name": str(i), "version": i.version} for i in plugins])
+        return [{"name": str(i), "version": i.version} for i in plugins]
 
     async def handle_plugin_frontend_assets(self, request: web.Request):
         plugin = self.plugins[request.match_info["plugin_name"]]
@@ -173,29 +175,31 @@ class Loader:
             args = await self.reload_queue.get()
             self.import_plugin(*args) # type: ignore
 
-    async def handle_plugin_method_call(self, request: web.Request):
-        res = {}
-        plugin = self.plugins[request.match_info["plugin_name"]]
-        method_name = request.match_info["method_name"]
-        try:
-            method_info = await request.json()
-            args: Any = method_info["args"]
-        except JSONDecodeError:
-            args = {}
+    async def handle_plugin_method_call_legacy(self, plugin_name: str, method_name: str, kwargs: Dict[Any, Any]):
+        res: Dict[Any, Any] = {}
+        plugin = self.plugins[plugin_name]
         try:
           if method_name.startswith("_"):
-              raise RuntimeError("Tried to call private method")
-          res["result"] = await plugin.execute_method(method_name, args)
+              raise RuntimeError(f"Plugin {plugin.name} tried to call private method {method_name}")
+          res["result"] = await plugin.execute_legacy_method(method_name, kwargs)
           res["success"] = True
         except Exception as e:
             res["result"] = str(e)
             res["success"] = False
-        return web.json_response(res)
+        return res
 
-    async def handle_backend_reload_request(self, request: web.Request):
-        plugin_name : str = request.match_info["plugin_name"]
+    async def handle_plugin_method_call(self, plugin_name: str, method_name: str, *args: List[Any]):
         plugin = self.plugins[plugin_name]
+        try:
+          if method_name.startswith("_"):
+              raise RuntimeError(f"Plugin {plugin.name} tried to call private method {method_name}")
+          result = await plugin.execute_method(method_name, *args)
+        except Exception as e:
+            self.logger.error(f"Method {method_name} of plugin {plugin.name} failed with the following exception:\n{format_exc()}")
+            raise e # throw again to pass the error to the frontend
+        return result
 
-        await self.reload_queue.put((plugin.file, plugin.plugin_directory))
+    async def handle_plugin_backend_reload(self, plugin_name: str):
+        plugin = self.plugins[plugin_name]
 
-        return web.Response(status=200)
\ No newline at end of file
+        await self.reload_queue.put((plugin.file, plugin.plugin_directory))
\ No newline at end of file
diff --git a/backend/decky_loader/main.py b/backend/decky_loader/main.py
index 71bc75f4..bf34ad17 100644
--- a/backend/decky_loader/main.py
+++ b/backend/decky_loader/main.py
@@ -1,7 +1,6 @@
 # Change PyInstaller files permissions
 import sys
 from typing import Dict
-from wsrouter import WSRouter
 from .localplatform.localplatform import (chmod, chown, service_stop, service_start,
                             ON_WINDOWS, get_log_level, get_live_reload, 
                             get_server_port, get_server_host, get_chown_plugin_path,
@@ -32,6 +31,7 @@ from .settings import SettingsManager
 from .updater import Updater
 from .utilities import Utilities
 from .customtypes import UserType
+from .wsrouter import WSRouter
 
 
 basicConfig(
@@ -102,7 +102,6 @@ class PluginManager:
         # await self.wait_for_server()
         logger.debug("Loading plugins")
         self.plugin_loader.import_plugins()
-        # await inject_to_tab("SP", "window.syncDeckyPlugins();")
         if self.settings.getSetting("pluginOrder", None) == None:
           self.settings.setSetting("pluginOrder", list(self.plugin_loader.plugins.keys()))
           logger.debug("Did not find pluginOrder setting, set it to default")
diff --git a/backend/decky_plugin.py b/backend/decky_loader/plugin/imports/decky.py
index 35b872a0..599d142f 100644
--- a/backend/decky_plugin.py
+++ b/backend/decky_loader/plugin/imports/decky.py
@@ -19,6 +19,8 @@ import subprocess
 import logging
 import time
 
+from typing import Dict, Any
+
 """
 Constants
 """
@@ -207,3 +209,13 @@ logger: logging.Logger = logging.getLogger()
 """The main plugin logger writing to `DECKY_PLUGIN_LOG`."""
 
 logger.setLevel(logging.INFO)
+
+"""
+Event handling
+"""
+# TODO better docstring im lazy
+async def emit_message(message: Dict[Any, Any]) -> None:
+    """
+    Send a message to the frontend.
+    """
+    pass
\ No newline at end of file
diff --git a/backend/decky_plugin.pyi b/backend/decky_loader/plugin/imports/decky.pyi
index b311a55a..e68b3853 100644
--- a/backend/decky_plugin.pyi
+++ b/backend/decky_loader/plugin/imports/decky.pyi
@@ -16,6 +16,8 @@ __version__ = '0.1.0'
 
 import logging
 
+from typing import Dict, Any
+
 """
 Constants
 """
@@ -171,3 +173,12 @@ Logging
 
 logger: logging.Logger
 """The main plugin logger writing to `DECKY_PLUGIN_LOG`."""
+
+"""
+Event handling
+"""
+# TODO better docstring im lazy
+async def emit_message(message: Dict[Any, Any]) -> None:
+    """
+    Send a message to the frontend.
+    """
\ No newline at end of file
diff --git a/backend/decky_loader/plugin/plugin.py b/backend/decky_loader/plugin/plugin.py
index 6c338106..01fc048c 100644
--- a/backend/decky_loader/plugin/plugin.py
+++ b/backend/decky_loader/plugin/plugin.py
@@ -4,11 +4,12 @@ from logging import getLogger
 from os import path
 from multiprocessing import Process
 
+
 from .sandboxed_plugin import SandboxedPlugin
 from .method_call_request import MethodCallRequest
 from ..localplatform.localsocket import LocalSocket
 
-from typing import Any, Callable, Coroutine, Dict
+from typing import Any, Callable, Coroutine, Dict, List
 
 class PluginWrapper:
     def __init__(self, file: str, plugin_directory: str, plugin_path: str) -> None:
@@ -39,6 +40,8 @@ class PluginWrapper:
 
         self.emitted_message_callback: Callable[[Dict[Any, Any]], Coroutine[Any, Any, Any]]
 
+        self.legacy_method_warning = False
+
     def __str__(self) -> str:
         return self.name
     
@@ -58,13 +61,27 @@ class PluginWrapper:
     def set_emitted_message_callback(self, callback: Callable[[Dict[Any, Any]], Coroutine[Any, Any, Any]]):
         self.emitted_message_callback = callback
 
-    async def execute_method(self, method_name: str, kwargs: Dict[Any, Any]):
+    async def execute_legacy_method(self, method_name: str, kwargs: Dict[Any, Any]):
+        if not self.legacy_method_warning:
+            self.legacy_method_warning = True
+            self.log.warn(f"Plugin {self.name} is using legacy method calls. This will be removed in a future release.")
+        if self.passive:
+            raise RuntimeError("This plugin is passive (aka does not implement main.py)")
+        
+        request = MethodCallRequest()
+        await self._socket.get_socket_connection()
+        await self._socket.write_single_line(dumps({ "method": method_name, "args": kwargs, "id": request.id, "legacy": True }, ensure_ascii=False))
+        self._method_call_requests[request.id] = request
+
+        return await request.wait_for_result()
+
+    async def execute_method(self, method_name: str, args: List[Any]):
         if self.passive:
             raise RuntimeError("This plugin is passive (aka does not implement main.py)")
         
         request = MethodCallRequest()
         await self._socket.get_socket_connection()
-        await self._socket.write_single_line(dumps({ "method": method_name, "args": kwargs, "id": request.id }, ensure_ascii=False))
+        await self._socket.write_single_line(dumps({ "method": method_name, "args": args, "id": request.id }, ensure_ascii=False))
         self._method_call_requests[request.id] = request
 
         return await request.wait_for_result()
diff --git a/backend/decky_loader/plugin/sandboxed_plugin.py b/backend/decky_loader/plugin/sandboxed_plugin.py
index 1c3c5bd2..d07cdc97 100644
--- a/backend/decky_loader/plugin/sandboxed_plugin.py
+++ b/backend/decky_loader/plugin/sandboxed_plugin.py
@@ -77,17 +77,28 @@ class SandboxedPlugin:
             keys = [key for key in sysmodules if key.startswith("decky_loader.")]
             for key in keys:
                 sysmodules[key.replace("decky_loader.", "")] = sysmodules[key]
+            
+            from .imports import decky
+            async def emit_message(message: Dict[Any, Any]):
+                await self._socket.write_single_line_server(dumps({
+                    "id": "0",
+                    "payload": message
+                }))
+            # copy the docstring over so we don't have to duplicate it
+            emit_message.__doc__ = decky.emit_message.__doc__
+            decky.emit_message = emit_message
+            sysmodules["decky"] = decky
+            # provided for compatibility
+            sysmodules["decky_plugin"] = decky
 
             spec = spec_from_file_location("_", self.file)
             assert spec is not None
             module = module_from_spec(spec)
             assert spec.loader is not None
             spec.loader.exec_module(module)
+            # TODO fix self weirdness once plugin.json versioning is done. need this before WS release!
             self.Plugin = module.Plugin
 
-            setattr(self.Plugin, "emit_message", self.emit_message)
-            #TODO: Find how to put emit_message on global namespace so it doesn't pollute Plugin
-
             if hasattr(self.Plugin, "_migration"):
                 get_event_loop().run_until_complete(self.Plugin._migration(self.Plugin))
             if hasattr(self.Plugin, "_main"):
@@ -124,15 +135,14 @@ class SandboxedPlugin:
 
         d: SocketResponseDict = {"res": None, "success": True, "id": data["id"]}
         try:
-            d["res"] = await getattr(self.Plugin, data["method"])(self.Plugin, **data["args"])
+            if data["legacy"]:
+                # Legacy kwargs
+                d["res"] = await getattr(self.Plugin, data["method"])(self.Plugin, **data["args"])
+            else:
+                # New args
+                d["res"] = await getattr(self.Plugin, data["method"])(self.Plugin, *data["args"])
         except Exception as e:
             d["res"] = str(e)
             d["success"] = False
         finally:
-            return dumps(d, ensure_ascii=False)
-        
-    async def emit_message(self, message: Dict[Any, Any]):
-        await self._socket.write_single_line_server(dumps({
-            "id": "0",
-            "payload": message
-        }))
\ No newline at end of file
+            return dumps(d, ensure_ascii=False)
\ No newline at end of file
diff --git a/backend/decky_loader/utilities.py b/backend/decky_loader/utilities.py
index 12dabd68..774f47db 100644
--- a/backend/decky_loader/utilities.py
+++ b/backend/decky_loader/utilities.py
@@ -59,10 +59,6 @@ class Utilities:
         self.rdt_proxy_task = None
 
         if context:
-            context.web_app.add_routes([
-                web.post("/methods/{method_name}", self._handle_server_method_call)
-            ])
-
             context.ws.add_route("utilities/ping", self.ping)
             context.ws.add_route("utilities/settings/get", self.get_setting)
             context.ws.add_route("utilities/settings/set", self.set_setting)
@@ -81,22 +77,20 @@ class Utilities:
             context.ws.add_route("utilities/enable_rdt", self.enable_rdt)
             context.ws.add_route("utilities/get_tab_id", self.get_tab_id)
             context.ws.add_route("utilities/get_user_info", self.get_user_info)
+            context.ws.add_route("utilities/http_request", self.http_request)
+            context.ws.add_route("utilities/_call_legacy_utility", self._call_legacy_utility)
 
-    async def _handle_server_method_call(self, request: web.Request):
-        method_name = request.match_info["method_name"]
-        try:
-            args = await request.json()
-        except JSONDecodeError:
-            args = {}
-        res = {}
+    async def _call_legacy_utility(self, method_name: str, kwargs: Dict[Any, Any]) -> Any:
+        self.logger.debug(f"Calling utility {method_name} with legacy kwargs");
+        res: Dict[Any, Any] = {}
         try:
-            r = await self.util_methods[method_name](**args)
+            r = await self.util_methods[method_name](**kwargs)
             res["result"] = r
             res["success"] = True
         except Exception as e:
             res["result"] = str(e)
             res["success"] = False
-        return web.json_response(res)
+        return res
 
     async def install_plugin(self, artifact: str="", name: str="No name", version: str="dev", hash: str="", install_type: PluginInstallType=PluginInstallType.INSTALL):
         return await self.context.plugin_browser.request_plugin_install(
@@ -121,9 +115,9 @@ class Utilities:
     async def uninstall_plugin(self, name: str):
         return await self.context.plugin_browser.uninstall_plugin(name)
 
-    async def http_request(self, method: str="", url: str="", **kwargs: Any):
+    async def http_request(self, method: str, url: str, extra_opts: Any = {}):
         async with ClientSession() as web:
-            res = await web.request(method, url, ssl=helpers.get_ssl_context(), **kwargs)
+            res = await web.request(method, url, ssl=helpers.get_ssl_context(), **extra_opts)
             text = await res.text()
         return {
             "status": res.status,
diff --git a/backend/decky_loader/wsrouter.py b/backend/decky_loader/wsrouter.py
index 034667ae..cdfe982c 100644
--- a/backend/decky_loader/wsrouter.py
+++ b/backend/decky_loader/wsrouter.py
@@ -12,7 +12,7 @@ from dataclasses import dataclass
 
 from traceback import format_exc
 
-from helpers import get_csrf_token
+from .helpers import get_csrf_token
 
 class MessageType(IntEnum):
     ERROR = -1
@@ -43,7 +43,7 @@ Route = Callable[..., Coroutine[Any, Any, Any]]
 class WSRouter:
     def __init__(self, loop: AbstractEventLoop, server_instance: Application) -> None:
         self.loop = loop
-        self.ws: WebSocketResponse | None
+        self.ws: WebSocketResponse | None = None
         self.instance_id = 0
         self.routes: Dict[str, Route]  = {}
         # self.subscriptions: Dict[str, Callable[[Any]]] = {}
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 9fbd2389..5c8ef169 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -6,8 +6,7 @@ license = "GPLv2"
 authors = []
 packages = [
     {include = "decky_loader"},
-    {include = "decky_plugin.py"},
-    {include = "decky_plugin.pyi"},
+    {include = "decky_loader/main.py"}
 ]
 include = ["decky_loader/static/*"]