handle crashloops and disable decky for the user

8 files changed, 269 insertions, 53 deletions

diff --git a/backend/decky_loader/helpers.py b/backend/decky_loader/helpers.py
index 2e0fe45f..8ca77632 100644
--- a/backend/decky_loader/helpers.py
+++ b/backend/decky_loader/helpers.py
@@ -52,6 +52,9 @@ async def csrf_middleware(request: Request, handler: Handler):
         return await handler(request)
     return Response(text='Forbidden', status=403)
 
+def create_inject_script(script: str) -> str:
+    return "try{if (window.deckyHasLoaded){setTimeout(() => SteamClient.Browser.RestartJSContext(), 100)}else{window.deckyHasLoaded = true;(async()=>{try{await import('http://localhost:1337/frontend/%s?v=%s')}catch(e){console.error(e)};})();}}catch(e){console.error(e)}" % (script, get_loader_version(), )
+
 # Get the default homebrew path unless a home_path is specified. home_path argument is deprecated
 def get_homebrew_path() -> str:
     return localplatform.get_unprivileged_path()
diff --git a/backend/decky_loader/main.py b/backend/decky_loader/main.py
index 5033126e..c268b387 100644
--- a/backend/decky_loader/main.py
+++ b/backend/decky_loader/main.py
@@ -15,6 +15,7 @@ from asyncio import AbstractEventLoop, CancelledError, Task, all_tasks, current_
 from logging import basicConfig, getLogger
 from os import path
 from traceback import format_exc
+from time import time
 import aiohttp_cors # pyright: ignore [reportMissingTypeStubs]
 
 # Partial imports
@@ -25,7 +26,7 @@ from setproctitle import getproctitle, setproctitle, setthreadtitle
 
 # local modules
 from .browser import PluginBrowser
-from .helpers import (REMOTE_DEBUGGER_UNIT, csrf_middleware, get_csrf_token, get_loader_version,
+from .helpers import (REMOTE_DEBUGGER_UNIT, create_inject_script, csrf_middleware, get_csrf_token, get_loader_version,
                      mkdir_as_user, get_system_pythonpaths, get_effective_user_id)
                      
 from .injector import get_gamepadui_tab, Tab
@@ -75,6 +76,9 @@ class PluginManager:
         self.plugin_browser = PluginBrowser(plugin_path, self.plugin_loader.plugins, self.plugin_loader, self.settings) 
         self.utilities = Utilities(self)
         self.updater = Updater(self)
+        self.last_webhelper_exit: float = 0
+        self.webhelper_crash_count: int = 0
+        self.inject_fallback: bool = False
 
         jinja_setup(self.web_app)
 
@@ -96,6 +100,21 @@ class PluginManager:
             self.cors.add(route) # pyright: ignore [reportUnknownMemberType]
         self.web_app.add_routes([static("/static", path.join(path.dirname(__file__), 'static'))])
 
+    async def handle_crash(self):
+        new_time = time()
+        if (new_time - self.last_webhelper_exit < 60):
+            self.webhelper_crash_count += 1
+            logger.warn(f"webhelper crashed within a minute from last crash! crash count: {self.webhelper_crash_count}")
+        else:
+            self.webhelper_crash_count = 0
+        self.last_webhelper_exit = new_time
+
+        # should never happen
+        if (self.webhelper_crash_count > 4):
+            await self.updater.do_shutdown()
+            # Give up
+            exit(0)
+
     async def shutdown(self, _: Application):
         try:
             logger.info(f"Shutting down...")
@@ -187,6 +206,7 @@ class PluginManager:
                 # If this is a forceful disconnect the loop will just stop without any failure message. In this case, injector.py will handle this for us so we don't need to close the socket.
                 # This is because of https://github.com/aio-libs/aiohttp/blob/3ee7091b40a1bc58a8d7846e7878a77640e96996/aiohttp/client_ws.py#L321
                 logger.info("CEF has disconnected...")
+                await self.handle_crash()
                 # At this point the loop starts again and we connect to the freshly started Steam client once it is ready.
             except Exception:
                 if not self.reinject:
@@ -194,6 +214,7 @@ class PluginManager:
                 logger.error("Exception while reading page events " + format_exc())
                 await tab.close_websocket()
                 self.js_ctx_tab = None
+                await self.handle_crash()
                 pass
         # while True:
         #     await sleep(5)
@@ -211,7 +232,11 @@ class PluginManager:
                 await restart_webhelper()
                 await sleep(1) # To give CEF enough time to close down the websocket
                 return # We'll catch the next tab in the main loop
-            await tab.evaluate_js("try{if (window.deckyHasLoaded){setTimeout(() => SteamClient.Browser.RestartJSContext(), 100)}else{window.deckyHasLoaded = true;(async()=>{try{await import('http://localhost:1337/frontend/index.js?v=%s')}catch(e){console.error(e)};})();}}catch(e){console.error(e)}" % (get_loader_version(), ), False, False, False)
+            await tab.evaluate_js(create_inject_script("index.js" if self.webhelper_crash_count < 3 else "fallback.js"), False, False, False)
+            if self.webhelper_crash_count > 2:
+                self.reinject = False
+                await sleep(1)
+                await self.updater.do_shutdown()
         except:
             logger.info("Failed to inject JavaScript into tab\n" + format_exc())
             pass
diff --git a/backend/decky_loader/utilities.py b/backend/decky_loader/utilities.py
index 17226ebc..4962da32 100644
--- a/backend/decky_loader/utilities.py
+++ b/backend/decky_loader/utilities.py
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
     from .main import PluginManager
 from .injector import inject_to_tab, get_gamepadui_tab, close_old_tabs, get_tab
 from . import helpers
-from .localplatform.localplatform import ON_WINDOWS, service_stop, service_start, get_home_path, get_username, get_use_cef_close_workaround, close_cef_socket
+from .localplatform.localplatform import ON_WINDOWS, service_stop, service_start, get_home_path, get_username, get_use_cef_close_workaround, close_cef_socket, restart_webhelper
 
 class FilePickerObj(TypedDict):
     file: Path
@@ -77,6 +77,7 @@ class Utilities:
             context.ws.add_route("utilities/get_tab_id", self.get_tab_id)
             context.ws.add_route("utilities/get_user_info", self.get_user_info)
             context.ws.add_route("utilities/http_request", self.http_request_legacy)
+            context.ws.add_route("utilities/restart_webhelper", self.restart_webhelper)
             context.ws.add_route("utilities/close_cef_socket", self.close_cef_socket)
             context.ws.add_route("utilities/_call_legacy_utility", self._call_legacy_utility)
 
@@ -291,6 +292,9 @@ class Utilities:
         if get_use_cef_close_workaround():
             await close_cef_socket()
 
+    async def restart_webhelper(self):
+        await restart_webhelper()
+
     async def filepicker_ls(self, 
                             path: str | None = None, 
                             include_files: bool = True,
diff --git a/frontend/rollup.config.js b/frontend/rollup.config.js
index 2c731e54..57804c4f 100644
--- a/frontend/rollup.config.js
+++ b/frontend/rollup.config.js
@@ -11,48 +11,62 @@ import { visualizer } from 'rollup-plugin-visualizer';
 
 const hiddenWarnings = ['THIS_IS_UNDEFINED', 'EVAL'];
 
-export default defineConfig({
-  input: 'src/index.ts',
-  plugins: [
-    del({ targets: '../backend/decky_loader/static/*', force: true }),
-    commonjs(),
-    nodeResolve({
-      browser: true,
-    }),
-    externalGlobals({
-      react: 'SP_REACT',
-      'react-dom': 'SP_REACTDOM',
-      // hack to shut up react-markdown
-      process: '{cwd: () => {}}',
-      path: '{dirname: () => {}, join: () => {}, basename: () => {}, extname: () => {}}',
-      url: '{fileURLToPath: (f) => f}',
-    }),
-    typescript(),
-    json(),
-    replace({
-      preventAssignment: false,
-      'process.env.NODE_ENV': JSON.stringify('production'),
-    }),
-    image(),
-    visualizer(),
-  ],
-  preserveEntrySignatures: false,
-  treeshake: {
-    // Assume all external modules have imports with side effects (the default) while allowing decky libraries to treeshake
-    pureExternalImports: true,
-    preset: 'smallest'
-  },
-  output: {
-    dir: '../backend/decky_loader/static',
-    format: 'esm',
-    chunkFileNames: (chunkInfo) => {
-      return 'chunk-[hash].js';
+export default defineConfig([
+  // Main bundle
+  {
+    input: 'src/index.ts',
+    plugins: [
+      del({ targets: ['../backend/decky_loader/static/*', '!../backend/decky_loader/static/fallback.js'], force: true }),
+      commonjs(),
+      nodeResolve({
+        browser: true,
+      }),
+      externalGlobals({
+        react: 'SP_REACT',
+        'react-dom': 'SP_REACTDOM',
+        // hack to shut up react-markdown
+        process: '{cwd: () => {}}',
+        path: '{dirname: () => {}, join: () => {}, basename: () => {}, extname: () => {}}',
+        url: '{fileURLToPath: (f) => f}',
+      }),
+      typescript(),
+      json(),
+      replace({
+        preventAssignment: false,
+        'process.env.NODE_ENV': JSON.stringify('production'),
+      }),
+      image(),
+      visualizer(),
+    ],
+    preserveEntrySignatures: false,
+    treeshake: {
+      // Assume all external modules have imports with side effects (the default) while allowing decky libraries to treeshake
+      pureExternalImports: true,
+      preset: 'smallest'
+    },
+    output: {
+      dir: '../backend/decky_loader/static',
+      format: 'esm',
+      chunkFileNames: (chunkInfo) => {
+        return 'chunk-[hash].js';
+      },
+      sourcemap: true,
+      sourcemapPathTransform: (relativeSourcePath) => relativeSourcePath.replace(/^\.\.\//, `decky://decky/loader/`),
+    },
+    onwarn: function (message, handleWarning) {
+      if (hiddenWarnings.some((warning) => message.code === warning)) return;
+      handleWarning(message);
     },
-    sourcemap: true,
-    sourcemapPathTransform: (relativeSourcePath) => relativeSourcePath.replace(/^\.\.\//, `decky://decky/loader/`),
-  },
-  onwarn: function (message, handleWarning) {
-    if (hiddenWarnings.some((warning) => message.code === warning)) return;
-    handleWarning(message);
   },
-});
+  // Fallback
+  {
+    input: 'src/fallback.ts',
+    plugins: [
+      typescript()
+    ],
+    output: {
+      file: '../backend/decky_loader/static/fallback.js',
+      format: 'esm',
+    }
+  }
+]);
diff --git a/frontend/src/components/DeckyErrorBoundary.tsx b/frontend/src/components/DeckyErrorBoundary.tsx
index 152e0f09..654db8a0 100644
--- a/frontend/src/components/DeckyErrorBoundary.tsx
+++ b/frontend/src/components/DeckyErrorBoundary.tsx
@@ -94,7 +94,7 @@ const DeckyErrorBoundary: FunctionComponent<DeckyErrorBoundaryProps> = ({ error,
                 style={{ marginRight: '5px', padding: '5px' }}
                 onClick={() => {
                   addLogLine('Restarting Steam...');
-                  SteamClient.User.StartRestart();
+                  SteamClient.User.StartRestart(false);
                 }}
               >
                 Restart Steam
@@ -121,7 +121,7 @@ const DeckyErrorBoundary: FunctionComponent<DeckyErrorBoundaryProps> = ({ error,
                   doShutdown();
                   await sleep(5000);
                   addLogLine('Restarting Steam...');
-                  SteamClient.User.StartRestart();
+                  SteamClient.User.StartRestart(false);
                 }}
               >
                 Disable Decky until next boot
@@ -166,7 +166,7 @@ const DeckyErrorBoundary: FunctionComponent<DeckyErrorBoundaryProps> = ({ error,
                     await sleep(2000);
                     addLogLine('Restarting Steam...');
                     await sleep(500);
-                    SteamClient.User.StartRestart();
+                    SteamClient.User.StartRestart(false);
                   }}
                 >
                   Uninstall {errorSource} and restart Decky
diff --git a/frontend/src/fallback.ts b/frontend/src/fallback.ts
new file mode 100644
index 00000000..fc39b272
--- /dev/null
+++ b/frontend/src/fallback.ts
@@ -0,0 +1,128 @@
+// THIS FILE MUST BE ENTIRELY SELF-CONTAINED! DO NOT USE PACKAGES!
+interface Window {
+  FocusNavController: any;
+  GamepadNavTree: any;
+  deckyFallbackLoaded?: boolean;
+}
+
+(async () => {
+  try {
+    if (window.deckyFallbackLoaded) return;
+    window.deckyFallbackLoaded = true;
+
+    // #region utils
+    function sleep(ms: number) {
+      return new Promise((res) => setTimeout(res, ms));
+    }
+    // #endregion
+
+    // #region DeckyIcon
+    const fallbackIcon = `
+      <svg class="fallbackDeckyIcon" xmlns="http://www.w3.org/2000/svg" height="100%" width="100%" viewBox="0 0 512 456">
+        <g>
+          <path
+            style="fill: none;"
+            d="M154.33,72.51v49.79c11.78-0.17,23.48,2,34.42,6.39c10.93,4.39,20.89,10.91,29.28,19.18
+            c8.39,8.27,15.06,18.13,19.61,29c4.55,10.87,6.89,22.54,6.89,34.32c0,11.78-2.34,23.45-6.89,34.32
+            c-4.55,10.87-11.21,20.73-19.61,29c-8.39,8.27-18.35,14.79-29.28,19.18c-10.94,4.39-22.63,6.56-34.42,6.39v49.77
+            c36.78,0,72.05-14.61,98.05-40.62c26-26.01,40.61-61.28,40.61-98.05c0-36.78-14.61-72.05-40.61-98.05
+            C226.38,87.12,191.11,72.51,154.33,72.51z"
+          />
+
+          <ellipse
+            transform="matrix(0.982 -0.1891 0.1891 0.982 -37.1795 32.9988)"
+            style="fill: none;"
+            cx="154.33"
+            cy="211.33"
+            rx="69.33"
+            ry="69.33"
+          />
+          <path style="fill: none;" d="M430,97h-52v187h52c7.18,0,13-5.82,13-13V110C443,102.82,437.18,97,430,97z" />
+          <path
+            style="fill: currentColor;"
+            d="M432,27h-54V0H0v361c0,52.47,42.53,95,95,95h188c52.47,0,95-42.53,95-95v-7h54c44.18,0,80-35.82,80-80V107
+            C512,62.82,476.18,27,432,27z M85,211.33c0-38.29,31.04-69.33,69.33-69.33c38.29,0,69.33,31.04,69.33,69.33
+            c0,38.29-31.04,69.33-69.33,69.33C116.04,280.67,85,249.62,85,211.33z M252.39,309.23c-26.01,26-61.28,40.62-98.05,40.62v-49.77
+            c11.78,0.17,23.48-2,34.42-6.39c10.93-4.39,20.89-10.91,29.28-19.18c8.39-8.27,15.06-18.13,19.61-29
+            c4.55-10.87,6.89-22.53,6.89-34.32c0-11.78-2.34-23.45-6.89-34.32c-4.55-10.87-11.21-20.73-19.61-29
+            c-8.39-8.27-18.35-14.79-29.28-19.18c-10.94-4.39-22.63-6.56-34.42-6.39V72.51c36.78,0,72.05,14.61,98.05,40.61
+            c26,26.01,40.61,61.28,40.61,98.05C293,247.96,278.39,283.23,252.39,309.23z M443,271c0,7.18-5.82,13-13,13h-52V97h52
+            c7.18,0,13,5.82,13,13V271z"
+          />
+        </g>
+      </svg>
+    `;
+    // #endregion
+
+    // #region findSP
+    // from @decky/ui
+    function getFocusNavController(): any {
+      return window.GamepadNavTree?.m_context?.m_controller || window.FocusNavController;
+    }
+
+    function getGamepadNavigationTrees(): any {
+      const focusNav = getFocusNavController();
+      const context = focusNav.m_ActiveContext || focusNav.m_LastActiveContext;
+      return context?.m_rgGamepadNavigationTrees;
+    }
+
+    function findSP(): Window {
+      // old (SP as host)
+      if (document.title == 'SP') return window;
+      // new (SP as popup)
+      const navTrees = getGamepadNavigationTrees();
+      return navTrees?.find((x: any) => x.m_ID == 'root_1_').Root.Element.ownerDocument.defaultView;
+    }
+    // #endregion
+
+    const fallbackCSS = `
+      .fallbackContainer {
+        width: 100vw;
+        height: 100vh;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        text-align: center;
+        flex-direction: column;
+        z-index: 99999999;
+        pointer-events: none;
+        position: absolute;
+        top: 0;
+        left: 0;
+        backdrop-filter: blur(8px) brightness(40%);
+      }
+      .fallbackDeckyIcon {
+        width: 96px;
+        height: 96px;
+        padding-bottom: 1rem;
+      }
+    `;
+
+    const fallbackHTML = `
+      <style>${fallbackCSS}</style>
+      ${fallbackIcon}
+      <span class="fallbackText">
+        <b>A crash loop has been detected and Decky has been disabled for this boot.</b>
+        <br>
+        <i>Steam will restart in 10 seconds...</i>
+      </span>
+    `;
+
+    await sleep(4000);
+
+    const win = findSP() || window;
+
+    const container = Object.assign(document.createElement('div'), {
+      innerHTML: fallbackHTML,
+    });
+    container.classList.add('fallbackContainer');
+
+    win.document.body.appendChild(container);
+
+    await sleep(10000);
+
+    SteamClient.User.StartShutdown(false);
+  } catch (e) {
+    console.error('Error showing fallback!', e);
+  }
+})();
diff --git a/frontend/src/plugin-loader.tsx b/frontend/src/plugin-loader.tsx
index f03877fa..4e0a01e9 100644
--- a/frontend/src/plugin-loader.tsx
+++ b/frontend/src/plugin-loader.tsx
@@ -172,11 +172,32 @@ class PluginLoader extends Logger {
       .then(() => this.log('Initialized'));
   }
 
+  private checkForSP(): boolean {
+    try {
+      return !!findSP();
+    } catch (e) {
+      this.warn('Error checking for SP tab', e);
+      return false;
+    }
+  }
+
+  private async runCrashChecker() {
+    const spExists = this.checkForSP();
+    await sleep(5000);
+    if (spExists && !this.checkForSP()) {
+      // SP died after plugin loaded. Give up and let the loader's crash loop detection handle it.
+      this.error('SP died during startup. Restarting webhelper.');
+      await this.restartWebhelper();
+    }
+  }
+
   private getPluginsFromBackend = DeckyBackend.callable<
     [],
     { name: string; version: string; load_type: PluginLoadType }[]
   >('loader/get_plugins');
 
+  private restartWebhelper = DeckyBackend.callable<[], void>('utilities/restart_webhelper');
+
   private async loadPlugins() {
     let registration: any;
     const uiMode = await new Promise(
@@ -192,6 +213,7 @@ class PluginLoader extends Logger {
         await sleep(100);
       }
     }
+    this.runCrashChecker();
     const plugins = await this.getPluginsFromBackend();
     const pluginLoadPromises = [];
     const loadStart = performance.now();
@@ -395,6 +417,7 @@ class PluginLoader extends Logger {
     version?: string,
     loadType: PluginLoadType = PluginLoadType.ESMODULE_V1,
   ) {
+    let spExists = this.checkForSP();
     try {
       switch (loadType) {
         case PluginLoadType.ESMODULE_V1:
@@ -442,7 +465,7 @@ class PluginLoader extends Logger {
           </PanelSectionRow>
           <PanelSectionRow>
             <pre style={{ overflowX: 'scroll' }}>
-              <code>{e instanceof Error ? e.stack : JSON.stringify(e)}</code>
+              <code>{e instanceof Error ? '' + e.stack : JSON.stringify(e)}</code>
             </pre>
           </PanelSectionRow>
           <PanelSectionRow>
@@ -474,6 +497,12 @@ class PluginLoader extends Logger {
         icon: <FaExclamationCircle />,
       });
     }
+
+    if (spExists && !this.checkForSP()) {
+      // SP died after plugin loaded. Give up and let the loader's crash loop detection handle it.
+      this.error('SP died after loading plugin. Restarting webhelper.');
+      await this.restartWebhelper();
+    }
   }
 
   async callServerMethod(methodName: string, args = {}) {
diff --git a/frontend/src/toaster.tsx b/frontend/src/toaster.tsx
index e45b14a4..4f67c589 100644
--- a/frontend/src/toaster.tsx
+++ b/frontend/src/toaster.tsx
@@ -1,5 +1,13 @@
 import type { ToastData, ToastNotification } from '@decky/api';
-import { Patch, callOriginal, findModuleExport, injectFCTrampoline, replacePatch } from '@decky/ui';
+import {
+  ErrorBoundary,
+  Patch,
+  callOriginal,
+  findModuleExport,
+  injectFCTrampoline,
+  replacePatch,
+  sleep,
+} from '@decky/ui';
 
 import Toast from './components/Toast';
 import Logger from './logger';
@@ -21,6 +29,8 @@ declare global {
 
 class Toaster extends Logger {
   private toastPatch?: Patch;
+  private markReady!: () => void;
+  private ready = new Promise<void>((r) => (this.markReady = r));
 
   constructor() {
     super('Toaster');
@@ -34,13 +44,16 @@ class Toaster extends Logger {
     this.toastPatch = replacePatch(patchedRenderer, 'component', (args: any[]) => {
       if (args?.[0]?.group?.decky || args?.[0]?.group?.notifications?.[0]?.decky) {
         return args[0].group.notifications.map((notification: any) => (
-          <Toast toast={notification.data} newIndicator={notification.bNewIndicator} location={args?.[0]?.location} />
+          <ErrorBoundary>
+            <Toast toast={notification.data} newIndicator={notification.bNewIndicator} location={args?.[0]?.location} />
+          </ErrorBoundary>
         ));
       }
       return callOriginal;
     });
 
     this.log('Initialized');
+    sleep(4000).then(this.markReady);
   }
 
   toast(toast: ToastData): ToastNotification {
@@ -107,7 +120,7 @@ class Toaster extends Logger {
         }
       }, toast.expiration);
     }
-    window.NotificationStore.ProcessNotification(info, toastData, ToastType.New);
+    this.ready.then(() => window.NotificationStore.ProcessNotification(info, toastData, ToastType.New));
     return toastResult;
   }