feat: optimize Apple NPU backend with safety checker detection, dynamic CFG enforcement, and improved model loading feedback

scripts/server/serve.cjs

@@ -371,6 +371,96 @@ let currentSettings = {
   height: 512,
 };
 
+const BACKEND_RESTART_SETTING_KEYS = [
+  "model",
+  "steps",
+  "cfgScale",
+  "sampler",
+  "threads",
+  "useGpu",
+  "backendType",
+  "vaeTiling",
+  "vaeOnCpu",
+  "flashAttn",
+  "width",
+  "height",
+];
+
+function normalizeBackendSettingValue(key, value) {
+  if (value === undefined || value === null) return value;
+  if (key === "model") return path.resolve(String(value));
+  if (["steps", "threads", "width", "height"].includes(key)) return parseInt(value);
+  if (key === "cfgScale") return Math.round(parseFloat(value) * 1000) / 1000;
+  if (["useGpu", "vaeTiling", "vaeOnCpu", "flashAttn"].includes(key)) return Boolean(value);
+  return String(value);
+}
+
+function backendSettingsMatch(current, requested) {
+  return BACKEND_RESTART_SETTING_KEYS.every((key) => (
+    normalizeBackendSettingValue(key, current[key]) === normalizeBackendSettingValue(key, requested[key])
+  ));
+}
+
+function appleNpuRuntimeMatches(current, requested) {
+  return normalizeBackendSettingValue("model", current.model) === normalizeBackendSettingValue("model", requested.model) &&
+    current.backendType === "apple-npu" &&
+    requested.backendType === "apple-npu" &&
+    normalizeBackendSettingValue("useGpu", current.useGpu) === normalizeBackendSettingValue("useGpu", requested.useGpu) &&
+    normalizeBackendSettingValue("width", current.width || 512) === normalizeBackendSettingValue("width", requested.width || 512) &&
+    normalizeBackendSettingValue("height", current.height || 512) === normalizeBackendSettingValue("height", requested.height || 512);
+}
+
+function setBackendLoadStage(phase, progress, extra = {}) {
+  if (backendReady) return;
+  backendLoadState = {
+    ...backendLoadState,
+    active: true,
+    phase,
+    progress: Math.max(backendLoadState.progress || 0, Math.min(99, progress)),
+    ...extra,
+  };
+}
+
+function updateCoreMLLoadProgress(output) {
+  if (currentSettings.backendType !== "apple-npu" || backendReady) return;
+  const cleanOutput = stripAnsi(output);
+
+  if (cleanOutput.includes("Loading PyTorch reference configuration")) {
+    setBackendLoadStage("Loading PyTorch reference configuration...", 8);
+  }
+
+  const pipelineMatch = cleanOutput.match(/Loading pipeline components.*?(\d+)%/);
+  if (pipelineMatch) {
+    const componentProgress = Math.max(0, Math.min(100, Number(pipelineMatch[1]) || 0));
+    setBackendLoadStage("Loading pipeline components...", 10 + Math.round(componentProgress * 0.25));
+  }
+
+  if (cleanOutput.includes("Loading Core ML models from")) {
+    setBackendLoadStage("Loading Core ML model bundle...", 40);
+  }
+  if (cleanOutput.includes("Loading text_encoder")) {
+    setBackendLoadStage("Loading text encoder...", 48);
+  }
+  if (cleanOutput.includes("Loading unet")) {
+    setBackendLoadStage("Loading UNet neural engine model...", 62);
+  }
+  if (cleanOutput.includes("Loading vae_decoder")) {
+    setBackendLoadStage("Loading VAE decoder...", 78);
+  }
+  if (cleanOutput.includes("Loading safety_checker")) {
+    setBackendLoadStage("Loading safety checker...", 88);
+  }
+  if (cleanOutput.includes("Initializing Core ML pipe")) {
+    setBackendLoadStage("Initializing Core ML pipeline...", 94);
+  }
+  if (cleanOutput.includes("Stable Diffusion configured")) {
+    setBackendLoadStage("Configuring image resolution...", 97);
+  }
+  if (cleanOutput.includes("Core ML models loaded successfully")) {
+    setBackendLoadStage("Core ML models loaded successfully.", 99);
+  }
+}
+
 let lastCpuSample = null;
 let cachedGpuInfo = null;
 let cachedBackendOptions = null;
@@ -3640,6 +3730,28 @@ function startBackendReadyPoll() {
   }, 500);
 }
 
+function startCoreMLLoadHeartbeat(proc) {
+  if (currentSettings.backendType !== "apple-npu") return null;
+  const startedAt = Date.now();
+  const interval = setInterval(() => {
+    if (backendProc !== proc || backendReady || !backendLoadState.active) {
+      clearInterval(interval);
+      return;
+    }
+    const elapsedSec = (Date.now() - startedAt) / 1000;
+    const targetProgress = Math.min(96, Math.round(8 + 88 * (1 - Math.exp(-elapsedSec / 55))));
+    if (targetProgress > (backendLoadState.progress || 0)) {
+      backendLoadState = {
+        ...backendLoadState,
+        active: true,
+        phase: backendLoadState.phase || "Loading Core ML model...",
+        progress: targetProgress,
+      };
+    }
+  }, 1000);
+  return interval;
+}
+
 function getDefaultModel() {
   try {
     const files = fs.readdirSync(MODELS).filter(isModelFile);
@@ -4128,7 +4240,7 @@ async function startBackend(settings = {}) {
     throw new Error(modelLoadIssue);
   }
 
-  PORT_BACKEND = await findAvailableBackendPort();
+  PORT_BACKEND = Number(settings.backendPort) > 0 ? Number(settings.backendPort) : await findAvailableBackendPort();
 
   const resolvedBackendType = resolveBackendType(currentSettings.useGpu, currentSettings.backendType, currentSettings.model);
   currentSettings.backendType = resolvedBackendType;
@@ -4260,10 +4372,12 @@ async function startBackend(settings = {}) {
   const proc = spawn(backendPath, args, { stdio: "pipe", env: spawnEnv });
   backendProc = proc;
   startBackendReadyPoll();
+  const coremlLoadHeartbeat = startCoreMLLoadHeartbeat(proc);
 
   proc.stdout.on("data", d => {
     const output = d.toString();
     process.stdout.write("  [sd] " + output);
+    updateCoreMLLoadProgress(output);
     const cleanOutput = stripAnsi(output);
     if (cleanOutput.includes("listening on")) {
       markBackendReady();
@@ -4344,6 +4458,7 @@ async function startBackend(settings = {}) {
   proc.stderr.on("data", d => {
     const output = d.toString();
     process.stderr.write("  [sd-err] " + output);
+    updateCoreMLLoadProgress(output);
     const cleanOutput = stripAnsi(output);
     const deviceMatch = cleanOutput.match(/Device\s+\d+:\s*([^,\r\n]+)/);
     if (deviceMatch) {
@@ -4381,6 +4496,7 @@ async function startBackend(settings = {}) {
     }
   });
   proc.on("exit", (code, signal) => {
+    if (coremlLoadHeartbeat) clearInterval(coremlLoadHeartbeat);
     if (backendProc !== proc) {
       console.log("  [backend] stale process exited with code", code, signal ? `(signal ${signal})` : "", `(process ${procSeq})`);
       return;
@@ -6283,6 +6399,15 @@ async function getLlmfitRecommendations(useCase = "chat", limit = 10) {
     try {
       const targetModel = newSettings.model || currentSettings.model || getDefaultModel();
       assertNoOtherActiveRuntime("image", targetModel);
+      const requestedSettings = { ...currentSettings, ...newSettings, model: targetModel };
+      const imageBackendReady = (backendReady && backendProc) || (openvinoReady && openvinoProc);
+      if (imageBackendReady && backendSettingsMatch(currentSettings, requestedSettings)) {
+        return json(res, 200, { ok: true, message: "Backend already running with requested settings.", settings: currentSettings, port: PORT_BACKEND });
+      }
+      if (backendReady && backendProc && appleNpuRuntimeMatches(currentSettings, requestedSettings)) {
+        currentSettings = requestedSettings;
+        return json(res, 200, { ok: true, message: "Apple NPU backend already running; updated generation defaults.", settings: currentSettings, port: PORT_BACKEND });
+      }
       await killBackend();
       await new Promise(r => setTimeout(r, 500));
       if (newSettings.backendType === "openvino-npu") {
@@ -6297,6 +6422,34 @@ async function getLlmfitRecommendations(useCase = "chat", limit = 10) {
         });
         return json(res, 200, { ok: true, message: "OpenVINO NPU backend starting...", settings: currentSettings, port: PORT_BACKEND });
       }
+      if (newSettings.backendType === "apple-npu") {
+        const backendPort = await findAvailableBackendPort();
+        PORT_BACKEND = backendPort;
+        newSettings.backendPort = backendPort;
+        currentSettings = { ...currentSettings, ...newSettings };
+        backendLoadState = {
+          active: true,
+          phase: "Starting Apple NPU backend...",
+          progress: 1,
+          current: 0,
+          total: 0,
+          speed: "",
+          model: path.basename(newSettings.model || targetModel),
+          backendMode: "Apple NPU",
+          backendBinary: path.basename(getCoreMLPythonPath()),
+          device: "",
+        };
+        startBackend(newSettings).catch((err) => {
+          backendError = err.message || String(err);
+          backendLoadState = {
+            ...backendLoadState,
+            active: false,
+            phase: "Apple NPU model load failed",
+          };
+          console.error("  [coreml-npu] Startup failed:", backendError);
+        });
+        return json(res, 200, { ok: true, message: "Apple NPU backend starting...", settings: { ...currentSettings, ...newSettings }, port: backendPort });
+      }
       await startBackend(newSettings);
       return json(res, 200, { ok: true, message: "Backend restarting...", settings: currentSettings, port: PORT_BACKEND });
     } catch (err) {

scripts/setup/setup-whisper.sh

@@ -44,6 +44,47 @@ download_and_extract() {
   fi
 }
 
+install_macos_whisper_from_homebrew() {
+  local cpu_dest="$APP_DIR/speech-backend/mac/cpu"
+  local lib_dest="$APP_DIR/speech-backend/mac/lib"
+  local brew_bin
+  brew_bin="$(command -v brew || true)"
+  if [[ -z "$brew_bin" ]]; then
+    return 1
+  fi
+
+  if ! "$brew_bin" list whisper-cpp >/dev/null 2>&1; then
+    echo "   >>   Installing whisper-cpp with Homebrew..."
+    "$brew_bin" install whisper-cpp
+  fi
+
+  local whisper_prefix
+  whisper_prefix="$("$brew_bin" --prefix whisper-cpp)"
+  if [[ ! -x "$whisper_prefix/bin/whisper-cli" ]]; then
+    echo "   XX   Homebrew whisper-cpp did not provide whisper-cli at $whisper_prefix/bin/whisper-cli" >&2
+    return 1
+  fi
+
+  mkdir -p "$cpu_dest" "$lib_dest"
+  cp "$whisper_prefix/bin/whisper-cli" "$cpu_dest/whisper-cli"
+  if [[ -x "$whisper_prefix/bin/whisper-server" ]]; then
+    cp "$whisper_prefix/bin/whisper-server" "$cpu_dest/whisper-server"
+  fi
+  chmod +x "$cpu_dest/whisper-cli" "$cpu_dest/whisper-server" 2>/dev/null || true
+
+  # Homebrew's whisper-cli uses @rpath/libwhisper.1.dylib. The app launcher runs
+  # it from app/speech-backend/mac/cpu, where ../lib is already in its rpath.
+  cp -P "$whisper_prefix"/lib/libwhisper*.dylib "$lib_dest/" 2>/dev/null || true
+
+  if "$cpu_dest/whisper-cli" --help >/dev/null 2>&1; then
+    echo "   OK   installed macOS whisper.cpp backend from Homebrew."
+    return 0
+  fi
+
+  echo "   XX   whisper-cli was copied but did not run successfully." >&2
+  return 1
+}
+
 if [[ "$PLATFORM" == "Linux" ]]; then
   mkdir -p "$APP_DIR/speech-backend/linux/cpu" "$APP_DIR/speech-backend/linux/vulkan"
   if [[ ! -x "$APP_DIR/speech-backend/linux/cpu/whisper-cli" && -x "$APP_DIR/speech-backend/linux/whisper-cli" ]]; then
@@ -71,9 +112,14 @@ elif [[ "$PLATFORM" == "Darwin" ]]; then
   fi
   if [[ -x "$APP_DIR/speech-backend/mac/cpu/whisper-cli" || -x "$APP_DIR/speech-backend/mac/metal/whisper-cli" ]]; then
     echo "   OK   whisper.cpp macOS speech backend already ready."
+  elif install_macos_whisper_from_homebrew; then
+    :
   else
-    echo "   !!   No official portable macOS whisper.cpp CLI archive is available for this setup script yet."
-    echo "        Build whisper.cpp manually and copy whisper-cli to app/speech-backend/mac/cpu or app/speech-backend/mac/metal."
+    echo "   !!   Could not install macOS whisper.cpp automatically."
+    echo "        Homebrew is not installed, and this app does not yet ship a portable macOS whisper.cpp archive."
+    echo "        For a distributable fix, bundle whisper-cli + whisper-server into app/speech-backend/mac/cpu"
+    echo "        during packaging, or add a Lemonade embeddable speech backend integration."
+    echo "        Manual workaround: install whisper.cpp and copy whisper-cli to app/speech-backend/mac/cpu."
   fi
 else
   echo "Unsupported platform: $PLATFORM" >&2

scripts/workers/coreml_server.py

@@ -121,6 +121,14 @@ def infer_model_sources(resource_dir: Path) -> str:
     return "packages"
 
 
+def has_coreml_safety_checker(resource_dir: Path, sources: str, model_version: str) -> bool:
+    names = {entry.name.lower() for entry in resource_dir.iterdir()}
+    if sources == "compiled":
+        return "safetychecker.mlmodelc" in names
+    package_name = f"Stable_Diffusion_version_{model_version}_safety_checker.mlpackage".replace("/", "_").lower()
+    return package_name in names
+
+
 def latest_png(root: Path) -> Path:
     images = sorted(root.rglob("*.png"), key=lambda p: p.stat().st_mtime, reverse=True)
     if not images:
@@ -148,6 +156,9 @@ class CoreMLServerState:
 
         compute_unit = os.environ.get("COREML_COMPUTE_UNIT", "CPU_AND_NE")
         sources = infer_model_sources(self.resources)
+        if not has_coreml_safety_checker(self.resources, sources, self.model_version):
+            print("[coreml-npu] Safety checker Core ML model not found; running without safety checker.", flush=True)
+            pytorch_pipe.safety_checker = None
 
         print(f"[coreml-npu] Loading Core ML models from: {self.resources} (compute unit: {compute_unit}, sources: {sources})", flush=True)
         self.pipe = get_coreml_pipe(
@@ -199,6 +210,9 @@ def make_handler(state: CoreMLServerState):
                 return
 
             try:
+                if not hasattr(state, "pipe") or not state.pipe:
+                    raise RuntimeError("CoreML pipeline is not initialized or failed to load.")
+
                 length = int(self.headers.get("Content-Length", "0") or "0")
                 payload = json.loads(self.rfile.read(length).decode("utf-8") or "{}")
                 prompt = str(payload.get("prompt") or "").strip()
@@ -211,6 +225,9 @@ def make_handler(state: CoreMLServerState):
 
                 steps = max(1, int(payload.get("steps") or state.steps or 30))
                 guidance = float(payload.get("cfg_scale") or state.cfg_scale or 7.0)
+                if guidance <= 1.0:
+                    print("[coreml-npu] CFG scale <= 1 can disable classifier-free guidance, but this compiled UNet expects CFG batch shape. Using cfg_scale=7.0.", flush=True)
+                    guidance = 7.0
                 seed = int(payload.get("seed")) if payload.get("seed") not in (None, -1) else int(time.time_ns() % (2 ** 32))
                 negative_prompt = str(payload.get("negative_prompt") or "").strip()