Merge remote-tracking branch 'origin/linux' into linux
# Conflicts: # scripts/serve.cjs
This commit is contained in:
orailnoor committed
commit
b55e62be7d
24 files changed
+1474
-179
No files matched your search
Binary file not shown.
@@ -0,0 +1,187 @@
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
import openvino as ov
|
||||
import openvino_genai as genai
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class WorkerState:
|
||||
def __init__(self, model_dir: Path, width: int, height: int, cache_dir: Path):
|
||||
self.model_dir = model_dir
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.cache_dir = cache_dir
|
||||
self.ready = False
|
||||
self.error = None
|
||||
self.pipeline = None
|
||||
self.devices = {}
|
||||
self.started_at = time.time()
|
||||
|
||||
def load(self) -> None:
|
||||
try:
|
||||
available_devices = ov.Core().available_devices
|
||||
if "NPU" not in available_devices:
|
||||
raise RuntimeError(
|
||||
f"Intel NPU is not available to OpenVINO. Devices: {', '.join(available_devices)}"
|
||||
)
|
||||
vae_device = "GPU" if "GPU" in available_devices else "CPU"
|
||||
self.devices = {
|
||||
"text_encoder": "CPU",
|
||||
"unet": "NPU",
|
||||
"vae_decoder": vae_device,
|
||||
}
|
||||
print(f"[openvino-npu] Loading model: {self.model_dir}", flush=True)
|
||||
print(
|
||||
"[openvino-npu] Devices: "
|
||||
f"text_encoder=CPU, unet=NPU, vae_decoder={vae_device}",
|
||||
flush=True,
|
||||
)
|
||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
pipe = genai.Text2ImagePipeline(str(self.model_dir))
|
||||
pipe.reshape(1, self.height, self.width, 1.0)
|
||||
pipe.compile(
|
||||
"CPU",
|
||||
"NPU",
|
||||
vae_device,
|
||||
config={"CACHE_DIR": str(self.cache_dir)},
|
||||
)
|
||||
self.pipeline = pipe
|
||||
self.ready = True
|
||||
print("[openvino-npu] READY", flush=True)
|
||||
except Exception as exc:
|
||||
self.error = str(exc)
|
||||
print(f"[openvino-npu] ERROR: {self.error}", file=sys.stderr, flush=True)
|
||||
raise
|
||||
|
||||
|
||||
def make_handler(state: WorkerState):
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def _json(self, code: int, payload: dict) -> None:
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_OPTIONS(self) -> None:
|
||||
self.send_response(204)
|
||||
self.send_header("Access-Control-Allow-Origin", "*")
|
||||
self.send_header("Access-Control-Allow-Headers", "Content-Type")
|
||||
self.send_header("Access-Control-Allow-Methods", "GET,POST,OPTIONS")
|
||||
self.end_headers()
|
||||
|
||||
def do_GET(self) -> None:
|
||||
if self.path == "/health":
|
||||
self._json(200, {
|
||||
"ok": state.ready and not state.error,
|
||||
"ready": state.ready,
|
||||
"error": state.error,
|
||||
"model": str(state.model_dir),
|
||||
"devices": state.devices,
|
||||
"uptime_sec": round(time.time() - state.started_at, 1),
|
||||
})
|
||||
return
|
||||
self._json(404, {"ok": False, "error": "Unknown endpoint"})
|
||||
|
||||
def do_POST(self) -> None:
|
||||
if self.path != "/generate":
|
||||
self._json(404, {"ok": False, "error": "Unknown endpoint"})
|
||||
return
|
||||
if not state.ready or state.pipeline is None:
|
||||
self._json(503, {"ok": False, "error": state.error or "OpenVINO NPU worker is not ready"})
|
||||
return
|
||||
|
||||
length = int(self.headers.get("Content-Length", "0") or "0")
|
||||
try:
|
||||
request = json.loads(self.rfile.read(length).decode("utf-8") or "{}")
|
||||
prompt = str(request.get("prompt") or "").strip()
|
||||
if not prompt:
|
||||
raise ValueError("Prompt is required")
|
||||
steps = max(1, min(8, int(request.get("steps") or 4)))
|
||||
guidance_scale = float(request.get("guidance_scale") or request.get("cfg_scale") or 1.0)
|
||||
width = int(request.get("width") or state.width)
|
||||
height = int(request.get("height") or state.height)
|
||||
if (width, height) not in ((512, 512), (1024, 1024)):
|
||||
raise ValueError("OpenVINO NPU supports 512x512 generation or 1024x1024 HD upscale.")
|
||||
|
||||
started = time.time()
|
||||
last_step_at = started
|
||||
|
||||
def progress_callback(step: int, num_steps: int, _latent) -> bool:
|
||||
nonlocal last_step_at
|
||||
now = time.time()
|
||||
step_duration = max(0.001, now - last_step_at)
|
||||
last_step_at = now
|
||||
print(
|
||||
f"[openvino-npu] PROGRESS {step + 1}/{num_steps} {1.0 / step_duration:.2f} it/s",
|
||||
flush=True,
|
||||
)
|
||||
return False
|
||||
|
||||
generate_args = {
|
||||
"width": state.width,
|
||||
"height": state.height,
|
||||
"num_inference_steps": steps,
|
||||
"guidance_scale": guidance_scale,
|
||||
"rng_seed": int(request.get("seed")) if request.get("seed") not in (None, -1) else int(time.time_ns() % (2**32)),
|
||||
"callback": progress_callback,
|
||||
}
|
||||
negative_prompt = str(request.get("negative_prompt") or "").strip()
|
||||
if negative_prompt and guidance_scale > 1.0:
|
||||
generate_args["negative_prompt"] = negative_prompt
|
||||
image_tensor = state.pipeline.generate(prompt, **generate_args)
|
||||
print("[openvino-npu] DECODING", flush=True)
|
||||
image = Image.fromarray(image_tensor.data[0])
|
||||
upscaled = width != state.width or height != state.height
|
||||
if upscaled:
|
||||
image = image.resize((width, height), Image.Resampling.LANCZOS)
|
||||
buf = BytesIO()
|
||||
image.save(buf, format="PNG")
|
||||
encoded = base64.b64encode(buf.getvalue()).decode("ascii")
|
||||
self._json(200, {
|
||||
"ok": True,
|
||||
"data": [{
|
||||
"b64_json": encoded,
|
||||
"seed": request.get("seed"),
|
||||
}],
|
||||
"generated_size": f"{state.width}x{state.height}",
|
||||
"output_size": f"{width}x{height}",
|
||||
"upscaled": upscaled,
|
||||
"duration_sec": round(time.time() - started, 2),
|
||||
})
|
||||
except Exception as exc:
|
||||
self._json(500, {"ok": False, "error": str(exc)})
|
||||
|
||||
def log_message(self, fmt: str, *args) -> None:
|
||||
print(f"[openvino-npu-http] {fmt % args}", flush=True)
|
||||
|
||||
return Handler
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model-dir", required=True, type=Path)
|
||||
parser.add_argument("--port", required=True, type=int)
|
||||
parser.add_argument("--width", type=int, default=512)
|
||||
parser.add_argument("--height", type=int, default=512)
|
||||
parser.add_argument("--cache-dir", type=Path, default=Path("app/tools/openvino-cache"))
|
||||
args = parser.parse_args()
|
||||
|
||||
state = WorkerState(args.model_dir, args.width, args.height, args.cache_dir)
|
||||
state.load()
|
||||
server = ThreadingHTTPServer(("127.0.0.1", args.port), make_handler(state))
|
||||
print(f"[openvino-npu] Listening on http://127.0.0.1:{args.port}", flush=True)
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -45,6 +45,19 @@ if (Test-Path $nodeModulesDir) {
|
||||
Remove-Item $nodeModulesDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
$nodeModulesMac = Join-Path $appDir "frontend\node_modules_mac"
|
||||
if (Test-Path $nodeModulesMac) {
|
||||
Write-Host " >> Removing frontend node_modules_mac..." -ForegroundColor Cyan
|
||||
Remove-Item $nodeModulesMac -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
$nodeModulesLinux = Join-Path $appDir "frontend\node_modules_linux"
|
||||
if (Test-Path $nodeModulesLinux) {
|
||||
Write-Host " >> Removing frontend node_modules_linux..." -ForegroundColor Cyan
|
||||
Remove-Item $nodeModulesLinux -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
|
||||
# Delete package-lock.json in frontend
|
||||
$lockFile = Join-Path $appDir "frontend\package-lock.json"
|
||||
if (Test-Path $lockFile) {
|
||||
|
||||
+11
-1
@@ -40,11 +40,21 @@ if [[ -d "$APP_DIR/models" ]]; then
|
||||
fi
|
||||
|
||||
# Delete node_modules in frontend
|
||||
if [[ -d "$APP_DIR/frontend/node_modules" ]]; then
|
||||
if [[ -L "$APP_DIR/frontend/node_modules" || -d "$APP_DIR/frontend/node_modules" ]]; then
|
||||
echo " >> Removing frontend node_modules..."
|
||||
rm -rf "$APP_DIR/frontend/node_modules"
|
||||
fi
|
||||
|
||||
if [[ -d "$APP_DIR/frontend/node_modules_mac" ]]; then
|
||||
echo " >> Removing frontend node_modules_mac..."
|
||||
rm -rf "$APP_DIR/frontend/node_modules_mac"
|
||||
fi
|
||||
|
||||
if [[ -d "$APP_DIR/frontend/node_modules_linux" ]]; then
|
||||
echo " >> Removing frontend node_modules_linux..."
|
||||
rm -rf "$APP_DIR/frontend/node_modules_linux"
|
||||
fi
|
||||
|
||||
# Delete package-lock.json in frontend
|
||||
if [[ -f "$APP_DIR/frontend/package-lock.json" ]]; then
|
||||
echo " >> Removing frontend package-lock.json..."
|
||||
|
||||
+547
-48
@@ -34,6 +34,7 @@ const BACKEND_PATHS = {
|
||||
linuxCuda: path.join(ROOT, "app", "backend", "linux", "cuda", "sd-server-cuda"),
|
||||
};
|
||||
let BACKEND_PATH = "";
|
||||
const backendSupportsFlags = {};
|
||||
if (osPlatform === "win32") {
|
||||
let hasNvidia = false;
|
||||
try {
|
||||
@@ -67,15 +68,38 @@ const MODELS = path.join(ROOT, "app", "models");
|
||||
if (!fs.existsSync(MODELS)) {
|
||||
fs.mkdirSync(MODELS, { recursive: true });
|
||||
}
|
||||
const OPENVINO_MODELS = path.join(ROOT, "app", "openvino-models");
|
||||
if (!fs.existsSync(OPENVINO_MODELS)) {
|
||||
fs.mkdirSync(OPENVINO_MODELS, { recursive: true });
|
||||
}
|
||||
const OUTPUTS = path.join(ROOT, "app", "outputs");
|
||||
if (!fs.existsSync(OUTPUTS)) {
|
||||
fs.mkdirSync(OUTPUTS, { recursive: true });
|
||||
}
|
||||
|
||||
const OPENVINO_NPU_MODELS = [
|
||||
{
|
||||
id: "lcm-dreamshaper-v7-fp16",
|
||||
name: "LCM DreamShaper v7 FP16",
|
||||
repo: "OpenVINO/LCM_Dreamshaper_v7-fp16-ov",
|
||||
folder: "LCM_Dreamshaper_v7-fp16-ov",
|
||||
approxSize: "2.0 GB",
|
||||
resolution: "512x512",
|
||||
notes: "OpenVINO LCM image model. Uses CPU text encoder, NPU UNet, and GPU or CPU VAE decoder.",
|
||||
},
|
||||
];
|
||||
|
||||
// ── Backend process state ─────────────────────────────────────────────────────
|
||||
let backendProc = null;
|
||||
let backendReady = false;
|
||||
let backendError = null;
|
||||
let openvinoProc = null;
|
||||
let openvinoReady = false;
|
||||
let openvinoError = null;
|
||||
let openvinoPort = null;
|
||||
let openvinoModel = null;
|
||||
let openvinoWidth = null;
|
||||
let openvinoHeight = null;
|
||||
let backendLoadState = {
|
||||
active: false,
|
||||
phase: "",
|
||||
@@ -104,6 +128,8 @@ let currentSettings = {
|
||||
vaeTiling: true,
|
||||
vaeOnCpu: false,
|
||||
flashAttn: true,
|
||||
width: 512,
|
||||
height: 512,
|
||||
};
|
||||
|
||||
let lastCpuSample = null;
|
||||
@@ -396,6 +422,110 @@ function getPathSize(targetPath) {
|
||||
}
|
||||
}
|
||||
|
||||
function getOpenVinoPythonCandidates() {
|
||||
const candidates = [];
|
||||
if (process.env.OPENVINO_PYTHON) candidates.push(process.env.OPENVINO_PYTHON);
|
||||
if (osPlatform === "win32") {
|
||||
candidates.push(path.join(ROOT, "app", "tools", "openvino-venv-win", "Scripts", "python.exe"));
|
||||
candidates.push(path.join(ROOT, "app", "tools", "openvino-venv", "Scripts", "python.exe")); // legacy fallback
|
||||
candidates.push("C:\\tmp\\npu-test-venv\\Scripts\\python.exe");
|
||||
candidates.push("python");
|
||||
} else {
|
||||
candidates.push(path.join(ROOT, "app", "tools", "openvino-venv-linux", "bin", "python"));
|
||||
candidates.push(path.join(ROOT, "app", "tools", "openvino-venv", "bin", "python")); // legacy fallback
|
||||
candidates.push("python3");
|
||||
candidates.push("python");
|
||||
}
|
||||
return [...new Set(candidates)];
|
||||
}
|
||||
|
||||
function getOpenVinoPython() {
|
||||
for (const candidate of getOpenVinoPythonCandidates()) {
|
||||
try {
|
||||
const result = spawnSync(candidate, [
|
||||
"-c",
|
||||
"import openvino, openvino_genai, PIL; print(openvino.__version__)",
|
||||
], { encoding: "utf8", timeout: 10000, stdio: ["ignore", "pipe", "pipe"] });
|
||||
if (result.status === 0) return candidate;
|
||||
} catch (_) {}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
let cachedOpenVinoNpuInfo = null;
|
||||
|
||||
function getOpenVinoNpuInfo() {
|
||||
if (cachedOpenVinoNpuInfo) return cachedOpenVinoNpuInfo;
|
||||
|
||||
if (osPlatform !== "win32" && osPlatform !== "linux") {
|
||||
cachedOpenVinoNpuInfo = { supported: false, reason: "OpenVINO NPU backend is supported on Windows and Linux Intel Core Ultra systems." };
|
||||
return cachedOpenVinoNpuInfo;
|
||||
}
|
||||
const python = getOpenVinoPython();
|
||||
if (!python) {
|
||||
const setupScript = osPlatform === "win32"
|
||||
? "scripts/setup-openvino-npu.ps1"
|
||||
: "bash scripts/setup-openvino-npu.sh";
|
||||
cachedOpenVinoNpuInfo = {
|
||||
supported: false,
|
||||
reason: `OpenVINO GenAI runtime is not installed. Run ${setupScript} first.`,
|
||||
};
|
||||
return cachedOpenVinoNpuInfo;
|
||||
}
|
||||
try {
|
||||
const script = [
|
||||
"import json, openvino as ov",
|
||||
"core=ov.Core()",
|
||||
"devices=core.available_devices",
|
||||
"info={'devices':devices,'npu':None}",
|
||||
"if 'NPU' in devices:",
|
||||
" info['npu']={'name':core.get_property('NPU','FULL_DEVICE_NAME'),'capabilities':core.get_property('NPU','OPTIMIZATION_CAPABILITIES')}",
|
||||
"print(json.dumps(info))",
|
||||
].join("\n");
|
||||
const result = spawnSync(python, ["-c", script], { encoding: "utf8", timeout: 15000, stdio: ["ignore", "pipe", "pipe"] });
|
||||
if (result.status !== 0) {
|
||||
cachedOpenVinoNpuInfo = { supported: false, python, reason: result.stderr.trim() || "OpenVINO NPU probe failed." };
|
||||
return cachedOpenVinoNpuInfo;
|
||||
}
|
||||
const info = JSON.parse(result.stdout.trim());
|
||||
if (!info.devices.includes("NPU")) {
|
||||
cachedOpenVinoNpuInfo = { supported: false, python, reason: `OpenVINO is installed, but NPU is not available. Devices: ${info.devices.join(", ")}` };
|
||||
return cachedOpenVinoNpuInfo;
|
||||
}
|
||||
cachedOpenVinoNpuInfo = { supported: true, platform: osPlatform, python, devices: info.devices, npu: info.npu };
|
||||
return cachedOpenVinoNpuInfo;
|
||||
} catch (err) {
|
||||
cachedOpenVinoNpuInfo = { supported: false, python, reason: err.message || String(err) };
|
||||
return cachedOpenVinoNpuInfo;
|
||||
}
|
||||
}
|
||||
|
||||
function getOpenVinoModelInfo() {
|
||||
return OPENVINO_NPU_MODELS.map((model) => {
|
||||
const modelPath = path.join(OPENVINO_MODELS, model.folder);
|
||||
const requiredFiles = [
|
||||
path.join(modelPath, "model_index.json"),
|
||||
path.join(modelPath, "text_encoder", "openvino_model.xml"),
|
||||
path.join(modelPath, "unet", "openvino_model.xml"),
|
||||
path.join(modelPath, "vae_decoder", "openvino_model.xml"),
|
||||
];
|
||||
const installed = requiredFiles.every((file) => fs.existsSync(file));
|
||||
return {
|
||||
...model,
|
||||
filename: model.id,
|
||||
path: modelPath,
|
||||
installed,
|
||||
sizeBytes: getPathSize(modelPath),
|
||||
size: formatBytes(getPathSize(modelPath)),
|
||||
format: "OpenVINO",
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function findOpenVinoModel(modelId) {
|
||||
return getOpenVinoModelInfo().find((model) => model.id === modelId || model.folder === modelId || model.name === modelId);
|
||||
}
|
||||
|
||||
function pathInside(childPath, parentPath) {
|
||||
const relative = path.relative(path.resolve(parentPath), path.resolve(childPath));
|
||||
return relative === "" || (!!relative && !relative.startsWith("..") && !path.isAbsolute(relative));
|
||||
@@ -504,7 +634,7 @@ async function getHealth() {
|
||||
|
||||
const ports = {
|
||||
frontend: { ...(await checkPort(PORT_FRONTEND)), expectedInUse: true },
|
||||
backend: { ...(await checkPort(PORT_BACKEND)), expectedInUse: backendProc !== null },
|
||||
backend: { ...(await checkPort(PORT_BACKEND)), expectedInUse: backendProc !== null || openvinoProc !== null },
|
||||
};
|
||||
ports.frontend.ok = !ports.frontend.available;
|
||||
ports.backend.preferred = PREFERRED_BACKEND_PORT;
|
||||
@@ -525,8 +655,8 @@ async function getHealth() {
|
||||
checks,
|
||||
ports,
|
||||
backend: {
|
||||
ready: backendReady,
|
||||
running: backendProc !== null,
|
||||
ready: backendReady || openvinoReady,
|
||||
running: backendProc !== null || openvinoProc !== null,
|
||||
error: backendError,
|
||||
settings: currentSettings,
|
||||
options: getBackendOptions(),
|
||||
@@ -622,6 +752,32 @@ function hasAmdGpu() {
|
||||
return info.includes("amd") || info.includes("advanced micro devices") || info.includes("radeon");
|
||||
}
|
||||
|
||||
// Detect whether we are running inside WSL2 (Windows Subsystem for Linux).
|
||||
// WSL2's GPU paravirtualization (GPU-PV) only exposes a D3D12/DirectX interface
|
||||
// to Linux. Vulkan hardware passthrough is only supported for NVIDIA (via CUDA).
|
||||
// Intel Arc and AMD GPUs in WSL2 fall back to llvmpipe (CPU software rendering).
|
||||
function isRunningInWSL() {
|
||||
if (osPlatform !== "linux") return false;
|
||||
try {
|
||||
const procVersion = fs.readFileSync("/proc/version", "utf8").toLowerCase();
|
||||
return procVersion.includes("microsoft") || procVersion.includes("wsl");
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function getVulkanUnavailableReason() {
|
||||
if (osPlatform === "linux" && isRunningInWSL()) {
|
||||
const gpuName = getGpuInfo().name;
|
||||
const lowerGpu = gpuName.toLowerCase();
|
||||
if (lowerGpu.includes("intel") || lowerGpu.includes("arc")) {
|
||||
return `Vulkan GPU is not available in WSL2 for Intel ${gpuName}. WSL2's GPU paravirtualization only exposes a DirectX interface — Intel Arc Vulkan requires running natively on Windows.`;
|
||||
}
|
||||
return "Vulkan GPU is not available in WSL2. WSL2's GPU paravirtualization does not support hardware Vulkan for this GPU. Run natively on Windows or Linux for GPU acceleration.";
|
||||
}
|
||||
return "Installed, but this binary did not register a Vulkan backend on this machine.";
|
||||
}
|
||||
|
||||
function getBackendOptions() {
|
||||
if (cachedBackendOptions) return cachedBackendOptions;
|
||||
|
||||
@@ -643,6 +799,9 @@ function getBackendOptions() {
|
||||
const metalInstalled = osPlatform === "darwin" && fs.existsSync(BACKEND_PATHS.mac);
|
||||
const metalAvailable = metalInstalled;
|
||||
const coremlAvailable = osPlatform === "darwin" && fs.existsSync("/Users/orailnoor/workspace/coreml_conversion/venv/bin/python");
|
||||
const openvinoNpu = getOpenVinoNpuInfo();
|
||||
const openvinoModels = getOpenVinoModelInfo();
|
||||
const openvinoNpuAvailable = openvinoNpu.supported && openvinoModels.some((model) => model.installed);
|
||||
|
||||
const options = [{ id: "cpu", label: "CPU", available: true }];
|
||||
if (metalAvailable) options.push({ id: "metal", label: "Metal GPU", available: true });
|
||||
@@ -650,10 +809,11 @@ function getBackendOptions() {
|
||||
if (vulkanAvailable) options.push({ id: "vulkan", label: "Vulkan GPU", available: true });
|
||||
if (rocmAvailable) options.push({ id: "rocm", label: "ROCm GPU (AMD)", available: true });
|
||||
if (cudaAvailable) options.push({ id: "cuda", label: "CUDA GPU", available: true });
|
||||
if (openvinoNpuAvailable) options.push({ id: "openvino-npu", label: "NPU (OpenVINO)", available: true });
|
||||
|
||||
const unavailable = [];
|
||||
if (vulkanInstalled && !vulkanAvailable) {
|
||||
unavailable.push({ id: "vulkan", label: "Vulkan GPU", reason: "Installed, but this binary did not register a Vulkan backend on this machine." });
|
||||
unavailable.push({ id: "vulkan", label: "Vulkan GPU", reason: getVulkanUnavailableReason() });
|
||||
}
|
||||
if (cudaInstalled && !cudaAvailable) {
|
||||
unavailable.push({ id: "cuda", label: "CUDA GPU", reason: "Installed, but CUDA backend validation failed." });
|
||||
@@ -664,6 +824,11 @@ function getBackendOptions() {
|
||||
if (metalInstalled && !metalAvailable) {
|
||||
unavailable.push({ id: "metal", label: "Metal GPU", reason: "Installed, but Metal backend validation failed." });
|
||||
}
|
||||
if (openvinoNpu.supported && !openvinoNpuAvailable) {
|
||||
unavailable.push({ id: "openvino-npu", label: "NPU (OpenVINO)", reason: "Runtime is ready, but no OpenVINO NPU model is downloaded." });
|
||||
} else if (!openvinoNpu.supported && (osPlatform === "win32" || osPlatform === "linux")) {
|
||||
unavailable.push({ id: "openvino-npu", label: "NPU (OpenVINO)", reason: openvinoNpu.reason });
|
||||
}
|
||||
|
||||
let defaultBackend = "cpu";
|
||||
if (coremlAvailable) {
|
||||
@@ -691,6 +856,9 @@ function getBackendOptions() {
|
||||
vulkanAvailable,
|
||||
rocmAvailable,
|
||||
metalAvailable,
|
||||
openvinoNpuAvailable,
|
||||
openvinoNpu,
|
||||
openvinoModels,
|
||||
defaultBackendType: defaultBackend,
|
||||
};
|
||||
return cachedBackendOptions;
|
||||
@@ -722,17 +890,19 @@ function backendAccepts(binaryPath, backendName) {
|
||||
const existing = spawnEnv.DYLD_LIBRARY_PATH || "";
|
||||
spawnEnv.DYLD_LIBRARY_PATH = dir + (existing ? ":" + existing : "");
|
||||
}
|
||||
let result = spawnSync(binaryPath, probeArgs, { encoding: "utf8", timeout: 5000, env: spawnEnv });
|
||||
let result = spawnSync(binaryPath, probeArgs, { env: spawnEnv, encoding: "utf8", timeout: 5000 });
|
||||
let output = `${result.stdout || ""}\n${result.stderr || ""}`;
|
||||
|
||||
let supportsFlags = true;
|
||||
// Some binaries do not support --backend. If we see "unknown argument",
|
||||
// retry without backend flags so we can still verify the binary launches.
|
||||
if (output.includes("unknown argument") && output.includes("--backend")) {
|
||||
supportsFlags = false;
|
||||
const fallbackArgs = [
|
||||
"--model", path.join(MODELS, "__backend_probe_missing__.safetensors"),
|
||||
"--listen-port", "18082",
|
||||
];
|
||||
result = spawnSync(binaryPath, fallbackArgs, { encoding: "utf8", timeout: 5000, env: spawnEnv });
|
||||
result = spawnSync(binaryPath, fallbackArgs, { env: spawnEnv, encoding: "utf8", timeout: 5000 });
|
||||
output = `${result.stdout || ""}\n${result.stderr || ""}`;
|
||||
}
|
||||
|
||||
@@ -745,7 +915,11 @@ function backendAccepts(binaryPath, backendName) {
|
||||
return false;
|
||||
}
|
||||
// A healthy binary prints project-specific log lines when it tries to load the model.
|
||||
return lower.includes("stable-diffusion.cpp") || lower.includes("loading model");
|
||||
const isOk = lower.includes("stable-diffusion.cpp") || lower.includes("loading model");
|
||||
if (isOk) {
|
||||
backendSupportsFlags[binaryPath] = supportsFlags;
|
||||
}
|
||||
return isOk;
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
@@ -844,6 +1018,208 @@ function pingBackendReady() {
|
||||
});
|
||||
}
|
||||
|
||||
function requestJson(url, payload = null, timeoutMs = 120000) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const parsed = new URL(url);
|
||||
const data = payload ? Buffer.from(JSON.stringify(payload), "utf8") : null;
|
||||
const req = http.request({
|
||||
hostname: parsed.hostname,
|
||||
port: parsed.port,
|
||||
path: parsed.pathname + parsed.search,
|
||||
method: data ? "POST" : "GET",
|
||||
headers: data ? {
|
||||
"Content-Type": "application/json",
|
||||
"Content-Length": data.length,
|
||||
} : {},
|
||||
timeout: timeoutMs,
|
||||
}, (res) => {
|
||||
let body = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk) => { body += chunk; });
|
||||
res.on("end", () => {
|
||||
try {
|
||||
const jsonBody = JSON.parse(body || "{}");
|
||||
if (res.statusCode < 200 || res.statusCode >= 300 || jsonBody.ok === false) {
|
||||
reject(new Error(jsonBody.error || `HTTP ${res.statusCode}`));
|
||||
return;
|
||||
}
|
||||
resolve(jsonBody);
|
||||
} catch (err) {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
});
|
||||
req.on("error", reject);
|
||||
req.on("timeout", () => req.destroy(new Error("Request timed out")));
|
||||
if (data) req.write(data);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function killOpenVinoWorker() {
|
||||
if (!openvinoProc) {
|
||||
openvinoReady = false;
|
||||
return;
|
||||
}
|
||||
console.log(" [openvino-npu] Stopping worker...");
|
||||
try { openvinoProc.kill("SIGTERM"); } catch (_) {}
|
||||
await new Promise((resolve) => setTimeout(resolve, 800));
|
||||
try { openvinoProc.kill("SIGKILL"); } catch (_) {}
|
||||
openvinoProc = null;
|
||||
openvinoReady = false;
|
||||
}
|
||||
|
||||
function getOpenVinoResolution(settings = {}) {
|
||||
const width = Number(settings.width) || 512;
|
||||
const height = Number(settings.height) || 512;
|
||||
const supported = (width === 512 && height === 512) || (width === 1024 && height === 1024);
|
||||
if (!supported) {
|
||||
throw new Error("OpenVINO NPU supports 512x512 generation or 1024x1024 HD upscale.");
|
||||
}
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
async function startOpenVinoWorker(settings = {}) {
|
||||
const npuInfo = getOpenVinoNpuInfo();
|
||||
if (!npuInfo.supported) throw new Error(npuInfo.reason || "OpenVINO NPU is not available.");
|
||||
const model = findOpenVinoModel(settings.model) || getOpenVinoModelInfo().find((item) => item.installed);
|
||||
if (!model || !model.installed) {
|
||||
throw new Error("OpenVINO NPU model is not downloaded. Download an OpenVINO NPU model from Model Manager first.");
|
||||
}
|
||||
const { width, height } = getOpenVinoResolution(settings);
|
||||
if (openvinoProc &&
|
||||
openvinoReady &&
|
||||
openvinoModel === model.id) {
|
||||
currentSettings = { ...currentSettings, ...settings, model: model.id, width, height };
|
||||
return;
|
||||
}
|
||||
|
||||
await killBackend();
|
||||
await killOpenVinoWorker();
|
||||
openvinoError = null;
|
||||
openvinoReady = false;
|
||||
openvinoModel = model.id;
|
||||
openvinoWidth = 512;
|
||||
openvinoHeight = 512;
|
||||
openvinoPort = await findAvailableBackendPort();
|
||||
PORT_BACKEND = openvinoPort;
|
||||
|
||||
currentSettings = { ...currentSettings, ...settings };
|
||||
currentSettings.backendType = "openvino-npu";
|
||||
currentSettings.useGpu = true;
|
||||
currentSettings.backendMode = "NPU (OpenVINO)";
|
||||
currentSettings.backendBinary = "openvino_npu_worker.py";
|
||||
currentSettings.backendDevice = npuInfo.npu?.name || "Intel AI Boost";
|
||||
currentSettings.model = model.id;
|
||||
currentSettings.width = width;
|
||||
currentSettings.height = height;
|
||||
|
||||
backendError = null;
|
||||
backendLoadState = {
|
||||
active: true,
|
||||
phase: "Compiling OpenVINO NPU pipeline (512x512)...",
|
||||
progress: 5,
|
||||
current: 0,
|
||||
total: 0,
|
||||
speed: "",
|
||||
model: model.name,
|
||||
backendMode: "NPU (OpenVINO)",
|
||||
backendBinary: "openvino_npu_worker.py",
|
||||
device: currentSettings.backendDevice,
|
||||
};
|
||||
|
||||
const workerPath = path.join(ROOT, "scripts", "openvino_npu_worker.py");
|
||||
const cacheDir = path.join(ROOT, "app", "tools", "openvino-cache", "512x512");
|
||||
console.log(` [openvino-npu] Starting 512x512 worker on port ${openvinoPort}`);
|
||||
openvinoProc = spawn(npuInfo.python, [
|
||||
workerPath,
|
||||
"--model-dir", model.path,
|
||||
"--port", String(openvinoPort),
|
||||
"--width", "512",
|
||||
"--height", "512",
|
||||
"--cache-dir", cacheDir,
|
||||
], { stdio: "pipe" });
|
||||
|
||||
let openvinoStdoutBuffer = "";
|
||||
openvinoProc.stdout.on("data", (data) => {
|
||||
const text = data.toString();
|
||||
process.stdout.write(" " + text);
|
||||
openvinoStdoutBuffer += text;
|
||||
const lines = openvinoStdoutBuffer.split(/\r?\n/);
|
||||
openvinoStdoutBuffer = lines.pop() || "";
|
||||
for (const line of lines) {
|
||||
if (line.includes("READY")) {
|
||||
openvinoReady = true;
|
||||
backendReady = true;
|
||||
backendLoadState = { ...backendLoadState, active: false, phase: "OpenVINO NPU model ready", progress: 100 };
|
||||
}
|
||||
const progressMatch = line.match(/PROGRESS\s+(\d+)\/(\d+)\s+(.+)$/);
|
||||
if (progressMatch) {
|
||||
generationState.active = true;
|
||||
generationState.step = Number(progressMatch[1]);
|
||||
generationState.steps = Number(progressMatch[2]);
|
||||
generationState.speed = progressMatch[3].trim();
|
||||
generationState.decoding = false;
|
||||
} else if (line.includes("DECODING") && generationState.active) {
|
||||
generationState.step = generationState.steps;
|
||||
generationState.decoding = true;
|
||||
generationState.speed = "";
|
||||
}
|
||||
}
|
||||
});
|
||||
openvinoProc.stderr.on("data", (data) => {
|
||||
const text = data.toString();
|
||||
process.stderr.write(" " + text);
|
||||
const cleanText = stripAnsi(text).trim();
|
||||
if (cleanText && (cleanText.includes("ERROR") || cleanText.includes("Traceback") || cleanText.includes("Exception"))) {
|
||||
openvinoError = cleanText;
|
||||
backendError = openvinoError;
|
||||
}
|
||||
});
|
||||
openvinoProc.on("exit", (code) => {
|
||||
console.log(" [openvino-npu] worker exited with code", code);
|
||||
openvinoProc = null;
|
||||
openvinoReady = false;
|
||||
backendReady = false;
|
||||
if (code !== 0 && code !== null && !backendError) backendError = `OpenVINO NPU worker exited with code ${code}`;
|
||||
backendLoadState.active = false;
|
||||
});
|
||||
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < 360000) {
|
||||
if (openvinoReady) return;
|
||||
if (backendError) throw new Error(backendError);
|
||||
if (!openvinoProc) throw new Error("OpenVINO NPU worker exited during startup.");
|
||||
const elapsed = Math.round((Date.now() - startedAt) / 1000);
|
||||
backendLoadState.progress = Math.min(95, 5 + Math.round(elapsed / 2));
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
await killOpenVinoWorker();
|
||||
throw new Error("OpenVINO NPU 512x512 compile timed out after 6 minutes.");
|
||||
}
|
||||
|
||||
async function generateWithOpenVino(body) {
|
||||
if (!openvinoReady || !openvinoPort) throw new Error("OpenVINO NPU worker is not ready. Load an OpenVINO NPU model first.");
|
||||
const steps = Math.max(1, Math.min(8, Number(body.steps) || 4));
|
||||
generationState = { active: true, step: 0, steps, speed: "", decoding: false };
|
||||
try {
|
||||
const result = await requestJson(`http://127.0.0.1:${openvinoPort}/generate`, {
|
||||
prompt: body.prompt,
|
||||
negative_prompt: body.negative_prompt || "",
|
||||
width: Number(body.width) || currentSettings.width || 512,
|
||||
height: Number(body.height) || currentSettings.height || 512,
|
||||
steps,
|
||||
cfg_scale: Number(body.cfg_scale) || 1.0,
|
||||
seed: body.seed,
|
||||
}, 300000);
|
||||
resetGenerationState();
|
||||
return result;
|
||||
} catch (err) {
|
||||
resetGenerationState();
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function startBackendReadyPoll() {
|
||||
let attempts = 0;
|
||||
const interval = setInterval(async () => {
|
||||
@@ -888,6 +1264,11 @@ function killBackend() {
|
||||
}
|
||||
|
||||
async function startBackend(settings = {}) {
|
||||
if (settings.backendType === "openvino-npu") {
|
||||
await startOpenVinoWorker(settings);
|
||||
return;
|
||||
}
|
||||
await killOpenVinoWorker();
|
||||
backendError = null;
|
||||
currentSettings = { ...currentSettings, ...settings };
|
||||
if (!currentSettings.model) currentSettings.model = getDefaultModel();
|
||||
@@ -945,6 +1326,8 @@ async function startBackend(settings = {}) {
|
||||
let args = [];
|
||||
const requestedBackend = resolveBackendType(currentSettings.useGpu, currentSettings.backendType, currentSettings.model);
|
||||
|
||||
const supportsFlags = backendSupportsFlags[backendPath] !== false;
|
||||
|
||||
if (requestedBackend === "apple-npu") {
|
||||
args = [
|
||||
path.join(ROOT, "app", "backend", "mac", "coreml_server.py"),
|
||||
@@ -964,39 +1347,30 @@ async function startBackend(settings = {}) {
|
||||
];
|
||||
|
||||
if (requestedBackend === "cpu") {
|
||||
args.push(
|
||||
"--backend", "cpu",
|
||||
"--params-backend", "cpu",
|
||||
"--rng", "cpu",
|
||||
"--sampler-rng", "cpu",
|
||||
);
|
||||
if (supportsFlags) {
|
||||
args.push("--backend", "cpu", "--params-backend", "cpu");
|
||||
}
|
||||
args.push("--rng", "cpu", "--sampler-rng", "cpu");
|
||||
} else if (requestedBackend === "vulkan") {
|
||||
args.push(
|
||||
"--backend", "vulkan0",
|
||||
"--params-backend", "vulkan0",
|
||||
"--rng", "cpu",
|
||||
"--sampler-rng", "cpu",
|
||||
);
|
||||
if (supportsFlags) {
|
||||
args.push("--backend", "vulkan0", "--params-backend", "vulkan0");
|
||||
}
|
||||
args.push("--rng", "cpu", "--sampler-rng", "cpu");
|
||||
} else if (requestedBackend === "cuda") {
|
||||
// CUDA is only supported on Windows. Linux has no reliable prebuilt CUDA binary.
|
||||
args.push(
|
||||
"--backend", "cuda0",
|
||||
"--params-backend", "cuda0",
|
||||
"--rng", "cuda",
|
||||
"--sampler-rng", "cuda",
|
||||
);
|
||||
if (supportsFlags) {
|
||||
args.push("--backend", "cuda0", "--params-backend", "cuda0");
|
||||
}
|
||||
args.push("--rng", "cuda", "--sampler-rng", "cuda");
|
||||
} else if (requestedBackend === "rocm") {
|
||||
args.push(
|
||||
"--backend", "rocm0",
|
||||
"--params-backend", "rocm0",
|
||||
"--rng", "cpu",
|
||||
"--sampler-rng", "cpu",
|
||||
);
|
||||
if (supportsFlags) {
|
||||
args.push("--backend", "rocm0", "--params-backend", "rocm0");
|
||||
}
|
||||
args.push("--rng", "cpu", "--sampler-rng", "cpu");
|
||||
} else if (requestedBackend === "metal") {
|
||||
args.push(
|
||||
"--rng", "cpu",
|
||||
"--sampler-rng", "cpu",
|
||||
);
|
||||
if (supportsFlags) {
|
||||
args.push("--backend", "metal0", "--params-backend", "metal0");
|
||||
}
|
||||
args.push("--rng", "cpu", "--sampler-rng", "cpu");
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1193,6 +1567,54 @@ let downloadState = {
|
||||
};
|
||||
let activeDownload = null;
|
||||
|
||||
function startOpenVinoModelDownload(modelId) {
|
||||
if (downloadState.active) return;
|
||||
const npuInfo = getOpenVinoNpuInfo();
|
||||
if (!npuInfo.supported) throw new Error(npuInfo.reason || "OpenVINO NPU runtime is not available.");
|
||||
const model = OPENVINO_NPU_MODELS.find((item) => item.id === modelId);
|
||||
if (!model) throw new Error("Unknown OpenVINO model.");
|
||||
|
||||
const destDir = path.join(OPENVINO_MODELS, model.folder);
|
||||
fs.mkdirSync(destDir, { recursive: true });
|
||||
downloadState = {
|
||||
active: true,
|
||||
filename: model.name,
|
||||
progress: -1,
|
||||
speed: "Downloading snapshot",
|
||||
eta: 0,
|
||||
totalBytes: 0,
|
||||
downloadedBytes: 0,
|
||||
error: null,
|
||||
};
|
||||
|
||||
const script = [
|
||||
"from huggingface_hub import snapshot_download",
|
||||
`snapshot_download(repo_id=${JSON.stringify(model.repo)}, local_dir=${JSON.stringify(destDir)}, ignore_patterns=['safety_checker/*','feature_extractor/*','*.safetensors'])`,
|
||||
"print('DONE')",
|
||||
].join("\n");
|
||||
console.log(` [openvino-download] Downloading ${model.repo} -> ${destDir}`);
|
||||
const proc = spawn(npuInfo.python, ["-c", script], { stdio: "pipe" });
|
||||
activeDownload = { process: proc, destPath: destDir, openvino: true };
|
||||
proc.stdout.on("data", (data) => process.stdout.write(" [openvino-download] " + data.toString()));
|
||||
proc.stderr.on("data", (data) => process.stderr.write(" [openvino-download] " + data.toString()));
|
||||
proc.on("exit", (code) => {
|
||||
activeDownload = null;
|
||||
if (String(downloadState.error || "").toLowerCase().includes("cancelled")) {
|
||||
return;
|
||||
}
|
||||
downloadState.active = false;
|
||||
if (code === 0) {
|
||||
downloadState.progress = 100;
|
||||
downloadState.downloadedBytes = getPathSize(destDir);
|
||||
downloadState.totalBytes = downloadState.downloadedBytes;
|
||||
downloadState.speed = "Complete";
|
||||
cachedBackendOptions = null;
|
||||
} else {
|
||||
downloadState.error = `OpenVINO model download failed with code ${code}`;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ── Generation State (Real-time progress parser) ─────────────────────────────
|
||||
let generationState = {
|
||||
active: false,
|
||||
@@ -1428,9 +1850,14 @@ function cancelModelDownload() {
|
||||
}
|
||||
const filename = downloadState.filename;
|
||||
if (activeDownload) {
|
||||
try { activeDownload.request.destroy(new Error("Download cancelled by user")); } catch (_) {}
|
||||
try { activeDownload.fileStream.destroy(); } catch (_) {}
|
||||
try { fs.unlinkSync(activeDownload.tempPath || activeDownload.destPath); } catch (_) {}
|
||||
if (activeDownload.process) {
|
||||
try { activeDownload.process.kill("SIGTERM"); } catch (_) {}
|
||||
setTimeout(() => { try { activeDownload?.process?.kill("SIGKILL"); } catch (_) {} }, 1000);
|
||||
} else {
|
||||
try { activeDownload.request.destroy(new Error("Download cancelled by user")); } catch (_) {}
|
||||
try { activeDownload.fileStream.destroy(); } catch (_) {}
|
||||
try { fs.unlinkSync(activeDownload.tempPath || activeDownload.destPath); } catch (_) {}
|
||||
}
|
||||
} else if (filename) {
|
||||
try { fs.unlinkSync(`${path.join(MODELS, filename)}.part`); } catch (_) {}
|
||||
}
|
||||
@@ -1600,9 +2027,9 @@ function safeOutputName(value) {
|
||||
}
|
||||
|
||||
function saveGeneratedOutput(imageDataUrl, metadata = {}) {
|
||||
const match = String(imageDataUrl || "").match(/^data:(image\/(?:png|jpeg|jpg|webp|svg\+xml));base64,(.+)$/);
|
||||
const match = String(imageDataUrl || "").match(/^data:(image\/(?:png|jpeg|jpg|webp));base64,(.+)$/);
|
||||
if (!match) {
|
||||
throw new Error("Expected a base64 image data URL");
|
||||
throw new Error("Expected a real base64 PNG, JPEG, or WebP image data URL");
|
||||
}
|
||||
|
||||
const mime = match[1];
|
||||
@@ -1611,9 +2038,26 @@ function saveGeneratedOutput(imageDataUrl, metadata = {}) {
|
||||
"image/jpeg": ".jpg",
|
||||
"image/jpg": ".jpg",
|
||||
"image/webp": ".webp",
|
||||
"image/svg+xml": ".svg",
|
||||
};
|
||||
const ext = extByMime[mime] || ".png";
|
||||
const imageBuffer = Buffer.from(match[2], "base64");
|
||||
const isPng = imageBuffer.length > 8 &&
|
||||
imageBuffer[0] === 0x89 &&
|
||||
imageBuffer[1] === 0x50 &&
|
||||
imageBuffer[2] === 0x4e &&
|
||||
imageBuffer[3] === 0x47;
|
||||
const isJpeg = imageBuffer.length > 3 &&
|
||||
imageBuffer[0] === 0xff &&
|
||||
imageBuffer[1] === 0xd8 &&
|
||||
imageBuffer[2] === 0xff;
|
||||
const isWebp = imageBuffer.length > 12 &&
|
||||
imageBuffer.toString("ascii", 0, 4) === "RIFF" &&
|
||||
imageBuffer.toString("ascii", 8, 12) === "WEBP";
|
||||
if ((mime === "image/png" && !isPng) ||
|
||||
((mime === "image/jpeg" || mime === "image/jpg") && !isJpeg) ||
|
||||
(mime === "image/webp" && !isWebp)) {
|
||||
throw new Error("Generated image payload did not match its declared image format.");
|
||||
}
|
||||
const createdAt = new Date().toISOString();
|
||||
const stamp = createdAt.replace(/[:.]/g, "-");
|
||||
const seed = metadata.seed !== undefined && metadata.seed !== null ? `-${safeOutputName(metadata.seed)}` : "";
|
||||
@@ -1623,7 +2067,7 @@ function saveGeneratedOutput(imageDataUrl, metadata = {}) {
|
||||
const imagePath = path.join(OUTPUTS, imageFilename);
|
||||
const metadataPath = path.join(OUTPUTS, metadataFilename);
|
||||
|
||||
fs.writeFileSync(imagePath, Buffer.from(match[2], "base64"));
|
||||
fs.writeFileSync(imagePath, imageBuffer);
|
||||
const savedMetadata = {
|
||||
...metadata,
|
||||
createdAt,
|
||||
@@ -1789,9 +2233,9 @@ const server = http.createServer(async (req, res) => {
|
||||
|
||||
// GET /api/backend-status
|
||||
if (req.url === "/api/backend-status" && req.method === "GET") {
|
||||
return json(res, 200, {
|
||||
ready: backendReady,
|
||||
running: backendProc !== null,
|
||||
return json(res, 200, {
|
||||
ready: backendReady || openvinoReady,
|
||||
running: backendProc !== null || openvinoProc !== null,
|
||||
port: PORT_BACKEND,
|
||||
preferredPort: PREFERRED_BACKEND_PORT,
|
||||
error: backendError,
|
||||
@@ -1828,6 +2272,17 @@ const server = http.createServer(async (req, res) => {
|
||||
} catch (_) { return json(res, 200, { models: [] }); }
|
||||
}
|
||||
|
||||
if (req.url === "/api/openvino-models" && req.method === "GET") {
|
||||
const npuInfo = getOpenVinoNpuInfo();
|
||||
return json(res, 200, {
|
||||
supported: npuInfo.supported,
|
||||
reason: npuInfo.reason || "",
|
||||
npu: npuInfo.npu || null,
|
||||
python: npuInfo.python || "",
|
||||
models: npuInfo.supported ? getOpenVinoModelInfo() : [],
|
||||
});
|
||||
}
|
||||
|
||||
// POST /api/restart-backend — restart with new settings
|
||||
if (req.url === "/api/restart-backend" && req.method === "POST") {
|
||||
const body = await readJsonBody(req, res);
|
||||
@@ -1836,11 +2291,13 @@ const server = http.createServer(async (req, res) => {
|
||||
await killBackend();
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
const newSettings = {};
|
||||
if (body.model) newSettings.model = path.join(MODELS, body.model);
|
||||
if (body.model) newSettings.model = body.backend_type === "openvino-npu" ? String(body.model) : path.join(MODELS, body.model);
|
||||
if (body.steps) newSettings.steps = parseInt(body.steps);
|
||||
if (body.cfgScale) newSettings.cfgScale = parseFloat(body.cfgScale);
|
||||
if (body.sampler) newSettings.sampler = body.sampler;
|
||||
if (body.threads) newSettings.threads = parseInt(body.threads);
|
||||
if (body.width) newSettings.width = parseInt(body.width);
|
||||
if (body.height) newSettings.height = parseInt(body.height);
|
||||
if (typeof body.use_gpu === "boolean") newSettings.useGpu = body.use_gpu;
|
||||
if (body.backend_type) {
|
||||
newSettings.backendType = String(body.backend_type);
|
||||
@@ -1850,6 +2307,18 @@ const server = http.createServer(async (req, res) => {
|
||||
if (typeof body.vae_on_cpu === "boolean") newSettings.vaeOnCpu = body.vae_on_cpu;
|
||||
if (typeof body.flash_attn === "boolean") newSettings.flashAttn = body.flash_attn;
|
||||
try {
|
||||
if (newSettings.backendType === "openvino-npu") {
|
||||
startBackend(newSettings).catch((err) => {
|
||||
backendError = err.message || String(err);
|
||||
backendLoadState = {
|
||||
...backendLoadState,
|
||||
active: false,
|
||||
phase: "OpenVINO NPU model load failed",
|
||||
};
|
||||
console.error(" [openvino-npu] Startup failed:", backendError);
|
||||
});
|
||||
return json(res, 200, { ok: true, message: "OpenVINO NPU backend starting...", settings: currentSettings, port: PORT_BACKEND });
|
||||
}
|
||||
await startBackend(newSettings);
|
||||
return json(res, 200, { ok: true, message: "Backend restarting...", settings: currentSettings, port: PORT_BACKEND });
|
||||
} catch (err) {
|
||||
@@ -1861,6 +2330,7 @@ const server = http.createServer(async (req, res) => {
|
||||
// POST /api/stop-backend
|
||||
if (req.url === "/api/stop-backend" && req.method === "POST") {
|
||||
await killBackend();
|
||||
await killOpenVinoWorker();
|
||||
return json(res, 200, { ok: true });
|
||||
}
|
||||
|
||||
@@ -1875,6 +2345,19 @@ const server = http.createServer(async (req, res) => {
|
||||
return json(res, 200, { ok: true, message: "Download started" });
|
||||
}
|
||||
|
||||
if (req.url === "/api/download-openvino-model" && req.method === "POST") {
|
||||
const body = await readJsonBody(req, res);
|
||||
if (!body) return;
|
||||
const modelId = String(body.model_id || body.modelId || "");
|
||||
if (!modelId) return json(res, 400, { ok: false, error: "model_id is required" });
|
||||
try {
|
||||
startOpenVinoModelDownload(modelId);
|
||||
return json(res, 200, { ok: true, message: "OpenVINO model download started" });
|
||||
} catch (err) {
|
||||
return json(res, 500, { ok: false, error: err.message || String(err) });
|
||||
}
|
||||
}
|
||||
|
||||
// GET /api/download-progress
|
||||
if (req.url === "/api/download-progress" && req.method === "GET") {
|
||||
return json(res, 200, downloadState);
|
||||
@@ -1891,6 +2374,16 @@ const server = http.createServer(async (req, res) => {
|
||||
return json(res, 200, generationState);
|
||||
}
|
||||
|
||||
if (req.url === "/api/openvino-generate" && req.method === "POST") {
|
||||
const body = await readJsonBody(req, res);
|
||||
if (!body) return;
|
||||
try {
|
||||
return json(res, 200, await generateWithOpenVino(body));
|
||||
} catch (err) {
|
||||
return json(res, 500, { ok: false, error: err.message || String(err) });
|
||||
}
|
||||
}
|
||||
|
||||
// GET /api/outputs
|
||||
if (req.url === "/api/outputs" && req.method === "GET") {
|
||||
return json(res, 200, { outputs: listGeneratedOutputs() });
|
||||
@@ -1958,11 +2451,17 @@ const server = http.createServer(async (req, res) => {
|
||||
const { filename } = body;
|
||||
if (!filename) return json(res, 400, { error: "Filename is required" });
|
||||
|
||||
const openvinoModel = findOpenVinoModel(filename);
|
||||
const safeFilename = path.basename(filename);
|
||||
const modelPath = path.join(MODELS, safeFilename);
|
||||
|
||||
try {
|
||||
if (fs.existsSync(modelPath)) {
|
||||
if (openvinoModel?.installed && openvinoModel.path && pathInside(openvinoModel.path, OPENVINO_MODELS)) {
|
||||
fs.rmSync(openvinoModel.path, { recursive: true, force: true });
|
||||
cachedBackendOptions = null;
|
||||
console.log(` [api] Deleted OpenVINO model: ${openvinoModel.id}`);
|
||||
return json(res, 200, { ok: true, message: `Deleted ${openvinoModel.name}` });
|
||||
} else if (fs.existsSync(modelPath)) {
|
||||
fs.unlinkSync(modelPath);
|
||||
console.log(` [api] Deleted model file: ${safeFilename}`);
|
||||
return json(res, 200, { ok: true, message: `Deleted ${safeFilename}` });
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# Optional OpenVINO NPU setup for Intel AI Boost systems.
|
||||
# This does not modify the existing stable-diffusion.cpp backends.
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
$rootDir = Split-Path -Parent $scriptDir
|
||||
$venvDir = Join-Path $rootDir "app\tools\openvino-venv-win"
|
||||
$pythonExe = Join-Path $venvDir "Scripts\python.exe"
|
||||
|
||||
Write-Host ""
|
||||
Write-Host " ============================================================"
|
||||
Write-Host " Local AI Image Generator - OpenVINO NPU Setup"
|
||||
Write-Host " ============================================================"
|
||||
Write-Host ""
|
||||
|
||||
$npu = Get-CimInstance Win32_PnPEntity -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.Name -like "*Intel(R) AI Boost*" -or ($_.Name -match "NPU" -and $_.PNPClass -eq "ComputeAccelerator") } |
|
||||
Select-Object -First 1
|
||||
|
||||
if (-not $npu) {
|
||||
Write-Host " [ERROR] Intel AI Boost NPU was not detected on this system." -ForegroundColor Red
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host " Detected: $($npu.Name)"
|
||||
|
||||
if (-not (Test-Path $pythonExe)) {
|
||||
Write-Host " Creating Python environment: $venvDir"
|
||||
python -m venv $venvDir
|
||||
}
|
||||
|
||||
Write-Host " Installing OpenVINO GenAI runtime..."
|
||||
& $pythonExe -m pip install --upgrade pip
|
||||
& $pythonExe -m pip install openvino openvino-genai pillow huggingface-hub
|
||||
|
||||
Write-Host ""
|
||||
Write-Host " Verifying OpenVINO NPU device..."
|
||||
& $pythonExe -c "import openvino as ov; core=ov.Core(); print('Available devices:', core.available_devices); raise SystemExit(0 if 'NPU' in core.available_devices else 2)"
|
||||
|
||||
Write-Host ""
|
||||
Write-Host " ============================================================"
|
||||
Write-Host " OpenVINO NPU setup complete."
|
||||
Write-Host " Restart windows.bat, then download an OpenVINO NPU model."
|
||||
Write-Host " ============================================================"
|
||||
Write-Host ""
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Optional OpenVINO NPU setup for Linux Intel Core Ultra systems.
|
||||
# The Intel NPU kernel/user-mode driver must already be installed.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
VENV_DIR="$ROOT_DIR/app/tools/openvino-venv-linux"
|
||||
PYTHON_BIN="$VENV_DIR/bin/python"
|
||||
|
||||
echo ""
|
||||
echo " ============================================================"
|
||||
echo " Local AI Image Generator - Linux OpenVINO NPU Setup"
|
||||
echo " ============================================================"
|
||||
echo ""
|
||||
|
||||
if [[ "$(uname -s)" != "Linux" ]]; then
|
||||
echo " [ERROR] This setup script is for Linux only." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "$(uname -m)" != "x86_64" ]]; then
|
||||
echo " [ERROR] OpenVINO NPU support requires x86_64 Linux." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
echo " [ERROR] python3 is required. Install Python 3 and the venv module." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
KERNEL_VERSION="$(uname -r)"
|
||||
KERNEL_MAJOR="${KERNEL_VERSION%%.*}"
|
||||
KERNEL_REST="${KERNEL_VERSION#*.}"
|
||||
KERNEL_MINOR="${KERNEL_REST%%.*}"
|
||||
if (( KERNEL_MAJOR < 6 || (KERNEL_MAJOR == 6 && KERNEL_MINOR < 6) )); then
|
||||
echo " [ERROR] Intel NPU support requires Linux kernel 6.6 or newer."
|
||||
echo " Current kernel: $KERNEL_VERSION"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -e /dev/accel/accel0 ]]; then
|
||||
echo " [ERROR] Intel NPU device /dev/accel/accel0 was not found."
|
||||
echo " Install the Intel Linux NPU driver and reboot first."
|
||||
echo " Driver: https://github.com/intel/linux-npu-driver"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -x "$PYTHON_BIN" ]]; then
|
||||
echo " Creating Python environment: $VENV_DIR"
|
||||
if ! python3 -m venv "$VENV_DIR"; then
|
||||
echo " [ERROR] Could not create the virtual environment."
|
||||
echo " On Ubuntu, install python3-venv and retry."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo " Installing OpenVINO GenAI runtime..."
|
||||
"$PYTHON_BIN" -m pip install --upgrade pip
|
||||
"$PYTHON_BIN" -m pip install openvino openvino-genai pillow huggingface-hub
|
||||
|
||||
echo ""
|
||||
echo " Verifying OpenVINO NPU device..."
|
||||
"$PYTHON_BIN" - <<'PY'
|
||||
import openvino as ov
|
||||
|
||||
core = ov.Core()
|
||||
devices = core.available_devices
|
||||
print(" Available devices:", ", ".join(devices))
|
||||
if "NPU" not in devices:
|
||||
raise SystemExit(
|
||||
" [ERROR] OpenVINO is installed, but NPU is unavailable. "
|
||||
"Verify the Intel NPU driver and device permissions."
|
||||
)
|
||||
print(" NPU:", core.get_property("NPU", "FULL_DEVICE_NAME"))
|
||||
PY
|
||||
|
||||
echo ""
|
||||
echo " ============================================================"
|
||||
echo " OpenVINO NPU setup complete."
|
||||
echo " Run ./linux.sh, then download an OpenVINO NPU model."
|
||||
echo " ============================================================"
|
||||
echo ""
|
||||
+24
-2
@@ -47,6 +47,18 @@ function Format-Speed {
|
||||
return "{0:N0} KB/s" -f ($bps / 1KB)
|
||||
}
|
||||
|
||||
function Enable-Tls12 {
|
||||
try {
|
||||
# Use the protocol's numeric value because older .NET enum definitions
|
||||
# do not expose the Tls12 member even when Windows supports TLS 1.2.
|
||||
$tls12 = [Enum]::ToObject([Net.SecurityProtocolType], 3072)
|
||||
[Net.ServicePointManager]::SecurityProtocol =
|
||||
[Net.ServicePointManager]::SecurityProtocol -bor $tls12
|
||||
} catch {
|
||||
throw "TLS 1.2 could not be enabled. This setup requires 64-bit Windows 10 or Windows 11 with current system updates. $($_.Exception.Message)"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-RichDownload {
|
||||
param([string]$Url, [string]$Dest, [string]$Label)
|
||||
Print-Info "Downloading: $Label"
|
||||
@@ -57,7 +69,7 @@ function Invoke-RichDownload {
|
||||
$lastTime = [DateTime]::Now
|
||||
|
||||
try {
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
||||
Enable-Tls12
|
||||
$req = [System.Net.HttpWebRequest]::Create($Url)
|
||||
$req.UserAgent = "Mozilla/5.0"
|
||||
$resp = $req.GetResponse()
|
||||
@@ -388,6 +400,16 @@ if (-not (Test-Path $npmCmd)) {
|
||||
Read-Host; exit 1
|
||||
}
|
||||
|
||||
# If node_modules is a Unix symlink/junction, remove it so Windows can install natively into a real directory
|
||||
$nodeModulesDir = Join-Path $frontendDir "node_modules"
|
||||
if (Test-Path $nodeModulesDir) {
|
||||
$item = Get-Item $nodeModulesDir
|
||||
if ($item.Attributes -match "ReparsePoint") {
|
||||
Print-Info "Removing Unix symlink for node_modules on Windows..."
|
||||
Remove-Item $nodeModulesDir -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
Push-Location $frontendDir
|
||||
$oldPath = $env:PATH
|
||||
try {
|
||||
@@ -419,7 +441,7 @@ try {
|
||||
# ── Done ─────────────────────────────────────────────────────────────────────
|
||||
Write-Host ""
|
||||
Write-Host " ============================================================" -ForegroundColor Green
|
||||
Write-Host " Setup complete! Just double-click start.bat to launch." -ForegroundColor Green
|
||||
Write-Host " Setup complete! Just double-click windows.bat to launch." -ForegroundColor Green
|
||||
Write-Host " ============================================================" -ForegroundColor Green
|
||||
Write-Host ""
|
||||
Read-Host " Press Enter to close..."
|
||||
+24
-1
@@ -361,6 +361,25 @@ if [[ ! -x "$NPM_BIN" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Ensure correct OS-specific node_modules folder is symlinked to avoid conflicts
|
||||
FRONTEND_NODE_MODULES="$FRONTEND_DIR/node_modules"
|
||||
if [[ "$PLATFORM" == "Darwin" ]]; then
|
||||
OS_NODE_MODULES="$FRONTEND_DIR/node_modules_mac"
|
||||
OS_LABEL="node_modules_mac"
|
||||
else
|
||||
OS_NODE_MODULES="$FRONTEND_DIR/node_modules_linux"
|
||||
OS_LABEL="node_modules_linux"
|
||||
fi
|
||||
|
||||
if [[ -d "$FRONTEND_NODE_MODULES" && ! -L "$FRONTEND_NODE_MODULES" ]]; then
|
||||
print_info "Migrating existing node_modules to $OS_LABEL..."
|
||||
mv "$FRONTEND_NODE_MODULES" "$OS_NODE_MODULES"
|
||||
fi
|
||||
|
||||
rm -f "$FRONTEND_NODE_MODULES"
|
||||
mkdir -p "$OS_NODE_MODULES"
|
||||
ln -sf "$OS_LABEL" "$FRONTEND_NODE_MODULES"
|
||||
|
||||
cd "$FRONTEND_DIR"
|
||||
export PATH="$NODE_DIR/bin:$PATH"
|
||||
|
||||
@@ -384,7 +403,11 @@ fi
|
||||
# ── Done ────────────────────────────────────────────────────────────────────
|
||||
echo ""
|
||||
echo " ============================================================"
|
||||
echo " Setup complete! Run ./start.sh to launch."
|
||||
if [[ "$PLATFORM" == "Darwin" ]]; then
|
||||
echo " Setup complete! Run ./mac.sh to launch."
|
||||
else
|
||||
echo " Setup complete! Run ./linux.sh to launch."
|
||||
fi
|
||||
echo " ============================================================"
|
||||
echo ""
|
||||
|
||||
|
||||
Reference in new issue
Block a user