truetype/transcriber
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add CoreML NPU support for Apple Silicon, include Flash Attention toggle, and improve process management for startup scripts.

Browse Source
This commit is contained in:
orailnoor committed 2026-06-14 09:56:16 +05:30
1 parent b55e62be7d
commit 74b0c3ba46
10 files changed
+515 -168
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/frontend/src/App.jsx
+50 -70
View File
@@ -340,75 +340,7 @@ function App() {
}
}, [activeModel]);
const activeTabContent = useMemo(() => {
switch (activeTab) {
case "generator":
return (
<Generator
prompt={prompt}
setPrompt={setPrompt}
negativePrompt={negativePrompt}
setNegativePrompt={setNegativePrompt}
constraints={constraints}
setConstraints={setConstraints}
activeModel={activeModel}
generatedImages={generatedImages}
setGeneratedImages={setGeneratedImages}
isGenerating={isGenerating}
setIsGenerating={setIsGenerating}
generationProgress={generationProgress}
setGenerationProgress={setGenerationProgress}
setActiveTab={setActiveTab}
showAlert={showAlert}
showConfirm={showConfirm}
/>
);
case "models":
return (
<ModelManager
activeModel={activeModel}
setActiveModel={setActiveModel}
serverRunning={serverRunning}
setServerRunning={setServerRunning}
constraints={constraints}
backendOptions={backendOptions}
showAlert={showAlert}
showConfirm={showConfirm}
/>
);
case "constraints":
return (
<ImageConstraints
constraints={constraints}
setConstraints={setConstraints}
activeModel={activeModel}
specs={specs}
backendOptions={backendOptions}
serverRunning={serverRunning}
setServerRunning={setServerRunning}
setActiveModel={setActiveModel}
showAlert={showAlert}
showConfirm={showConfirm}
/>
);
default:
return null;
}
}, [
activeTab,
prompt,
negativePrompt,
constraints,
activeModel,
generatedImages,
isGenerating,
generationProgress,
serverRunning,
specs,
backendOptions,
showAlert,
showConfirm,
]);
// Tab contents are kept mounted to preserve loading/generation state when switching tabs
const sidebarContent = useMemo(() => (
<Sidebar
@@ -507,7 +439,55 @@ function App() {
)}
{/* Dynamic Workspace Container */}
{activeTabContent}
<div style={{ display: activeTab === "generator" ? "flex" : "none", flex: 1, flexDirection: "column", overflow: "hidden" }}>
<Generator
prompt={prompt}
setPrompt={setPrompt}
negativePrompt={negativePrompt}
setNegativePrompt={setNegativePrompt}
constraints={constraints}
setConstraints={setConstraints}
activeModel={activeModel}
generatedImages={generatedImages}
setGeneratedImages={setGeneratedImages}
isGenerating={isGenerating}
setIsGenerating={setIsGenerating}
generationProgress={generationProgress}
setGenerationProgress={setGenerationProgress}
setActiveTab={setActiveTab}
showAlert={showAlert}
showConfirm={showConfirm}
/>
</div>
<div style={{ display: activeTab === "models" ? "flex" : "none", flex: 1, flexDirection: "column", overflow: "hidden" }}>
<ModelManager
activeModel={activeModel}
setActiveModel={setActiveModel}
serverRunning={serverRunning}
setServerRunning={setServerRunning}
constraints={constraints}
backendOptions={backendOptions}
showAlert={showAlert}
showConfirm={showConfirm}
activeTab={activeTab}
/>
</div>
<div style={{ display: activeTab === "constraints" ? "flex" : "none", flex: 1, flexDirection: "column", overflow: "hidden" }}>
<ImageConstraints
constraints={constraints}
setConstraints={setConstraints}
activeModel={activeModel}
specs={specs}
backendOptions={backendOptions}
serverRunning={serverRunning}
setServerRunning={setServerRunning}
setActiveModel={setActiveModel}
showAlert={showAlert}
showConfirm={showConfirm}
/>
</div>
</div>
{dialog && (
app/frontend/src/components/Generator.jsx
+63 -56
View File
@@ -60,6 +60,12 @@ function Generator({
const abortControllerRef = useRef(null);
const hasRealGenerationStepRef = useRef(false);
React.useEffect(() => {
if (constraints.backendType === "apple-npu" || constraints.backendType === "openvino-npu") {
setBaseImage(null);
}
}, [constraints.backendType]);
const handleBaseImageSelect = (e) => {
const file = e.target.files?.[0];
e.target.value = "";
@@ -114,10 +120,9 @@ function Generator({
try {
console.log("Settings out of sync, restarting backend...");
await startServer(activeModel, constraints);
let isReady = false;
let isReady = false;
let crashError = null;
const maxStartupPolls = constraints.backendType === "openvino-npu" ? 1200 : 240;
const maxStartupPolls = (constraints.backendType === "openvino-npu" || constraints.backendType === "apple-npu") ? 1200 : 240;
for (let i = 0; i < maxStartupPolls; i++) {
const status = await getBackendStatus();
if (status.loading) {
@@ -610,61 +615,63 @@ function Generator({
/>
</div>
<div className="m3-text-field">
<label className="m3-text-field-label">Base Image (Optional)</label>
{baseImage ? (
<div style={{ display: "flex", gap: "12px", alignItems: "center", padding: "12px", background: "var(--md-sys-color-surface-variant)", borderRadius: "var(--md-shape-corner-medium)", border: "1px solid var(--md-sys-color-outline-variant)" }}>
<img src={baseImage} alt="Base" style={{ width: "64px", height: "64px", objectFit: "cover", borderRadius: "8px", flexShrink: 0 }} />
<div style={{ flex: 1, minWidth: 0 }}>
<div style={{ fontWeight: 600, fontSize: "0.85rem" }}>Base image ready</div>
<div style={{ fontSize: "0.75rem", color: "var(--md-sys-color-outline)" }}>
The AI will redraw this image guided by your prompt.
{constraints.backendType !== "apple-npu" && constraints.backendType !== "openvino-npu" && (
<div className="m3-text-field">
<label className="m3-text-field-label">Base Image (Optional)</label>
{baseImage ? (
<div style={{ display: "flex", gap: "12px", alignItems: "center", padding: "12px", background: "var(--md-sys-color-surface-variant)", borderRadius: "var(--md-shape-corner-medium)", border: "1px solid var(--md-sys-color-outline-variant)" }}>
<img src={baseImage} alt="Base" style={{ width: "64px", height: "64px", objectFit: "cover", borderRadius: "8px", flexShrink: 0 }} />
<div style={{ flex: 1, minWidth: 0 }}>
<div style={{ fontWeight: 600, fontSize: "0.85rem" }}>Base image ready</div>
<div style={{ fontSize: "0.75rem", color: "var(--md-sys-color-outline)" }}>
The AI will redraw this image guided by your prompt.
</div>
</div>
<button type="button" className="m3-btn m3-btn-error" style={{ height: "34px", flexShrink: 0 }} onClick={handleClearBaseImage} disabled={isGenerating}>
<Trash2 size={14} />
<span>Remove</span>
</button>
</div>
) : (
<label className="import-box" style={{ margin: 0, padding: "16px", cursor: isGenerating ? "not-allowed" : "pointer" }}>
<input
type="file"
accept="image/png,image/jpeg,image/webp"
style={{ display: "none" }}
onChange={handleBaseImageSelect}
disabled={isGenerating}
/>
<ImagePlus className="import-icon" />
<span style={{ fontWeight: 600 }}>Upload a base image</span>
<span style={{ fontSize: "0.75rem", color: "var(--md-sys-color-outline)", textAlign: "center" }}>
Optional. Generate a new image based on one of your own.
</span>
</label>
)}
{baseImage && (
<div style={{ marginTop: "12px" }}>
<div style={{ display: "flex", justifyContent: "space-between", fontSize: "0.8rem", marginBottom: "4px" }}>
<span>Transformation strength</span>
<span>{constraints.denoisingStrength.toFixed(2)}</span>
</div>
<input
type="range"
min="0.1"
max="1"
step="0.05"
value={constraints.denoisingStrength}
onChange={(e) => setConstraints((prev) => ({ ...prev, denoisingStrength: parseFloat(e.target.value) }))}
disabled={isGenerating}
style={{ width: "100%" }}
/>
<div style={{ fontSize: "0.72rem", color: "var(--md-sys-color-outline)", marginTop: "2px" }}>
Lower = closer to your image; higher = more creative freedom. 0.7 is a good start.
</div>
</div>
<button type="button" className="m3-btn m3-btn-error" style={{ height: "34px", flexShrink: 0 }} onClick={handleClearBaseImage} disabled={isGenerating}>
<Trash2 size={14} />
<span>Remove</span>
</button>
</div>
) : (
<label className="import-box" style={{ margin: 0, padding: "16px", cursor: isGenerating ? "not-allowed" : "pointer" }}>
<input
type="file"
accept="image/png,image/jpeg,image/webp"
style={{ display: "none" }}
onChange={handleBaseImageSelect}
disabled={isGenerating}
/>
<ImagePlus className="import-icon" />
<span style={{ fontWeight: 600 }}>Upload a base image</span>
<span style={{ fontSize: "0.75rem", color: "var(--md-sys-color-outline)", textAlign: "center" }}>
Optional. Generate a new image based on one of your own.
</span>
</label>
)}
{baseImage && (
<div style={{ marginTop: "12px" }}>
<div style={{ display: "flex", justifyContent: "space-between", fontSize: "0.8rem", marginBottom: "4px" }}>
<span>Transformation strength</span>
<span>{constraints.denoisingStrength.toFixed(2)}</span>
</div>
<input
type="range"
min="0.1"
max="1"
step="0.05"
value={constraints.denoisingStrength}
onChange={(e) => setConstraints((prev) => ({ ...prev, denoisingStrength: parseFloat(e.target.value) }))}
disabled={isGenerating}
style={{ width: "100%" }}
/>
<div style={{ fontSize: "0.72rem", color: "var(--md-sys-color-outline)", marginTop: "2px" }}>
Lower = closer to your image; higher = more creative freedom. 0.7 is a good start.
</div>
</div>
)}
</div>
)}
</div>
)}
{/* Configuration Status Chips (Material 3 style) */}
<div className="m3-text-field">
app/frontend/src/components/ImageConstraints.jsx
+34 -7
View File
@@ -36,6 +36,8 @@ function ImageConstraints({
const availableBackends = backendOptions?.options?.length
? backendOptions.options
: [{ id: "cpu", label: "CPU", available: true }];
const isMac = availableBackends.some(b => b.id === "metal" || b.id === "apple-npu") ||
(specs?.os_name && (specs.os_name.toLowerCase().includes("darwin") || specs.os_name.toLowerCase().includes("mac")));
useEffect(() => {
if (isOpenVinoNpu && constraints.steps > 8) {
@@ -185,7 +187,7 @@ function ImageConstraints({
}}>
<Info size={16} style={{ color: "rgb(251, 191, 36)", flexShrink: 0, marginTop: "2px" }} />
<div>
<strong>Low-VRAM Protection Active:</strong> Capped at 512x512 for the loaded SD 1.x model (<code>{activeModel}</code>). Generating at larger sizes (like 1024x1024) is disabled to prevent out-of-memory crashes on your {specs && specs.gpu_name && !specs.gpu_name.includes("Loading") ? specs.gpu_name : "graphics processor"}.
<strong>Graphics Memory Protection Active:</strong> Locked at 512x512 size for this model (<code>{activeModel}</code>). Creating larger images (like 1024x1024) is disabled to prevent your {specs && specs.gpu_name && !specs.gpu_name.includes("Loading") ? specs.gpu_name : "graphics processor"} from running out of memory and crashing.
</div>
</div>
)}
@@ -357,9 +359,13 @@ function ImageConstraints({
</div>
))}
</div>
<span style={{ fontSize: "0.75rem", color: "var(--md-sys-color-outline)", marginTop: "4px", lineHeight: 1.35 }}>
CPU is slow but safest. Vulkan works on supported GPUs. CUDA is shown only when NVIDIA CUDA support is available.
</span>
<span style={{ fontSize: "0.75rem", color: "var(--md-sys-color-outline)", marginTop: "4px", lineHeight: 1.35 }}>
{isMac ? (
"CPU is slow but safest. Metal uses your Mac's graphics card. Apple Neural Engine (NPU) is highly power-efficient and fast."
) : (
"CPU is slow but safest. Vulkan works on supported GPUs. CUDA is shown only when NVIDIA CUDA support is available."
)}
</span>
{constraints.backendType === "cuda" && specs?.gpu_name && String(specs.gpu_name).toLowerCase().includes("gtx") && (
<div style={{
marginTop: "12px",
@@ -386,7 +392,7 @@ function ImageConstraints({
</div>
<div className="m3-text-field" style={{ marginTop: "20px" }}>
<label className="m3-text-field-label">Memory Optimization (GPU VRAM)</label>
<label className="m3-text-field-label">Memory Optimization (Graphics Memory)</label>
<div style={{ display: "flex", flexDirection: "column", gap: "14px", marginTop: "10px" }}>
<label style={{ display: "flex", alignItems: "flex-start", gap: "10px", cursor: "pointer", fontSize: "0.85rem" }}>
@@ -405,7 +411,7 @@ function ImageConstraints({
<div>
<strong style={{ color: "var(--md-sys-color-on-surface)" }}>Enable VAE Tiling</strong>
<div style={{ fontSize: "0.75rem", color: "var(--md-sys-color-outline)", marginTop: "2px", lineHeight: 1.35 }}>
Processes VAE decoding in smaller tiles. Drastically reduces VRAM usage (from 2GB+ down to ~100MB) with no speed loss. Highly recommended for GPUs with 4GB-6GB VRAM.
Builds the image in smaller, bite-sized sections. This heavily reduces graphics memory usage with no loss in speed. Highly recommended for computers with standard graphics cards.
</div>
</div>
</label>
@@ -426,7 +432,28 @@ function ImageConstraints({
<div>
<strong style={{ color: "var(--md-sys-color-on-surface)" }}>Run VAE on CPU</strong>
<div style={{ fontSize: "0.75rem", color: "var(--md-sys-color-outline)", marginTop: "2px", lineHeight: 1.35 }}>
Offloads the heavy VAE decoder computation from GPU VRAM to system memory (RAM). Saves ~2GB of VRAM, but makes the final decoding stage slightly slower.
Performs the final image rendering step using your computer's main memory (RAM) instead of graphics memory. Saves graphics memory, but makes the final stage of creating the image slightly slower.
</div>
</div>
</label>
<label style={{ display: "flex", alignItems: "flex-start", gap: "10px", cursor: "pointer", fontSize: "0.85rem" }}>
<input
type="checkbox"
checked={constraints.useFlashAttn !== false}
onChange={(e) => updateConstraint("useFlashAttn", e.target.checked)}
style={{
width: "16px",
height: "16px",
marginTop: "3px",
accentColor: "var(--md-sys-color-primary)",
cursor: "pointer"
}}
/>
<div>
<strong style={{ color: "var(--md-sys-color-on-surface)" }}>Enable Flash Attention</strong>
<div style={{ fontSize: "0.75rem", color: "var(--md-sys-color-outline)", marginTop: "2px", lineHeight: 1.35 }}>
Accelerates generation using memory-efficient attention. On some specific Mac models or GPUs, this may cause a slight slowdown, so you can disable it if needed.
</div>
</div>
</label>
app/frontend/src/components/ModelManager.jsx
+91 -11
View File
@@ -27,7 +27,7 @@ const MODEL_LIBRARY = [
],
},
{
group: "SD 1.5 - Ultra Fast / Low VRAM",
group: "SD 1.5 - Ultra Fast / Low Memory",
items: [
{
name: "DreamShaper 8",
@@ -79,7 +79,24 @@ const OPENVINO_MODEL_LIBRARY = [
},
];
function ModelManager({ activeModel, setActiveModel, serverRunning, setServerRunning, constraints, backendOptions, showAlert = async ({ message }) => window.alert(message), showConfirm = async ({ message }) => window.confirm(message) }) {
const COREML_MODEL_LIBRARY = [
{
group: "Apple Silicon NPU - CoreML Test",
items: [
{
name: "Stable Diffusion v1.5 CoreML (split_einsum, palettized 6-bit)",
filename: "coreml-stable-diffusion-v1-5-palettized_split_einsum_v2_compiled",
format: "CoreML",
approxSize: "2.0 GB",
resolution: "512x512",
notes: "Apple Silicon ANE (Neural Engine) optimized 6-bit palettized Stable Diffusion 1.5 model. Unzips automatically on download completion and runs extremely fast on Mac NPUs.",
url: "https://huggingface.co/apple/coreml-stable-diffusion-v1-5-palettized/resolve/main/coreml-stable-diffusion-v1-5-palettized_split_einsum_v2_compiled.zip",
},
],
},
];
function ModelManager({ activeModel, setActiveModel, serverRunning, setServerRunning, constraints, backendOptions, showAlert = async ({ message }) => window.alert(message), showConfirm = async ({ message }) => window.confirm(message), activeTab }) {
const [localModels, setLocalModels] = useState([]);
const [isLoadingModels, setIsLoadingModels] = useState(true);
const [downloadingModelId, setDownloadingModelId] = useState(null);
@@ -96,12 +113,42 @@ function ModelManager({ activeModel, setActiveModel, serverRunning, setServerRun
const [pendingLoadModel, setPendingLoadModel] = useState(null);
const [backendInfo, setBackendInfo] = useState({ backendMode: "", backendBinary: "", backendDevice: "" });
const cancelLoadRef = React.useRef(false);
const handleCancelLoad = async () => {
cancelLoadRef.current = true;
try {
await stopServer();
} catch (err) {
console.warn("Failed to stop server on cancel:", err);
}
};
const modelNames = localModels.map((model) => normalizeModel(model).filename);
const isBusy = loadingModelId !== null || isUnloading;
const openvinoSupported = Boolean(backendOptions?.openvinoNpu?.supported);
const visibleModelLibrary = openvinoSupported ? [...MODEL_LIBRARY, ...OPENVINO_MODEL_LIBRARY] : MODEL_LIBRARY;
const appleNpuSupported = Boolean(
backendOptions?.options?.some((opt) => opt.id === "apple-npu") ||
backendOptions?.unavailable?.some((opt) => opt.id === "apple-npu")
);
let visibleModelLibrary = [...MODEL_LIBRARY];
if (openvinoSupported) {
visibleModelLibrary = [...visibleModelLibrary, ...OPENVINO_MODEL_LIBRARY];
}
if (appleNpuSupported) {
visibleModelLibrary = [...visibleModelLibrary, ...COREML_MODEL_LIBRARY];
}
const getLocalModelInfo = (modelId) => localModels.map(normalizeModel).find((model) => model.filename === modelId);
useEffect(() => {
if (activeTab === "models") {
fetchModels();
checkActiveDownload();
}
}, [activeTab]);
useEffect(() => {
fetchModels();
checkActiveDownload();
@@ -258,6 +305,10 @@ function ModelManager({ activeModel, setActiveModel, serverRunning, setServerRun
}
return;
}
if (model.format === "CoreML") {
downloadByUrl(model.url, `${model.filename}.zip`);
return;
}
downloadByUrl(model.url, model.filename);
};
@@ -306,6 +357,7 @@ function ModelManager({ activeModel, setActiveModel, serverRunning, setServerRun
return;
}
cancelLoadRef.current = false;
setLoadingModelId(modelId);
setModelLoadProgress({
progress: 0,
@@ -320,7 +372,18 @@ function ModelManager({ activeModel, setActiveModel, serverRunning, setServerRun
});
try {
const modelInfo = getLocalModelInfo(modelId);
const loadConstraints = modelInfo?.backendType === "openvino-npu"
const isCoreMLModel = modelInfo?.format === "CoreML" || modelInfo?.backendType === "apple-npu";
const loadConstraints = isCoreMLModel
? {
...constraints,
backendType: "apple-npu",
useGpu: true,
width: 512,
height: 512,
steps: constraints.steps || 20,
cfgScale: constraints.cfgScale || 7,
}
: modelInfo?.backendType === "openvino-npu"
? {
...constraints,
backendType: "openvino-npu",
@@ -337,8 +400,12 @@ function ModelManager({ activeModel, setActiveModel, serverRunning, setServerRun
let isReady = false;
let crashError = null;
const isOpenVinoModel = modelInfo?.backendType === "openvino-npu";
const maxStartupPolls = isOpenVinoModel ? 1200 : 240;
const maxStartupPolls = (isOpenVinoModel || isCoreMLModel) ? 1200 : 240;
for (let i = 0; i < maxStartupPolls; i++) {
if (cancelLoadRef.current) {
crashError = "Model load cancelled by the user.";
break;
}
const status = await getBackendStatus();
if (status.loading) {
setModelLoadProgress({
@@ -385,7 +452,9 @@ function ModelManager({ activeModel, setActiveModel, serverRunning, setServerRun
}
} catch (e) {
console.error("Failed to load model:", e);
showAlert({ title: "Model Load Failed", message: e.message || String(e), danger: true });
if (e.message !== "Model load cancelled by the user.") {
showAlert({ title: "Model Load Failed", message: e.message || String(e), danger: true });
}
} finally {
setTimeout(() => setModelLoadProgress(null), 800);
setLoadingModelId(null);
@@ -580,9 +649,18 @@ function ModelManager({ activeModel, setActiveModel, serverRunning, setServerRun
{modelLoadProgress && (
<div className="m3-card" style={{ borderLeft: "4px solid var(--md-sys-color-secondary)", marginTop: "16px" }}>
<h4 style={{ fontWeight: 700, marginBottom: "8px" }}>
Loading Model: {modelLoadProgress.model || loadingModelId}
</h4>
<div style={{ display: "flex", justifyContent: "space-between", alignItems: "center", gap: "16px", marginBottom: "12px" }}>
<h4 style={{ fontWeight: 700, margin: 0 }}>
Loading Model: {modelLoadProgress.model || loadingModelId}
</h4>
<button
className="m3-btn m3-btn-error"
style={{ height: "30px", padding: "0 12px", fontSize: "0.78rem", borderRadius: "8px" }}
onClick={handleCancelLoad}
>
Cancel Loading
</button>
</div>
{(modelLoadProgress.backendMode || modelLoadProgress.backendBinary || modelLoadProgress.device) && (
<div style={{ display: "flex", flexWrap: "wrap", gap: "8px", marginBottom: "10px" }}>
{modelLoadProgress.backendMode && (
@@ -611,7 +689,9 @@ function ModelManager({ activeModel, setActiveModel, serverRunning, setServerRun
</span>
<span>
{modelLoadProgress.total > 0
? `Loaded ${modelLoadProgress.current} / ${modelLoadProgress.total} tensors`
? modelLoadProgress.total <= 10
? `Loading components: ${modelLoadProgress.current} / ${modelLoadProgress.total} (${Math.round(modelLoadProgress.progress)}%)`
: `Loaded ${modelLoadProgress.current} / ${modelLoadProgress.total} tensors`
: `${Math.round(modelLoadProgress.progress)}%`}
</span>
</div>
@@ -723,7 +803,7 @@ function ModelManager({ activeModel, setActiveModel, serverRunning, setServerRun
<div style={{ display: "grid", gridTemplateColumns: "repeat(auto-fit, minmax(280px, 1fr))", gap: "12px" }}>
{section.items.map((model) => {
const installed = modelNames.includes(model.filename);
const downloading = downloadingModelId === model.filename;
const downloading = downloadingModelId === model.filename || downloadingModelId === `${model.filename}.zip`;
return (
<div key={model.filename} style={{ display: "flex", flexDirection: "column", gap: "10px", padding: "14px", background: "var(--md-sys-color-surface-variant)", border: "1px solid var(--md-sys-color-outline-variant)", borderRadius: "var(--md-shape-corner-medium)" }}>
<div style={{ display: "flex", flexDirection: "column", gap: "4px" }}>
linux.sh
+10 -4
View File
@@ -102,9 +102,12 @@ if [[ -n "$SETUP_REASON" ]]; then
echo ""
read -rp " Press Enter to continue, or Ctrl+C to cancel."
# Clear any existing frontend server process
if command -v fuser >/dev/null 2>&1; then
# Clear any existing frontend and backend server processes
if command -v lsof >/dev/null 2>&1; then
lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
elif command -v fuser >/dev/null 2>&1; then
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
fuser -k "8080/tcp" >/dev/null 2>&1 || true
fi
if ! bash "$SETUP_SCRIPT" $MAX_PERF_FLAG; then
@@ -123,9 +126,12 @@ echo " LOCAL AI IMAGE GENERATOR | Launching..."
echo " ============================================================"
echo ""
# Clear frontend port
if command -v fuser >/dev/null 2>&1; then
# Clear frontend and backend ports
if command -v lsof >/dev/null 2>&1; then
lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
elif command -v fuser >/dev/null 2>&1; then
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
fuser -k "8080/tcp" >/dev/null 2>&1 || true
fi
# Start the server
mac.sh
+10 -4
View File
@@ -74,9 +74,12 @@ if [[ -n "$SETUP_REASON" ]]; then
echo ""
read -rp " Press Enter to continue, or Ctrl+C to cancel."
# Clear any existing frontend server process
if command -v fuser >/dev/null 2>&1; then
# Clear any existing frontend and backend server processes
if command -v lsof >/dev/null 2>&1; then
lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
elif command -v fuser >/dev/null 2>&1; then
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
fuser -k "8080/tcp" >/dev/null 2>&1 || true
fi
if ! bash "$SETUP_SCRIPT"; then
@@ -95,9 +98,12 @@ echo " LOCAL AI IMAGE GENERATOR | Launching..."
echo " ============================================================"
echo ""
# Clear frontend port
if command -v fuser >/dev/null 2>&1; then
# Clear frontend and backend ports
if command -v lsof >/dev/null 2>&1; then
lsof -t -i:"${FRONTEND_PORT}" -i:8080 | xargs kill -9 >/dev/null 2>&1 || true
elif command -v fuser >/dev/null 2>&1; then
fuser -k "${FRONTEND_PORT}/tcp" >/dev/null 2>&1 || true
fuser -k "8080/tcp" >/dev/null 2>&1 || true
fi
# Start the server
scripts/serve.cjs
+163 -13
View File
@@ -778,6 +778,103 @@ function getVulkanUnavailableReason() {
return "Installed, but this binary did not register a Vulkan backend on this machine.";
}
let cachedCoreMLPythonPath = null;
let cachedCoreMLAvailable = null;
function getCoreMLPythonPath() {
if (cachedCoreMLAvailable !== null) {
return cachedCoreMLPythonPath;
}
if (osPlatform !== "darwin") {
cachedCoreMLAvailable = false;
cachedCoreMLPythonPath = null;
return null;
}
// 1. Check environment variable COREML_PYTHON
if (process.env.COREML_PYTHON && fs.existsSync(process.env.COREML_PYTHON)) {
cachedCoreMLPythonPath = process.env.COREML_PYTHON;
cachedCoreMLAvailable = true;
return cachedCoreMLPythonPath;
}
// 2. Check local/in-project venvs
const localPaths = [
path.join(ROOT, "app", "backend", "mac", "coreml_venv", "bin", "python"),
path.join(ROOT, "app", "backend", "mac", "venv", "bin", "python"),
path.join(ROOT, "venv", "bin", "python"),
];
for (const p of localPaths) {
if (fs.existsSync(p)) {
cachedCoreMLPythonPath = p;
cachedCoreMLAvailable = true;
return p;
}
}
// 3. Check dynamically in home directory (as fallback to user's setup)
const homeDir = os.homedir();
const homePaths = [
path.join(homeDir, "workspace", "coreml_conversion", "venv", "bin", "python"),
path.join(homeDir, "ml-stable-diffusion", "venv", "bin", "python"),
];
for (const p of homePaths) {
if (fs.existsSync(p)) {
cachedCoreMLPythonPath = p;
cachedCoreMLAvailable = true;
return p;
}
}
// 4. Fallback: Probe system python3 if it has required imports
try {
const probeResult = spawnSync("python3", [
"-c",
"import python_coreml_stable_diffusion, diffusers, transformers"
], { timeout: 2000 });
if (probeResult.status === 0) {
// Find absolute path of python3
const whichResult = spawnSync("which", ["python3"], { encoding: "utf8" });
const python3Path = whichResult.stdout ? whichResult.stdout.trim() : "python3";
cachedCoreMLPythonPath = python3Path;
cachedCoreMLAvailable = true;
return python3Path;
}
} catch (err) {
// python3 not in path or crashed
}
cachedCoreMLAvailable = false;
cachedCoreMLPythonPath = null;
return null;
}
function getCoreMLNpuInfo() {
const isAppleSilicon = osPlatform === "darwin" && os.arch() === "arm64";
if (!isAppleSilicon) {
return { supported: false, reason: "Apple Silicon ANE (NPU) is only available on Apple Silicon macOS devices." };
}
const pythonPath = getCoreMLPythonPath();
if (!pythonPath) {
return {
supported: true,
ready: false,
reason: "CoreML Python environment is not set up. Run scripts/setup-coreml-npu.sh first.",
};
}
return {
supported: true,
ready: true,
python: pythonPath,
};
}
function getBackendOptions() {
if (cachedBackendOptions) return cachedBackendOptions;
@@ -798,14 +895,14 @@ function getBackendOptions() {
const cpuInstalled = osPlatform === "linux" && fs.existsSync(BACKEND_PATHS.linuxCpu);
const metalInstalled = osPlatform === "darwin" && fs.existsSync(BACKEND_PATHS.mac);
const metalAvailable = metalInstalled;
const coremlAvailable = osPlatform === "darwin" && fs.existsSync("/Users/orailnoor/workspace/coreml_conversion/venv/bin/python");
const coremlNpu = getCoreMLNpuInfo();
const openvinoNpu = getOpenVinoNpuInfo();
const openvinoModels = getOpenVinoModelInfo();
const openvinoNpuAvailable = openvinoNpu.supported && openvinoModels.some((model) => model.installed);
const options = [{ id: "cpu", label: "CPU", available: true }];
if (metalAvailable) options.push({ id: "metal", label: "Metal GPU", available: true });
if (coremlAvailable) options.push({ id: "apple-npu", label: "Apple Neural Engine (NPU)", available: true });
if (coremlNpu.supported && coremlNpu.ready) options.push({ id: "apple-npu", label: "Apple Neural Engine (NPU)", available: true });
if (vulkanAvailable) options.push({ id: "vulkan", label: "Vulkan GPU", available: true });
if (rocmAvailable) options.push({ id: "rocm", label: "ROCm GPU (AMD)", available: true });
if (cudaAvailable) options.push({ id: "cuda", label: "CUDA GPU", available: true });
@@ -824,6 +921,9 @@ function getBackendOptions() {
if (metalInstalled && !metalAvailable) {
unavailable.push({ id: "metal", label: "Metal GPU", reason: "Installed, but Metal backend validation failed." });
}
if (coremlNpu.supported && !coremlNpu.ready) {
unavailable.push({ id: "apple-npu", label: "Apple Neural Engine (NPU)", reason: coremlNpu.reason });
}
if (openvinoNpu.supported && !openvinoNpuAvailable) {
unavailable.push({ id: "openvino-npu", label: "NPU (OpenVINO)", reason: "Runtime is ready, but no OpenVINO NPU model is downloaded." });
} else if (!openvinoNpu.supported && (osPlatform === "win32" || osPlatform === "linux")) {
@@ -831,7 +931,7 @@ function getBackendOptions() {
}
let defaultBackend = "cpu";
if (coremlAvailable) {
if (coremlNpu.supported && coremlNpu.ready) {
defaultBackend = "apple-npu";
} else if (metalAvailable) {
defaultBackend = "metal";
@@ -859,6 +959,7 @@ function getBackendOptions() {
openvinoNpuAvailable,
openvinoNpu,
openvinoModels,
coremlNpu,
defaultBackendType: defaultBackend,
};
return cachedBackendOptions;
@@ -948,7 +1049,7 @@ function selectBackendPath(useGpu, backendType = "auto", modelPath = "") {
return BACKEND_PATH;
}
if (osPlatform === "darwin") {
if (resolvedType === "apple-npu") return "/Users/orailnoor/workspace/coreml_conversion/venv/bin/python";
if (resolvedType === "apple-npu") return getCoreMLPythonPath();
if (fs.existsSync(BACKEND_PATHS.mac)) return BACKEND_PATHS.mac;
return BACKEND_PATH;
}
@@ -963,9 +1064,10 @@ function resolveBackendType(useGpu, backendType = "auto", modelPath = "") {
if (modelPath && osPlatform === "darwin") {
const lower = modelPath.toLowerCase();
const isDir = fs.existsSync(modelPath) && fs.statSync(modelPath).isDirectory();
if (lower.endsWith(".safetensors") || lower.endsWith(".gguf")) {
if (requestedType === "apple-npu") requestedType = "metal";
} else if (lower.endsWith(".coreml") || lower.endsWith(".mlpackage") || lower.endsWith(".mlmodelc")) {
} else if (lower.endsWith(".coreml") || lower.endsWith(".mlpackage") || lower.endsWith(".mlmodelc") || isDir) {
if (requestedType === "metal") requestedType = "apple-npu";
}
}
@@ -1222,9 +1324,11 @@ async function generateWithOpenVino(body) {
function startBackendReadyPoll() {
let attempts = 0;
const isNpu = currentSettings.backendType === "apple-npu" || currentSettings.backendType === "openvino-npu";
const maxAttempts = isNpu ? 1200 : 240;
const interval = setInterval(async () => {
attempts += 1;
if (!backendProc || backendReady || attempts > 240) {
if (!backendProc || backendReady || attempts > maxAttempts) {
clearInterval(interval);
return;
}
@@ -1367,9 +1471,6 @@ async function startBackend(settings = {}) {
}
args.push("--rng", "cpu", "--sampler-rng", "cpu");
} else if (requestedBackend === "metal") {
if (supportsFlags) {
args.push("--backend", "metal0", "--params-backend", "metal0");
}
args.push("--rng", "cpu", "--sampler-rng", "cpu");
}
@@ -1451,14 +1552,17 @@ async function startBackend(settings = {}) {
const current = parseInt(loadMatch[1], 10);
const total = parseInt(loadMatch[2], 10);
const progress = total > 0 ? Math.round((current / total) * 100) : 0;
const isCoreML = currentSettings.backendType === "apple-npu" || total <= 10;
const phaseDesc = isCoreML ? stripAnsi(loadMatch[3]).trim() : "Loading model weights...";
const speedDesc = isCoreML ? "" : stripAnsi(loadMatch[3]).trim();
backendLoadState = {
...backendLoadState,
active: !backendReady,
phase: "Loading model weights...",
phase: phaseDesc,
progress: Math.max(backendLoadState.progress, Math.min(99, progress)),
current,
total,
speed: stripAnsi(loadMatch[3]).trim(),
speed: speedDesc,
};
}
@@ -1887,7 +1991,41 @@ const MIME = {
function isModelFile(filename) {
const lower = String(filename || "").toLowerCase();
return lower.endsWith(".safetensors") || lower.endsWith(".gguf") || lower.endsWith(".ckpt") || lower.endsWith(".coreml") || lower.endsWith(".coreml.zip");
if (lower.endsWith(".safetensors") || lower.endsWith(".gguf") || lower.endsWith(".ckpt") || lower.endsWith(".coreml") || lower.endsWith(".coreml.zip")) {
return true;
}
const fullPath = path.join(MODELS, filename);
try {
const stat = fs.statSync(fullPath);
if (stat.isDirectory()) {
const files = fs.readdirSync(fullPath).map(f => f.toLowerCase());
const hasTextEncoder = files.includes("text_encoder.mlpackage") ||
files.includes("text_encoder.mlmodelc") ||
files.includes("textencoder.mlpackage") ||
files.includes("textencoder.mlmodelc");
const hasUnet = files.includes("unet.mlpackage") ||
files.includes("unet.mlmodelc");
if (hasTextEncoder && hasUnet) return true;
const checkSubdirs = ["split_einsum/packages", "split_einsum/compiled", "original/packages", "original/compiled", "split_einsum", "original"];
for (const subdir of checkSubdirs) {
const subPath = path.join(fullPath, subdir);
if (fs.existsSync(subPath)) {
const subFiles = fs.readdirSync(subPath).map(f => f.toLowerCase());
const subTextEncoder = subFiles.includes("text_encoder.mlpackage") ||
subFiles.includes("text_encoder.mlmodelc") ||
subFiles.includes("textencoder.mlpackage") ||
subFiles.includes("textencoder.mlmodelc");
const subUnet = subFiles.includes("unet.mlpackage") ||
subFiles.includes("unet.mlmodelc");
if (subTextEncoder && subUnet) return true;
}
}
}
} catch (_) {}
return false;
}
function formatBytes(bytes) {
@@ -1984,7 +2122,19 @@ function describeBackendExitCode(code, backendPath) {
function getModelInfo(filename) {
const safeFilename = path.basename(filename || "");
const stats = fs.statSync(path.join(MODELS, safeFilename));
const fullPath = path.join(MODELS, safeFilename);
const stats = fs.statSync(fullPath);
if (stats.isDirectory()) {
const dirSize = getPathSize(fullPath);
return {
filename: safeFilename,
sizeBytes: dirSize,
size: formatBytes(dirSize),
format: "CoreML",
};
}
return {
filename: safeFilename,
sizeBytes: stats.size,
scripts/setup-coreml-npu.sh
Executable
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env bash
#
# Optional CoreML NPU setup for Apple Silicon macOS systems.
# Sets up a virtual environment and installs Apple's ml-stable-diffusion.
#
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="$(dirname "$SCRIPT_DIR")"
VENV_DIR="$ROOT_DIR/app/backend/mac/coreml_venv"
PYTHON_BIN="$VENV_DIR/bin/python"
echo ""
echo " ============================================================"
echo " Local AI Image Generator - Apple Silicon CoreML NPU Setup"
echo " ============================================================"
echo ""
if [[ "$(uname -s)" != "Darwin" ]]; then
echo " [ERROR] This setup script is for macOS only." >&2
exit 1
fi
if [[ "$(uname -m)" != "arm64" ]]; then
echo " [ERROR] CoreML NPU support requires Apple Silicon (M1/M2/M3/M4/etc. arm64)." >&2
exit 1
fi
if ! command -v python3 >/dev/null 2>&1; then
echo " [ERROR] python3 is required. Install Python 3 first (via Homebrew or Xcode)." >&2
exit 1
fi
if [[ ! -x "$PYTHON_BIN" ]]; then
echo " Creating Python environment: $VENV_DIR"
if ! python3 -m venv "$VENV_DIR"; then
echo " [ERROR] Could not create the virtual environment."
exit 1
fi
fi
echo " Installing CoreML Stable Diffusion dependencies (numpy, coremltools, diffusers)..."
"$PYTHON_BIN" -m pip install --upgrade pip
"$PYTHON_BIN" -m pip install numpy coremltools diffusers transformers huggingface-hub pillow
echo " Installing Apple's python-coreml-stable-diffusion package..."
if ! "$PYTHON_BIN" -m pip install "git+https://github.com/apple/ml-stable-diffusion.git"; then
echo " [ERROR] Failed to install Apple's ml-stable-diffusion repository."
echo " Make sure 'git' is installed on your system."
exit 1
fi
echo ""
echo " Verifying CoreML environment..."
if ! "$PYTHON_BIN" -c "from python_coreml_stable_diffusion.pipeline import CoreMLStableDiffusionPipeline; print(' ANE (CoreML) Pipeline verified successfully!')"; then
echo " [ERROR] Verification failed. Please check the error details above."
exit 1
fi
echo ""
echo " ============================================================"
echo " CoreML NPU setup complete."
echo " Restart the launcher (./mac.sh) and download a CoreML model!"
echo " ============================================================"
echo ""
scripts/setup.sh
+23
View File
@@ -285,6 +285,29 @@ if [[ "$PLATFORM" == "Darwin" ]]; then
rm -rf "$BACKEND_DIR/extracted"
print_ok "macOS Metal backend installed."
fi
# CoreML NPU Environment setup (macOS Apple Silicon only)
print_info "Setting up CoreML Python virtual environment for Apple Silicon ANE (NPU)..."
VENV_DIR="$BACKEND_DIR/coreml_venv"
PYTHON_BIN="$VENV_DIR/bin/python"
if [[ ! -x "$PYTHON_BIN" ]]; then
print_info "Creating Python virtual environment at $VENV_DIR..."
if ! python3 -m venv "$VENV_DIR"; then
print_warn "Could not create the virtual environment for CoreML. CoreML NPU mode will be unavailable."
fi
fi
if [[ -x "$PYTHON_BIN" ]]; then
print_info "Installing CoreML dependencies (this may take a couple of minutes)..."
if "$PYTHON_BIN" -m pip install --upgrade pip >/dev/null 2>&1 && \
"$PYTHON_BIN" -m pip install numpy coremltools diffusers transformers huggingface-hub pillow >/dev/null 2>&1 && \
"$PYTHON_BIN" -m pip install "git+https://github.com/apple/ml-stable-diffusion.git" >/dev/null 2>&1; then
print_ok "CoreML ANE (NPU) environment ready."
else
print_warn "CoreML dependencies installation failed. CoreML NPU mode will be unavailable."
fi
fi
else
VENDOR="$(detect_gpu_vendor)"
print_step 2 $TOTAL_STEPS "Detecting GPU vendor: ${VENDOR:-none}"
windows.bat
+5 -3
View File
@@ -50,8 +50,9 @@ echo.
echo Press any key to continue, or Ctrl+C to cancel.
pause >nul
:: Clear old frontend server process before setup so app/tools/node-win can be replaced
:: Clear old frontend and backend server processes before setup so app/tools/node-win can be replaced
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%FRONTEND_PORT% "') do taskkill /f /pid %%a >nul 2>nul
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":8080 "') do taskkill /f /pid %%a >nul 2>nul
powershell -ExecutionPolicy Bypass -File "%SETUP%"
if errorlevel 1 (
@@ -73,9 +74,10 @@ echo LOCAL AI IMAGE GENERATOR ^| Launching...
echo ============================================================
echo.
:: Clear only the frontend port. The backend auto-selects a free port.
echo Clearing frontend port %FRONTEND_PORT%...
:: Clear frontend and backend ports to prevent address conflicts.
echo Clearing frontend port %FRONTEND_PORT% and backend port 8080...
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":%FRONTEND_PORT% "') do taskkill /f /pid %%a >nul 2>nul
for /f "tokens=5" %%a in ('netstat -aon 2^>nul ^| findstr ":8080 "') do taskkill /f /pid %%a >nul 2>nul
:: Start frontend server + backend manager (serve.cjs manages sd-vulkan.exe)
echo Starting Local AI Image Generator...