feat(npu): Ascend multi-card Ulysses SP with CFG/FSDP support

diffsynth_engine/configs/pipeline.py

@@ -35,6 +35,7 @@ class AttnImpl(Enum):
     SAGE = "sage"  # Sage Attention
     SPARGE = "sparge"  # Sparge Attention
     VSA = "vsa"  # Video Sparse Attention
+    MINDIE = "mindie"  # Mindie Attention
 
 
 @dataclass

diffsynth_engine/models/basic/attention.py

@@ -15,6 +15,7 @@
     SPARGE_ATTN_AVAILABLE,
     VIDEO_SPARSE_ATTN_AVAILABLE,
     AITER_AVAILABLE,
+    MINDIE_AVAILABLE,
 )
 from diffsynth_engine.utils.platform import DTYPE_FP8
 
@@ -107,6 +108,16 @@ def sparge_attn(
         distributed_video_sparse_attn,
     )
 
+if MINDIE_AVAILABLE:
+    from mindiesd.layers.flash_attn.attention_forward import attention_forward
+
+    def mindie_attn(q, k, v, attn_mask=None, scale=None):
+        return attention_forward(
+            query=q, key=k, value=v,
+            attn_mask=attn_mask, scale=scale,
+            fused=True, head_first=False,
+        )
+
 
 def eager_attn(q, k, v, attn_mask=None, scale=None):
     q = q.transpose(1, 2)
@@ -152,6 +163,7 @@ def attention(
         "sage",
         "sparge",
         "vsa",
+        "mindie",
     ]
     flash_attn3_compatible = q.shape[-1] <= FA3_MAX_HEADDIM
     if attn_impl is None or attn_impl == "auto":
@@ -192,6 +204,8 @@ def attention(
                 )
         if XFORMERS_AVAILABLE:
             return xformers_attn(q, k, v, attn_mask=attn_mask, scale=scale)
+        if MINDIE_AVAILABLE:
+            return mindie_attn(q, k, v, attn_mask=attn_mask, scale=scale)
         if SDPA_AVAILABLE:
             return sdpa_attn(q, k, v, attn_mask=attn_mask, scale=scale)
         if FLASH_ATTN_2_AVAILABLE:
@@ -263,6 +277,8 @@ def attention(
                 cdfthreshd=kwargs.get("cdfthreshd", 0.98),
                 pvthreshd=kwargs.get("pvthreshd", 50),
             )
+        if attn_impl == "mindie":
+            return mindie_attn(q, k, v, attn_mask=attn_mask, scale=scale)
         if attn_impl == "vsa":
             return video_sparse_attn(
                 q,
@@ -324,6 +340,44 @@ def forward(
         return self.to_out(out)
 
 
+def _npu_ulysses_mindie_attention(
+    q: torch.Tensor,
+    k: torch.Tensor,
+    v: torch.Tensor,
+    attn_mask: Optional[torch.Tensor] = None,
+    scale: Optional[float] = None,
+):
+    """Ulysses SP on NPU: SeqAllToAll4D + MindIE local attn. ring_degree>1 not supported."""
+    from yunchang.comm.all_to_all import SeqAllToAll4D
+
+    from diffsynth_engine.utils.process_group import get_sp_ring_world_size, get_sp_ulysses_group
+
+    if q.device.type != "npu":
+        raise RuntimeError("mindie long-context attention is only supported on NPU")
+    if not MINDIE_AVAILABLE:
+        raise RuntimeError(
+            "NPU Ulysses sequence parallel requires MindIE attention, but MindIE-SD is not available"
+        )
+    if get_sp_ring_world_size() > 1:
+        raise RuntimeError(
+            "NPU long-context attention currently supports Ulysses only "
+            f"(sp_ring_degree must be 1, got {get_sp_ring_world_size()})"
+        )
+    assert attn_mask is None, "long context attention does not support attention mask"
+
+    # scatter heads (dim=2), gather sequence (dim=1) — same as video_sparse / v1 USP
+    scatter_idx, gather_idx = 2, 1
+    group = get_sp_ulysses_group()
+    q = SeqAllToAll4D.apply(group, q, scatter_idx, gather_idx)
+    k = SeqAllToAll4D.apply(group, k, scatter_idx, gather_idx)
+    v = SeqAllToAll4D.apply(group, v, scatter_idx, gather_idx)
+
+    # Must not call attention() here — it is patched to long_context_attention under SP.
+    out = mindie_attn(q, k, v, attn_mask=attn_mask, scale=scale)
+    out = SeqAllToAll4D.apply(group, out, gather_idx, scatter_idx)
+    return out
+
+
 def long_context_attention(
     q: torch.Tensor,
     k: torch.Tensor,
@@ -354,10 +408,14 @@ def long_context_attention(
         "sage",
         "sparge",
         "vsa",
+        "mindie",
     ]
     assert attn_mask is None, "long context attention does not support attention mask"
     flash_attn3_compatible = q.shape[-1] <= FA3_MAX_HEADDIM
     if attn_impl is None or attn_impl == "auto":
+        # NPU has no FA/yunchang TORCH_EFFICIENT kernel; pick MindIE Ulysses when available.
+        if q.device.type == "npu" and MINDIE_AVAILABLE:
+            return _npu_ulysses_mindie_attention(q, k, v, attn_mask=attn_mask, scale=scale)
         if FLASH_ATTN_3_AVAILABLE:
             if flash_attn3_compatible:
                 return LongContextAttention(attn_type=AttnType.FA3)(q, k, v, softmax_scale=scale)
@@ -378,6 +436,8 @@ def long_context_attention(
             return LongContextAttention(attn_type=AttnType.FA)(q, k, v, softmax_scale=scale)
         raise ValueError("No available long context attention implementation")
     else:
+        if attn_impl == "mindie":
+            return _npu_ulysses_mindie_attention(q, k, v, attn_mask=attn_mask, scale=scale)
         if attn_impl == "fa3" or attn_impl == "fa3_fp8":
             if not flash_attn3_compatible:
                 raise RuntimeError(

diffsynth_engine/pipelines/qwen_image.py

@@ -376,7 +376,9 @@ def update_weights(self, state_dicts: QwenImageStateDicts) -> None:
         self.update_component(self.vae, state_dicts.vae, self.config.device, self.config.vae_dtype)
 
     def compile(self):
-        self.dit.compile_repeated_blocks()
+        from diffsynth_engine.platforms import resolve_platform
+        platform_cls = resolve_platform(self.config.device)
+        self.dit.compile_repeated_blocks(**platform_cls.compile_kwargs())
 
     def load_loras(self, lora_list: List[Tuple[str, float]], fused: bool = True, save_original_weight: bool = False):
         assert self.config.tp_degree is None or self.config.tp_degree == 1, (

diffsynth_engine/platforms/__init__.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import platform as host_platform
+from functools import lru_cache
+from typing import Type
+
+import torch
+
+from .ascend import (
+    AscendPlatform,
+    probe_ascend_capabilities,
+    probe_ascend_feature,
+    reset_ascend_capability_cache,
+)
+from .base import PlatformBackend, PlatformCapabilities
+
+
+class CPUPlatform(PlatformBackend):
+    name = "cpu"
+    device_type = "cpu"
+
+
+class CUDAPlatform(PlatformBackend):
+    name = "cuda"
+    device_type = "cuda"
+
+    @classmethod
+    def is_available(cls) -> bool:
+        return torch.cuda.is_available()
+
+    @classmethod
+    def set_device(cls, index: int | str | torch.device) -> None:
+        torch.cuda.set_device(index)
+
+    @classmethod
+    def synchronize(cls) -> None:
+        torch.cuda.synchronize()
+
+    @classmethod
+    def empty_cache(cls) -> None:
+        torch.cuda.empty_cache()
+
+    @classmethod
+    def distributed_backend(cls) -> str:
+        return "nccl"
+
+
+class ROCmPlatform(CUDAPlatform):
+    name = "rocm"
+
+
+class MPSPlatform(PlatformBackend):
+    name = "mps"
+    device_type = "mps"
+
+    @classmethod
+    def is_available(cls) -> bool:
+        return torch.backends.mps.is_available()
+
+    @classmethod
+    def synchronize(cls) -> None:
+        torch.mps.synchronize()
+
+    @classmethod
+    def empty_cache(cls) -> None:
+        torch.mps.empty_cache()
+
+
+_PLATFORM_REGISTRY: dict[str, Type[PlatformBackend]] = {
+    "cpu": CPUPlatform,
+    "cuda": ROCmPlatform if torch.version.hip else CUDAPlatform,
+    "mps": MPSPlatform,
+    "npu": AscendPlatform,
+}
+
+
+def register_platform(device_type: str, platform_cls: Type[PlatformBackend], *, overwrite: bool = False) -> None:
+    if device_type in _PLATFORM_REGISTRY and not overwrite:
+        raise ValueError(f"Platform for device type {device_type!r} is already registered")
+    _PLATFORM_REGISTRY[device_type] = platform_cls
+
+
+@lru_cache(maxsize=None)
+def auto_detect_device() -> str:
+    """auto detect device type in order of cuda(gpu/rocm), npu, mps, cpu"""
+    for device_type in ("cuda", "npu", "mps", "cpu"):
+        try:
+            if resolve_platform(device_type).is_available():
+                return device_type
+        except Exception:
+            continue
+    return "cpu"
+
+
+def get_device_type(device: str | torch.device | None = None) -> str:
+    if device is None or (isinstance(device, str) and device.lower() in ("auto", "")):
+        return auto_detect_device()
+    if isinstance(device, torch.device):
+        return device.type
+    return str(device).split(":", 1)[0].lower()
+
+
+def resolve_platform(device: str | torch.device) -> Type[PlatformBackend]:
+    device_type = get_device_type(device)
+    try:
+        return _PLATFORM_REGISTRY[device_type]
+    except KeyError as exc:
+        available = ", ".join(sorted(_PLATFORM_REGISTRY))
+        raise ValueError(f"Unsupported device type {device_type!r}. Registered device types: {available}") from exc
+
+
+def get_preferred_fp8_dtype(device: str | torch.device = "cuda") -> torch.dtype:
+    platform_cls = resolve_platform(device)
+    if platform_cls is ROCmPlatform and platform_cls.is_available():
+        properties = torch.cuda.get_device_properties(0)
+        if "gfx94" in properties.gcnArchName:
+            return torch.float8_e4m3fnuz
+    return torch.float8_e4m3fn
+
+
+def pin_memory(
+    tensor: torch.Tensor,
+    device: str | torch.device | None = None,
+) -> torch.Tensor:
+    if host_platform.system() != "Linux":
+        return tensor
+    platform_cls = resolve_platform(get_device_type(device))
+    return platform_cls.pin_memory(tensor)
+
+
+__all__ = [
+    "AscendPlatform",
+    "CPUPlatform",
+    "CUDAPlatform",
+    "MPSPlatform",
+    "PlatformBackend",
+    "PlatformCapabilities",
+    "ROCmPlatform",
+    "auto_detect_device",
+    "get_device_type",
+    "get_preferred_fp8_dtype",
+    "pin_memory",
+    "probe_ascend_capabilities",
+    "probe_ascend_feature",
+    "register_platform",
+    "reset_ascend_capability_cache",
+    "resolve_platform",
+]