diff --git a/README.md b/README.md index 227a1552..baf79494 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ We believe that a well-developed open-source code framework can lower the thresh > Currently, the development personnel of this project are limited, with most of the work handled by [Artiprocher](https://github.com/Artiprocher) and [mi804](https://github.com/mi804). Therefore, the progress of new feature development will be relatively slow, and the speed of responding to and resolving issues is limited. We apologize for this and ask developers to understand. -- **July 28, 2026** LingBot-Video open-sourced, welcome a new member to the video model family! Support includes text-to-video, image-to-video and text-to-image generation, low VRAM inference, and LoRA / full training capabilities. For details, please refer to the [documentation](/docs/en/Model_Details/LingBot-Video.md) and [example code](/examples/lingbot_video/). Huge thanks to [NancyFyong](https://github.com/NancyFyong) for contributing the integration of this model! +- **July 28, 2026** LingBot-Video open-sourced, welcome a new member to the video model family! Support includes text-to-video, image-to-video and text-to-image generation, low VRAM inference, and LoRA / full training capabilities. The MoE-30B-A3B variant serves the same three tasks with 30B total parameters and ~3B active per token. For details, please refer to the [documentation](/docs/en/Model_Details/LingBot-Video.md) and [example code](/examples/lingbot_video/). Huge thanks to [NancyFyong](https://github.com/NancyFyong) for contributing the integration of this model! - **July 21, 2026** We have open-sourced [DiffSynth-Studio Model Integration Skills](https://www.modelscope.cn/collections/DiffSynth-Studio/DiffSynth-Studio-Model-Integration-Skills). This is a composable collection of Agent Skills that automates the entire workflow of integrating external diffusion models into DiffSynth-Studio, significantly improving the standardization and efficiency of model integration. Get started with the [example](https://www.modelscope.cn/skills/DiffSynth-Studio/diffsynth-integrator/file/view/master/example.md?status=1)! @@ -1518,6 +1518,11 @@ Example code for LingBot-Video is available at: [/examples/lingbot_video/](/exam |[Robbyant/lingbot-video-dense-1.3b: T2V](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2v.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2v.py)|[code](/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_t2v.sh)|[code](/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_t2v.py)|[code](/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_t2v.sh)|[code](/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_t2v.py)| |[Robbyant/lingbot-video-dense-1.3b: TI2V](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_ti2v.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_ti2v.py)|[code](/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_ti2v.sh)|[code](/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_ti2v.py)|[code](/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_ti2v.sh)|[code](/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_ti2v.py)| |[Robbyant/lingbot-video-dense-1.3b: T2I](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2i.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2i.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: T2V](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v.py)|[code](/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_t2v.sh)|[code](/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_t2v.py)|[code](/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_t2v.sh)|[code](/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_t2v.py)| +|[Robbyant/lingbot-video-moe-30b-a3b: TI2V](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v.py)|[code](/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_ti2v.sh)|[code](/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_ti2v.py)|[code](/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_ti2v.sh)|[code](/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_ti2v.py)| +|[Robbyant/lingbot-video-moe-30b-a3b: T2I](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2i.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2i.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: T2V + Refinement](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v_refiner.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v_refiner.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: TI2V + Refinement](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v_refiner.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v_refiner.py)|-|-|-|-| diff --git a/README_zh.md b/README_zh.md index 57db38cf..3627986e 100644 --- a/README_zh.md +++ b/README_zh.md @@ -36,7 +36,7 @@ DiffSynth 目前包括两个开源项目： > 目前本项目的开发人员有限，大部分工作由 [Artiprocher](https://github.com/Artiprocher) 和 [mi804](https://github.com/mi804) 负责，因此新功能的开发进展会比较缓慢，issue 的回复和解决速度有限，我们对此感到非常抱歉，请各位开发者理解。 -- **2026年7月28日** LingBot-Video 开源，欢迎加入视频生成模型家族！支持文生视频、图生视频、文生图推理、低显存推理以及 LoRA / 全量训练能力。详情请参考[文档](/docs/zh/Model_Details/LingBot-Video.md)和[示例代码](/examples/lingbot_video/)。特别感谢 [NancyFyong](https://github.com/NancyFyong) 贡献了本模型的接入代码！ +- **2026年7月28日** LingBot-Video 开源，欢迎加入视频生成模型家族！支持文生视频、图生视频、文生图推理、低显存推理以及 LoRA / 全量训练能力。MoE-30B-A3B 总参数量 30B、每个 token 激活约 3B，同样支持这三种任务。详情请参考[文档](/docs/zh/Model_Details/LingBot-Video.md)和[示例代码](/examples/lingbot_video/)。特别感谢 [NancyFyong](https://github.com/NancyFyong) 贡献了本模型的接入代码！ - **2026年7月21日** 我们开源了 [DiffSynth-Studio Model Integration Skills](https://www.modelscope.cn/collections/DiffSynth-Studio/DiffSynth-Studio-Model-Integration-Skills)。这是一套可组合的 Agent Skill 合集，将外部扩散模型接入 DiffSynth-Studio 的全流程自动化，大幅提升模型接入标准化程度与效率。从[使用示例](https://www.modelscope.cn/skills/DiffSynth-Studio/diffsynth-integrator/file/view/master/example.md?status=1)开始体验吧！ @@ -1518,6 +1518,11 @@ LingBot-Video 的示例代码位于：[/examples/lingbot_video/](/examples/lingb |[Robbyant/lingbot-video-dense-1.3b: T2V](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2v.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2v.py)|[code](/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_t2v.sh)|[code](/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_t2v.py)|[code](/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_t2v.sh)|[code](/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_t2v.py)| |[Robbyant/lingbot-video-dense-1.3b: TI2V](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_ti2v.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_ti2v.py)|[code](/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_ti2v.sh)|[code](/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_ti2v.py)|[code](/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_ti2v.sh)|[code](/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_ti2v.py)| |[Robbyant/lingbot-video-dense-1.3b: T2I](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2i.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2i.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: T2V](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v.py)|[code](/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_t2v.sh)|[code](/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_t2v.py)|[code](/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_t2v.sh)|[code](/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_t2v.py)| +|[Robbyant/lingbot-video-moe-30b-a3b: TI2V](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v.py)|[code](/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_ti2v.sh)|[code](/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_ti2v.py)|[code](/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_ti2v.sh)|[code](/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_ti2v.py)| +|[Robbyant/lingbot-video-moe-30b-a3b: T2I](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2i.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2i.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: T2V + Refinement](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v_refiner.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v_refiner.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: TI2V + Refinement](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v_refiner.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v_refiner.py)|-|-|-|-| diff --git a/diffsynth/configs/model_configs.py b/diffsynth/configs/model_configs.py index 4ba42ba4..96325ae2 100644 --- a/diffsynth/configs/model_configs.py +++ b/diffsynth/configs/model_configs.py @@ -1330,6 +1330,14 @@ "model_class": "diffsynth.models.lingbot_video_dit.LingBotVideoDiT", "state_dict_converter": "diffsynth.utils.state_dict_converters.lingbot_video_dit.LingBotVideoDiTStateDictConverter", }, + { + # Example: ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors") + "model_hash": "65b83aa625cd362ff5ff3409fb367a6f", + "model_name": "lingbot_video_dit", + "model_class": "diffsynth.models.lingbot_video_dit.LingBotVideoDiT", + "state_dict_converter": "diffsynth.utils.state_dict_converters.lingbot_video_dit.LingBotVideoDiTStateDictConverter", + "extra_kwargs": {'depth': 48, 'axes_lens': (4096, 512, 512), 'num_experts': 128, 'moe_intermediate_size': 768, 'n_group': 4, 'topk_group': 2, 'n_shared_experts': 1, 'routed_scaling_factor': 2.5}, + }, ] MODEL_CONFIGS = ( diff --git a/diffsynth/configs/vram_management_module_maps.py b/diffsynth/configs/vram_management_module_maps.py index 9362be22..45906df5 100644 --- a/diffsynth/configs/vram_management_module_maps.py +++ b/diffsynth/configs/vram_management_module_maps.py @@ -412,7 +412,9 @@ "transformers.models.qwen3_vl.modeling_qwen3_vl.Qwen3VLVisionRotaryEmbedding": "diffsynth.core.vram.layers.AutoWrappedModule", }, "diffsynth.models.lingbot_video_dit.LingBotVideoDiT": { - "diffsynth.models.lingbot_video_dit.LingBotVideoBlock": "diffsynth.core.vram.layers.AutoWrappedModule", + "diffsynth.models.lingbot_video_dit.LingBotVideoBlock": "diffsynth.core.vram.layers.AutoWrappedNonRecurseModule", + "diffsynth.models.lingbot_video_dit.LingBotVideoRouter": "diffsynth.core.vram.layers.AutoWrappedNonRecurseModule", + "diffsynth.models.lingbot_video_dit.LingBotVideoGroupedExperts": "diffsynth.core.vram.layers.AutoWrappedNonRecurseModule", "diffsynth.models.lingbot_video_dit.LingBotVideoRMSNorm": "diffsynth.core.vram.layers.AutoWrappedModule", "torch.nn.Linear": "diffsynth.core.vram.layers.AutoWrappedLinear", }, diff --git a/diffsynth/diffusion/flow_match.py b/diffsynth/diffusion/flow_match.py index b9104e86..5097ea43 100644 --- a/diffsynth/diffusion/flow_match.py +++ b/diffsynth/diffusion/flow_match.py @@ -5,7 +5,7 @@ class FlowMatchScheduler(): - def __init__(self, template: Literal["FLUX.1", "Wan", "Qwen-Image", "FLUX.2", "Z-Image", "LTX-2", "Qwen-Image-Lightning", "ERNIE-Image", "ACE-Step", "Ideogram4", "Krea-2", "Boogu"] = "FLUX.1"): + def __init__(self, template: Literal["FLUX.1", "Wan", "Qwen-Image", "FLUX.2", "Z-Image", "LTX-2", "Qwen-Image-Lightning", "ERNIE-Image", "ACE-Step", "Ideogram4", "Krea-2", "Boogu", "LingBot-Video"] = "FLUX.1"): self.set_timesteps_fn = { "FLUX.1": FlowMatchScheduler.set_timesteps_flux, "Wan": FlowMatchScheduler.set_timesteps_wan, @@ -20,6 +20,7 @@ def __init__(self, template: Literal["FLUX.1", "Wan", "Qwen-Image", "FLUX.2", "Z "Ideogram4": FlowMatchScheduler.set_timesteps_ideogram4, "Krea-2": FlowMatchScheduler.set_timesteps_krea2, "Boogu": FlowMatchScheduler.set_timesteps_boogu, + "LingBot-Video": FlowMatchScheduler.set_timesteps_lingbot_video, }.get(template, FlowMatchScheduler.set_timesteps_flux) self.num_train_timesteps = 1000 @@ -79,6 +80,28 @@ def set_timesteps_qwen_image(num_inference_steps=100, denoising_strength=1.0, ex timesteps = sigmas * num_train_timesteps return sigmas, timesteps + @staticmethod + def set_timesteps_lingbot_video(num_inference_steps=100, denoising_strength=1.0, shift=None, t_thresh=None, sigma_tail_steps=0): + sigma_min = 0.0 + sigma_max = 1.0 + shift = 5 if shift is None else shift + num_train_timesteps = 1000 + sigma_start = sigma_min + (sigma_max - sigma_min) * denoising_strength + sigmas = torch.linspace(sigma_start, sigma_min, num_inference_steps + 1)[:-1] + sigmas = shift * sigmas / (1 + (shift - 1) * sigmas) + if t_thresh is not None: + # Refinement schedule: keep the sub-threshold part of the shifted grid, pin the first + # sigma exactly at t_thresh, then append extra low-noise steps that end at sigma_min. + sigmas = sigmas[sigmas <= t_thresh + 1e-6] + if sigmas.numel() == 0 or abs(float(sigmas[0]) - t_thresh) > 1e-6: + sigmas = torch.cat([torch.tensor([t_thresh], dtype=sigmas.dtype), sigmas]) + if sigma_tail_steps > 0: + tail_start = float(sigmas[-1]) + tail = torch.linspace(tail_start, min(sigma_min, tail_start), sigma_tail_steps + 2)[1:-1] + sigmas = torch.cat([sigmas, tail.to(dtype=sigmas.dtype)]) + timesteps = sigmas * num_train_timesteps + return sigmas, timesteps + @staticmethod def set_timesteps_qwen_image_lightning(num_inference_steps=100, denoising_strength=1.0, exponential_shift_mu=None, dynamic_shift_len=None): sigma_min = 0.0 diff --git a/diffsynth/models/lingbot_video_dit.py b/diffsynth/models/lingbot_video_dit.py index c98557fc..ed2ca620 100644 --- a/diffsynth/models/lingbot_video_dit.py +++ b/diffsynth/models/lingbot_video_dit.py @@ -212,7 +212,7 @@ def __init__(self, hidden_size, num_experts, top_k, score_func, norm_topk_prob, self.topk_group = topk_group self.route_scale = route_scale self.weight = nn.Parameter(torch.empty(num_experts, hidden_size)) - self.register_buffer("e_score_correction_bias", torch.zeros(num_experts), persistent=True) + self.e_score_correction_bias = nn.Parameter(torch.zeros(num_experts), requires_grad=False) def _group_limited_topk(self, scores_for_choice): seq_len = scores_for_choice.shape[0] @@ -228,12 +228,12 @@ def _group_limited_topk(self, scores_for_choice): def forward(self, tokens: torch.Tensor): with torch.amp.autocast(tokens.device.type, enabled=False): - logits = F.linear(tokens.float(), self.weight.float()) + logits = F.linear(tokens.float(), self.weight.to(device=tokens.device, dtype=torch.float32)) if self.score_func == "softmax": scores = F.softmax(logits, dim=-1) else: scores = logits.sigmoid() - scores_for_choice = scores + self.e_score_correction_bias.unsqueeze(0) + scores_for_choice = scores + self.e_score_correction_bias.to(device=scores.device, dtype=scores.dtype).unsqueeze(0) if self.n_group is not None and self.n_group > 1: top_indices = self._group_limited_topk(scores_for_choice) else: @@ -329,13 +329,14 @@ def _unpad_grouped_tokens(output, input_shape, permuted_indices): return unpermuted[:-1] def _run_grouped_experts(self, tokens, counts): - if not hasattr(torch, "_grouped_mm"): + if not hasattr(torch, "_grouped_mm") or tokens.device.type != "cuda": return self._run_experts_for_loop(tokens, counts) input_shape, padded_tokens, permuted_indices, aligned_counts = self._pad_grouped_tokens(tokens, counts) offsets = torch.cumsum(aligned_counts, dim=0, dtype=torch.int32) - h = F.silu(torch._grouped_mm(padded_tokens.bfloat16(), self.experts.w1.bfloat16().transpose(-2, -1), offs=offsets)) - h = h * torch._grouped_mm(padded_tokens.bfloat16(), self.experts.w3.bfloat16().transpose(-2, -1), offs=offsets) - out = torch._grouped_mm(h, self.experts.w2.bfloat16().transpose(-2, -1), offs=offsets).type_as(padded_tokens) + w1, w2, w3 = (w.to(device=tokens.device, dtype=torch.bfloat16) for w in (self.experts.w1, self.experts.w2, self.experts.w3)) + h = F.silu(torch._grouped_mm(padded_tokens.bfloat16(), w1.transpose(-2, -1), offs=offsets)) + h = h * torch._grouped_mm(padded_tokens.bfloat16(), w3.transpose(-2, -1), offs=offsets) + out = torch._grouped_mm(h, w2.transpose(-2, -1), offs=offsets).type_as(padded_tokens) return self._unpad_grouped_tokens(out, input_shape, permuted_indices) def _run_experts_for_loop(self, tokens, counts): @@ -345,9 +346,9 @@ def _run_experts_for_loop(self, tokens, counts): for expert_idx, expert_tokens in enumerate(splits): if expert_tokens.numel() == 0: continue - h = F.silu(expert_tokens @ self.experts.w1[expert_idx].transpose(-2, -1)) - h = h * (expert_tokens @ self.experts.w3[expert_idx].transpose(-2, -1)) - h = h @ self.experts.w2[expert_idx].transpose(-2, -1) + h = F.silu(expert_tokens @ self.experts.w1[expert_idx].to(device=expert_tokens.device, dtype=expert_tokens.dtype).transpose(-2, -1)) + h = h * (expert_tokens @ self.experts.w3[expert_idx].to(device=expert_tokens.device, dtype=expert_tokens.dtype).transpose(-2, -1)) + h = h @ self.experts.w2[expert_idx].to(device=expert_tokens.device, dtype=expert_tokens.dtype).transpose(-2, -1) outputs.append(h) if not outputs: return tokens.new_zeros(tokens.shape) @@ -420,7 +421,7 @@ def forward(self, x, temb6, rotary_emb, attention_mask=None, moe_padding_mask=No "LingBotVideoBlock expects token-level temb6 with shape (B*S, 6D); " f"got {tuple(temb6.shape)} for hidden states {tuple(x.shape)}." ) - mod = temb6.view(x.shape[0], x.shape[1], -1) + self.scale_shift_table.unsqueeze(0) + mod = temb6.view(x.shape[0], x.shape[1], -1) + self.scale_shift_table.to(dtype=temb6.dtype, device=temb6.device).unsqueeze(0) shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = mod.chunk(6, dim=-1) gate_msa, gate_mlp = gate_msa.tanh(), gate_mlp.tanh() scale_msa, scale_mlp = 1.0 + scale_msa, 1.0 + scale_mlp diff --git a/diffsynth/pipelines/lingbot_video.py b/diffsynth/pipelines/lingbot_video.py index 28973198..081877c8 100644 --- a/diffsynth/pipelines/lingbot_video.py +++ b/diffsynth/pipelines/lingbot_video.py @@ -25,7 +25,7 @@ def __init__(self, device=get_device_type(), torch_dtype=torch.bfloat16): height_division_factor=16, width_division_factor=16, time_division_factor=4, time_division_remainder=1, ) - self.scheduler = FlowMatchScheduler(template="Wan") + self.scheduler = FlowMatchScheduler(template="LingBot-Video") self.text_encoder: Krea2TextEncoder = None self.dit: LingBotVideoDiT = None self.vae: QwenImageVAE = None @@ -102,11 +102,18 @@ def __call__( # Scheduler num_inference_steps: int = 40, sigma_shift: float = 3.0, + # Refinement pass: start from a partially noised input video at sigma=t_thresh and + # append extra low-noise steps at the tail of the schedule. + t_thresh: float = None, + sigma_tail_steps: int = 2, # progress_bar progress_bar_cmd=tqdm, ): # Scheduler - self.scheduler.set_timesteps(num_inference_steps, denoising_strength=denoising_strength, shift=sigma_shift) + self.scheduler.set_timesteps( + num_inference_steps, denoising_strength=denoising_strength, shift=sigma_shift, + t_thresh=t_thresh, sigma_tail_steps=sigma_tail_steps, + ) # Inputs inputs_posi = {"prompt": prompt} @@ -116,7 +123,7 @@ def __call__( "input_video": input_video, "denoising_strength": denoising_strength, "seed": seed, "rand_device": rand_device, "height": height, "width": width, "num_frames": num_frames, - "cfg_scale": cfg_scale, + "cfg_scale": cfg_scale, "t_thresh": t_thresh, } for unit in self.units: inputs_shared, inputs_posi, inputs_nega = self.unit_runner(unit, self, inputs_shared, inputs_posi, inputs_nega) @@ -132,6 +139,11 @@ def __call__( **models, timestep=timestep, progress_id=progress_id ) inputs_shared["latents"] = self.step(self.scheduler, progress_id=progress_id, noise_pred=noise_pred, **inputs_shared) + if t_thresh is not None and inputs_shared.get("first_frame_latents") is not None: + # The refiner re-pins the clean condition latent after every step, keeping frame 0 + # identical to the input image while the rest of the clip denoises against it. + first_frame_latents = inputs_shared["first_frame_latents"] + inputs_shared["latents"][:, :, :first_frame_latents.shape[2]] = first_frame_latents self.load_models_to_device(['vae']) latents = inputs_shared["latents"].to(dtype=self.torch_dtype, device=self.device) @@ -197,12 +209,12 @@ class LingBotVideoUnit_ImageEmbedder(PipelineUnit): def __init__(self): super().__init__( - input_params=("input_image", "latents", "height", "width"), + input_params=("input_image", "latents", "height", "width", "t_thresh"), output_params=("latents", "first_frame_latents", "vlm_image"), onload_model_names=("vae",), ) - def process(self, pipe: LingBotVideoPipeline, input_image, latents, height, width): + def process(self, pipe: LingBotVideoPipeline, input_image, latents, height, width, t_thresh=None): if input_image is None: return {} pipe.load_models_to_device(self.onload_model_names) @@ -211,7 +223,8 @@ def process(self, pipe: LingBotVideoPipeline, input_image, latents, height, widt pixel = self.preprocess_cond_image(input_image, height, width) pixel = pixel.to(dtype=pipe.torch_dtype, device=pipe.device) first_frame_latents = pipe.vae.encode_video(pixel * 2.0 - 1.0).to(dtype=pipe.torch_dtype, device=pipe.device) - vlm_image = self.vlm_image(pipe, pixel) + # The refiner conditions on text only and re-pins the frame-0 latent every step instead. + vlm_image = None if t_thresh is not None else self.vlm_image(pipe, pixel) # Pin the clean condition latent into the first temporal slot before sampling. cond_t = first_frame_latents.shape[2] latents[:, :, :cond_t] = first_frame_latents diff --git a/docs/en/Model_Details/LingBot-Video.md b/docs/en/Model_Details/LingBot-Video.md index 61532103..f0dab55d 100644 --- a/docs/en/Model_Details/LingBot-Video.md +++ b/docs/en/Model_Details/LingBot-Video.md @@ -70,11 +70,20 @@ save_video(video, "video.mp4", fps=15, quality=10) ## Model Overview +MoE-30B-A3B is the larger variant: 30B total parameters with ~3B active per token, where each MoE layer holds 128 routed experts plus 1 shared expert and routes every token to 8 experts with group-limited top-k (4 groups, top-2 groups). It serves the same three tasks through the same pipeline, only the model ID and the shard glob change. + +The MoE release additionally ships a **refiner** under `refiner/`: a second 30B-A3B checkpoint with the same architecture as `transformer/`, fine-tuned for a short high-resolution second pass over an already generated clip. It is not listed in `model_index.json` and is loaded through the same pipeline, only the shard glob changes. See [Two-stage refinement](#two-stage-refinement) below. + |Model ID|Inference|Low VRAM Inference|Full Training|Full Training Validation|LoRA Training|LoRA Training Validation| |-|-|-|-|-|-|-| |[Robbyant/lingbot-video-dense-1.3b: T2V](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_t2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_t2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_t2v.py)| |[Robbyant/lingbot-video-dense-1.3b: TI2V](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_ti2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_ti2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_ti2v.py)| |[Robbyant/lingbot-video-dense-1.3b: T2I](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2i.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2i.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: T2V](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_t2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_t2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_t2v.py)| +|[Robbyant/lingbot-video-moe-30b-a3b: TI2V](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_ti2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_ti2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_ti2v.py)| +|[Robbyant/lingbot-video-moe-30b-a3b: T2I](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2i.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2i.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: T2V + Refinement](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v_refiner.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v_refiner.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: TI2V + Refinement](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v_refiner.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v_refiner.py)|-|-|-|-| ## Model Inference @@ -93,12 +102,33 @@ The input parameters for `LingBotVideoPipeline` inference include: * `cfg_scale`: Classifier-free guidance scale, default `3.0`. * `num_inference_steps`: Number of inference steps, default `40`. * `sigma_shift`: Flow-matching timestep shift, default `3.0`. +* `t_thresh`: Refinement start sigma, default `None` (plain generation). When set, the schedule is truncated so that sampling starts at `sigma=t_thresh` and `input_video` is noised to exactly that level. Only meaningful together with `input_video`; TI2V additionally re-pins the clean first-frame latent after every step. The official refiner setting is `0.85`. +* `sigma_tail_steps`: Number of extra low-noise steps appended to the tail of the refinement schedule, default `2`. Only effective when `t_thresh` is set. * `seed`: Random seed. Default is `None`, meaning completely random. * `rand_device`: Device for generating the initial noise, default `"cpu"`. * `progress_bar_cmd`: Progress bar, default `tqdm`. Can be disabled by setting to `lambda x: x`. If VRAM is insufficient, please enable [VRAM Management](../Pipeline_Usage/VRAM_management.md). We provide recommended low-VRAM configurations for each task in the example code, see the table in the "Model Overview" section above. +### Two-stage refinement + +The MoE refiner performs a short second pass at a higher resolution: the official setup generates at 480×832 with 40 steps, then refines at 1088×1920 with 8 steps. Load the pipeline with the `refiner/` shards instead of `transformer/`, feed the base clip back in through `input_video` at the higher resolution, and set `t_thresh`: + +```python +input_video = VideoData("video_base.mp4", height=1088, width=1920) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, + height=1088, width=1920, num_frames=81, + num_inference_steps=8, cfg_scale=3.0, + t_thresh=0.85, sigma_tail_steps=2, + seed=0, +) +``` + +The upscaled clip is VAE-encoded and noised back to `sigma=t_thresh`, so the pass keeps the structure of the base clip and regenerates detail at the target resolution. Pass the same caption as the base pass and keep the same aspect ratio. The refinement resolution dominates the cost — at 1088×1920 the sequence is ~5× longer than at 480×832 — so run this pass with VRAM management enabled. + ### Prompt rewriting LingBot-Video is trained on **structured-JSON captions**, not free-form prose. Feeding a flat sentence is out-of-distribution and visibly degrades quality. The pipeline accepts a caption as a `dict` (the format used at training time) or a plain string, and normalises the `dict` internally. @@ -183,4 +213,6 @@ modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --inclu Training captions should be **structured-JSON captions** (the same in-distribution format used at inference). If your dataset stores raw prose, rewrite it once offline with [`examples/lingbot_video/model_training/scripts/rewrite_captions.py`](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/scripts/rewrite_captions.py) before training. +The MoE-30B-A3B scripts launch with [`examples/lingbot_video/model_training/full/accelerate_config_moe.yaml`](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/full/accelerate_config_moe.yaml) (DeepSpeed ZeRO-2, optimizer CPU offload, bf16), and the full-parameter scripts additionally enable `--use_gradient_checkpointing_offload`. Note that `--lora_target_modules "to_q,to_k,to_v,to_out"` only reaches the attention projections: the routed experts and the router are stored as bare parameter tensors for grouped matmul, so LoRA leaves them frozen. Use full-parameter training to update the expert stack. + We provide recommended training scripts for each task, please refer to the table in "Model Overview" above. For guidance on writing model training scripts, see [Model Training](../Pipeline_Usage/Model_Training.md); for more advanced training algorithms, see [Training Framework Overview](https://github.com/modelscope/DiffSynth-Studio/tree/main/docs/en/Training/). diff --git a/docs/zh/Model_Details/LingBot-Video.md b/docs/zh/Model_Details/LingBot-Video.md index eca7dfb2..1e21b124 100644 --- a/docs/zh/Model_Details/LingBot-Video.md +++ b/docs/zh/Model_Details/LingBot-Video.md @@ -70,11 +70,20 @@ save_video(video, "video.mp4", fps=15, quality=10) ## 模型总览 +MoE-30B-A3B 是更大的版本：总参数量 30B，每个 token 激活约 3B，每个 MoE 层包含 128 个路由专家和 1 个共享专家，并使用 group-limited top-k（4 组，取 top-2 组）将每个 token 路由到 8 个专家。它通过同一条 Pipeline 支持同样的三种任务，只需更换模型 ID 与分片通配符。 + +MoE 版本还额外提供了 `refiner/` 目录下的 **refiner**：一份与 `transformer/` 架构相同的 30B-A3B 权重，专门微调用于对已生成视频做一次高分辨率精修。它没有出现在 `model_index.json` 中，通过同一条 Pipeline 加载，只需更换分片通配符。详见下方[两阶段精修](#两阶段精修)。 + |模型 ID|推理|低显存推理|全量训练|全量训练后验证|LoRA 训练|LoRA 训练后验证| |-|-|-|-|-|-|-| |[Robbyant/lingbot-video-dense-1.3b: T2V](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_t2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_t2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_t2v.py)| |[Robbyant/lingbot-video-dense-1.3b: TI2V](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/full/lingbot-video-dense-1.3b_ti2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_full/lingbot-video-dense-1.3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/lora/lingbot-video-dense-1.3b_ti2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_lora/lingbot-video-dense-1.3b_ti2v.py)| |[Robbyant/lingbot-video-dense-1.3b: T2I](https://modelscope.cn/models/Robbyant/lingbot-video-dense-1.3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-dense-1.3b_t2i.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-dense-1.3b_t2i.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: T2V](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_t2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_t2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_t2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_t2v.py)| +|[Robbyant/lingbot-video-moe-30b-a3b: TI2V](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_ti2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_ti2v.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_ti2v.sh)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_ti2v.py)| +|[Robbyant/lingbot-video-moe-30b-a3b: T2I](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2i.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2i.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: T2V + Refinement](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v_refiner.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v_refiner.py)|-|-|-|-| +|[Robbyant/lingbot-video-moe-30b-a3b: TI2V + Refinement](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v_refiner.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v_refiner.py)|-|-|-|-| ## 模型推理 @@ -93,12 +102,33 @@ save_video(video, "video.mp4", fps=15, quality=10) * `cfg_scale`: 无分类器指导强度，默认 `3.0`。 * `num_inference_steps`: 推理步数，默认 `40`。 * `sigma_shift`: Flow-matching 时间步 shift，默认 `3.0`。 +* `t_thresh`: 精修起始 sigma，默认 `None`（普通生成）。设置后调度会被截断，使采样从 `sigma=t_thresh` 开始，并把 `input_video` 加噪到该噪声水平。仅在提供 `input_video` 时有意义；TI2V 还会在每个采样步之后重新写入干净的首帧 latent。官方精修配置为 `0.85`。 +* `sigma_tail_steps`: 精修调度尾部追加的额外低噪声步数，默认 `2`。仅在设置了 `t_thresh` 时生效。 * `seed`: 随机种子，默认 `None`（完全随机）。 * `rand_device`: 生成初始噪声的设备，默认 `"cpu"`。 * `progress_bar_cmd`: 进度条，默认 `tqdm`，可设为 `lambda x: x` 关闭。 显存不足时请参考[显存管理](../Pipeline_Usage/VRAM_management.md)启用显存管理功能。我们在示例代码中提供了每个任务的推荐低显存配置，见上方"模型总览"中的表格。 +### 两阶段精修 + +MoE 的 refiner 会在更高分辨率上执行一次短程精修：官方配置先以 480×832、40 步生成，再以 1088×1920、8 步精修。加载时把分片通配符从 `transformer/` 换成 `refiner/`，将基础阶段的视频以目标分辨率通过 `input_video` 传回，并设置 `t_thresh`： + +```python +input_video = VideoData("video_base.mp4", height=1088, width=1920) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, + height=1088, width=1920, num_frames=81, + num_inference_steps=8, cfg_scale=3.0, + t_thresh=0.85, sigma_tail_steps=2, + seed=0, +) +``` + +放大后的视频会被 VAE 编码并重新加噪到 `sigma=t_thresh`，因此这一阶段保留基础阶段的结构，并在目标分辨率上重新生成细节。请使用与基础阶段相同的 caption，并保持相同的宽高比。精修分辨率决定了主要开销——1088×1920 的序列长度约为 480×832 的 5 倍——建议开启显存管理运行该阶段。 + ### 提示词改写 LingBot-Video 训练时使用的是**结构化 JSON caption**，直接喂平铺句子属于分布外输入，会明显降低生成质量。Pipeline 接受 `dict` 形式的 caption（与训练一致的格式）或纯字符串，`dict` 会被内部归一化。 @@ -183,4 +213,6 @@ modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --inclu 训练时 `prompt` 字段应存放**结构化 JSON caption**（与推理时使用的分布内格式一致）。如果数据集里存的是原始散文，可先使用 [`examples/lingbot_video/model_training/scripts/rewrite_captions.py`](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/scripts/rewrite_captions.py) 离线改写一次。 +MoE-30B-A3B 的训练脚本通过 [`examples/lingbot_video/model_training/full/accelerate_config_moe.yaml`](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/lingbot_video/model_training/full/accelerate_config_moe.yaml) 启动（DeepSpeed ZeRO-2、优化器状态 CPU offload、bf16），全量训练脚本还额外开启了 `--use_gradient_checkpointing_offload`。需要注意的是，`--lora_target_modules "to_q,to_k,to_v,to_out"` 只覆盖注意力投影层：路由专家与路由器以裸参数张量的形式存放以便做分组矩阵乘法，LoRA 不会训练它们。如需更新专家部分，请使用全量训练。 + 我们为每个任务编写了推荐的训练脚本，请参考前文"模型总览"中的表格。关于如何编写模型训练脚本，请参考[模型训练](../Pipeline_Usage/Model_Training.md)；更多高阶训练算法，请参考[训练框架详解](https://github.com/modelscope/DiffSynth-Studio/tree/main/docs/zh/Training/)。 diff --git a/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2i.py b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2i.py new file mode 100644 index 00000000..98c34736 --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2i.py @@ -0,0 +1,32 @@ +import json +import torch +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2i/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2i/t2i_example.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +frames = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt_image, + height=480, width=832, num_frames=1, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +frames[0].save("image_lingbot-video-moe-30b-a3b_t2i.png") diff --git a/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v.py b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v.py new file mode 100644 index 00000000..fcbb2883 --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v.py @@ -0,0 +1,50 @@ +import torch +import json +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +# --- Text-to-video ------------------------------------------------------------------- +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +# LingBot-Video is trained on structured-JSON captions, not free-form prose. This example +# runs on a released in-distribution caption; see the bottom for turning a brief idea into +# such a caption with the two-stage prompt rewriter. +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v.mp4", fps=15, quality=10) + +# --- Video-to-video ------------------------------------------------------------------ +# denoising_strength < 1 keeps part of the input structure. +input_video = VideoData("video_lingbot-video-moe-30b-a3b_t2v.mp4", height=480, width=832) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, denoising_strength=0.7, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=1, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_v2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v_refiner.py b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v_refiner.py new file mode 100644 index 00000000..7c4345cc --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v_refiner.py @@ -0,0 +1,61 @@ +import torch +import json +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +# --- Stage 1: base generation at 480x832 --------------------------------------------- +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v.mp4", fps=15, quality=10) + +del pipe +torch.cuda.empty_cache() + +# --- Stage 2: refinement at 1088x1920 ------------------------------------------------ +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="refiner/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +input_video = VideoData("video_lingbot-video-moe-30b-a3b_t2v.mp4", height=1088, width=1920) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, + height=1088, width=1920, num_frames=81, + num_inference_steps=8, cfg_scale=3.0, + t_thresh=0.85, sigma_tail_steps=2, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v_refined.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v.py b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v.py new file mode 100644 index 00000000..3225f9d2 --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v.py @@ -0,0 +1,39 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v_refiner.py b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v_refiner.py new file mode 100644 index 00000000..017f67d0 --- /dev/null +++ b/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v_refiner.py @@ -0,0 +1,67 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +# --- Stage 1: base generation at 480x832 --------------------------------------------- +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v.mp4", fps=15, quality=10) + +del pipe +torch.cuda.empty_cache() + +# --- Stage 2: refinement at 1088x1920 ------------------------------------------------ +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="refiner/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) + +input_video = VideoData("video_lingbot-video-moe-30b-a3b_ti2v.mp4", height=1088, width=1920) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + input_video=input_video, + height=1088, width=1920, num_frames=81, + num_inference_steps=8, cfg_scale=3.0, + t_thresh=0.85, sigma_tail_steps=2, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v_refined.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2i.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2i.py new file mode 100644 index 00000000..8d34665e --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2i.py @@ -0,0 +1,44 @@ +import json +import torch +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2i/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2i/t2i_example.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +frames = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt_image, + height=480, width=832, num_frames=1, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +frames[0].save("image_lingbot-video-moe-30b-a3b_t2i_low_vram.png") diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v.py new file mode 100644 index 00000000..b1bacb7d --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v.py @@ -0,0 +1,58 @@ +import torch +import json +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +# --- Text-to-video ------------------------------------------------------------------- +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v.mp4", fps=15, quality=10) + +# --- Video-to-video ------------------------------------------------------------------ +input_video = VideoData("video_lingbot-video-moe-30b-a3b_t2v.mp4", height=480, width=832) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, denoising_strength=0.7, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=1, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_v2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v_refiner.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v_refiner.py new file mode 100644 index 00000000..eea37799 --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v_refiner.py @@ -0,0 +1,74 @@ +import torch +import json +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +# --- Stage 1: base generation at 480x832 --------------------------------------------- +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v.mp4", fps=15, quality=10) + +del pipe +torch.cuda.empty_cache() + +# --- Stage 2: refinement at 1088x1920 ------------------------------------------------ +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="refiner/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +input_video = VideoData("video_lingbot-video-moe-30b-a3b_t2v.mp4", height=1088, width=1920) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_video=input_video, + height=1088, width=1920, num_frames=81, + num_inference_steps=8, cfg_scale=3.0, + t_thresh=0.85, sigma_tail_steps=2, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v_refined.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v.py new file mode 100644 index 00000000..bb58682c --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v.py @@ -0,0 +1,51 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v_low_vram.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v_refiner.py b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v_refiner.py new file mode 100644 index 00000000..4795e043 --- /dev/null +++ b/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v_refiner.py @@ -0,0 +1,80 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video, VideoData +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +vram_config = { + "offload_dtype": "disk", + "offload_device": "disk", + "onload_dtype": torch.float8_e4m3fn, + "onload_device": "cpu", + "preparing_dtype": torch.float8_e4m3fn, + "preparing_device": "cuda", + "computation_dtype": torch.bfloat16, + "computation_device": "cuda", +} + +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +# --- Stage 1: base generation at 480x832 --------------------------------------------- +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v.mp4", fps=15, quality=10) + +del pipe +torch.cuda.empty_cache() + +# --- Stage 2: refinement at 1088x1920 ------------------------------------------------ +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="refiner/diffusion_pytorch_model*.safetensors", **vram_config), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors", **vram_config), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors", **vram_config), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), + vram_limit=torch.cuda.mem_get_info("cuda")[1] / (1024 ** 3) - 0.5, +) + +input_video = VideoData("video_lingbot-video-moe-30b-a3b_ti2v.mp4", height=1088, width=1920) +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + input_video=input_video, + height=1088, width=1920, num_frames=81, + num_inference_steps=8, cfg_scale=3.0, + t_thresh=0.85, sigma_tail_steps=2, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v_refined.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/full/accelerate_config_moe.yaml b/examples/lingbot_video/model_training/full/accelerate_config_moe.yaml new file mode 100644 index 00000000..3875a9da --- /dev/null +++ b/examples/lingbot_video/model_training/full/accelerate_config_moe.yaml @@ -0,0 +1,22 @@ +compute_environment: LOCAL_MACHINE +debug: false +deepspeed_config: + gradient_accumulation_steps: 1 + offload_optimizer_device: cpu + offload_param_device: cpu + zero3_init_flag: false + zero_stage: 2 +distributed_type: DEEPSPEED +downcast_bf16: 'no' +enable_cpu_affinity: false +machine_rank: 0 +main_training_function: main +mixed_precision: bf16 +num_machines: 1 +num_processes: 8 +rdzv_backend: static +same_network: true +tpu_env: [] +tpu_use_cluster: false +tpu_use_sudo: false +use_cpu: false diff --git a/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_t2v.sh b/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_t2v.sh new file mode 100644 index 00000000..77935b88 --- /dev/null +++ b/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_t2v.sh @@ -0,0 +1,19 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-dense-1.3b_t2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch --config_file examples/lingbot_video/model_training/full/accelerate_config_moe.yaml examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-moe-30b-a3b:transformer/diffusion_pytorch_model*.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-moe-30b-a3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-moe-30b-a3b_t2v_full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --use_gradient_checkpointing_offload diff --git a/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_ti2v.sh b/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_ti2v.sh new file mode 100644 index 00000000..8dcb0c16 --- /dev/null +++ b/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_ti2v.sh @@ -0,0 +1,20 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-dense-1.3b_ti2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch --config_file examples/lingbot_video/model_training/full/accelerate_config_moe.yaml examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --first_frame_as_condition \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-moe-30b-a3b:transformer/diffusion_pytorch_model*.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-moe-30b-a3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-5 \ + --num_epochs 2 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-moe-30b-a3b_ti2v_full" \ + --trainable_models "dit" \ + --use_gradient_checkpointing \ + --use_gradient_checkpointing_offload diff --git a/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_t2v.sh b/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_t2v.sh new file mode 100644 index 00000000..9f9aa4ce --- /dev/null +++ b/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_t2v.sh @@ -0,0 +1,20 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-dense-1.3b_t2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch --config_file examples/lingbot_video/model_training/full/accelerate_config_moe.yaml examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-moe-30b-a3b:transformer/diffusion_pytorch_model*.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-moe-30b-a3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-moe-30b-a3b_t2v_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_q,to_k,to_v,to_out" \ + --lora_rank 32 \ + --use_gradient_checkpointing diff --git a/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_ti2v.sh b/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_ti2v.sh new file mode 100644 index 00000000..6f044218 --- /dev/null +++ b/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_ti2v.sh @@ -0,0 +1,21 @@ +modelscope download --dataset DiffSynth-Studio/diffsynth_example_dataset --include "lingbot_video/lingbot-video-dense-1.3b_ti2v/*" --local_dir ./data/diffsynth_example_dataset + +accelerate launch --config_file examples/lingbot_video/model_training/full/accelerate_config_moe.yaml examples/lingbot_video/model_training/train.py \ + --dataset_base_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v \ + --dataset_metadata_path data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v/metadata.json \ + --data_file_keys "video" \ + --height 480 \ + --width 832 \ + --num_frames 81 \ + --first_frame_as_condition \ + --dataset_repeat 50 \ + --model_id_with_origin_paths "Robbyant/lingbot-video-moe-30b-a3b:transformer/diffusion_pytorch_model*.safetensors,Qwen/Qwen3-VL-4B-Instruct:*.safetensors,Robbyant/lingbot-video-moe-30b-a3b:vae/diffusion_pytorch_model.safetensors" \ + --processor_path "Qwen/Qwen3-VL-4B-Instruct:" \ + --learning_rate 1e-4 \ + --num_epochs 5 \ + --remove_prefix_in_ckpt "pipe.dit." \ + --output_path "./models/train/lingbot-video-moe-30b-a3b_ti2v_lora" \ + --lora_base_model "dit" \ + --lora_target_modules "to_q,to_k,to_v,to_out" \ + --lora_rank 32 \ + --use_gradient_checkpointing diff --git a/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_t2v.py b/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_t2v.py new file mode 100644 index 00000000..4a822274 --- /dev/null +++ b/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_t2v.py @@ -0,0 +1,36 @@ +import torch +import json +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from diffsynth import load_state_dict +from modelscope import dataset_snapshot_download + + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +state_dict = load_state_dict("models/train/lingbot-video-moe-30b-a3b_t2v_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict) +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_ti2v.py b/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_ti2v.py new file mode 100644 index 00000000..a9adcb0d --- /dev/null +++ b/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_ti2v.py @@ -0,0 +1,42 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from diffsynth import load_state_dict +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +state_dict = load_state_dict("models/train/lingbot-video-moe-30b-a3b_ti2v_full/epoch-1.safetensors") +pipe.dit.load_state_dict(state_dict) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_t2v.py b/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_t2v.py new file mode 100644 index 00000000..0cf7766a --- /dev/null +++ b/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_t2v.py @@ -0,0 +1,34 @@ +import torch +import json +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.load_lora(pipe.dit, "models/train/lingbot-video-moe-30b-a3b_t2v_lora/epoch-4.safetensors", alpha=1) +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_t2v/*", +) +with open("data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_t2v/t2v_example_1.json", "r", encoding="utf-8") as f: + caption = json.load(f) + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_t2v.mp4", fps=15, quality=10) diff --git a/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_ti2v.py b/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_ti2v.py new file mode 100644 index 00000000..33102cd3 --- /dev/null +++ b/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_ti2v.py @@ -0,0 +1,40 @@ +import os +import json +import torch +from PIL import Image +from diffsynth.utils.data import save_video +from diffsynth.pipelines.lingbot_video import LingBotVideoPipeline, ModelConfig +from modelscope import dataset_snapshot_download + +pipe = LingBotVideoPipeline.from_pretrained( + torch_dtype=torch.bfloat16, + device="cuda", + model_configs=[ + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"), + ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern="*.safetensors"), + ModelConfig(model_id="Robbyant/lingbot-video-moe-30b-a3b", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"), + ], + processor_config=ModelConfig(model_id="Qwen/Qwen3-VL-4B-Instruct", origin_file_pattern=""), +) +pipe.load_lora(pipe.dit, "models/train/lingbot-video-moe-30b-a3b_ti2v_lora/epoch-4.safetensors", alpha=1) + +# The condition first frame and its paired caption ship in the example dataset. +dataset_snapshot_download( + dataset_id="DiffSynth-Studio/diffsynth_example_dataset", + local_dir="data/diffsynth_example_dataset", + allow_file_pattern="lingbot_video/lingbot-video-dense-1.3b_ti2v/*", +) +base = "data/diffsynth_example_dataset/lingbot_video/lingbot-video-dense-1.3b_ti2v" +with open(os.path.join(base, "ti2v_example.json"), "r", encoding="utf-8") as f: + caption = json.load(f) +input_image = Image.open(os.path.join(base, "ti2v_first_frame.png")).convert("RGB") + +video = pipe( + prompt=caption, + negative_prompt=pipe.default_negative_prompt, + input_image=input_image, + height=480, width=832, num_frames=81, + num_inference_steps=40, cfg_scale=3.0, + seed=0, +) +save_video(video, "video_lingbot-video-moe-30b-a3b_ti2v.mp4", fps=15, quality=10)