From 075233d16f3e5086d48d5fdcb8c3203f8cf94d76 Mon Sep 17 00:00:00 2001 From: lucasruan1618 Date: Tue, 18 Aug 2026 03:29:15 +0000 Subject: [PATCH 1/2] [Modular]:Support Wan2 video-to-video modular pipeline --- src/diffusers/__init__.py | 4 + src/diffusers/modular_pipelines/__init__.py | 4 + .../modular_pipelines/modular_pipeline.py | 5 + .../modular_pipelines/wan/__init__.py | 4 + .../modular_pipelines/wan/before_denoise.py | 144 ++++++++++++++++- .../modular_pipelines/wan/denoise.py | 2 +- .../modular_pipelines/wan/encoders.py | 67 +++++++- .../wan/modular_blocks_wan_v2v.py | 150 ++++++++++++++++++ .../modular_pipelines/wan/modular_pipeline.py | 10 ++ .../wan/test_modular_pipeline_wan.py | 60 ++++++- 10 files changed, 444 insertions(+), 6 deletions(-) create mode 100644 src/diffusers/modular_pipelines/wan/modular_blocks_wan_v2v.py diff --git a/src/diffusers/__init__.py b/src/diffusers/__init__.py index 4bec0f5bd7ff..91051eba642f 100644 --- a/src/diffusers/__init__.py +++ b/src/diffusers/__init__.py @@ -573,6 +573,8 @@ "WanImage2VideoAutoBlocks", "WanImage2VideoModularPipeline", "WanModularPipeline", + "WanVideoToVideoBlocks", + "WanVideoToVideoModularPipeline", "ZImageAutoBlocks", "ZImageModularPipeline", ] @@ -1416,6 +1418,8 @@ WanImage2VideoAutoBlocks, WanImage2VideoModularPipeline, WanModularPipeline, + WanVideoToVideoBlocks, + WanVideoToVideoModularPipeline, ZImageAutoBlocks, ZImageModularPipeline, ) diff --git a/src/diffusers/modular_pipelines/__init__.py b/src/diffusers/modular_pipelines/__init__.py index a8a23a39d517..710c556d13a2 100644 --- a/src/diffusers/modular_pipelines/__init__.py +++ b/src/diffusers/modular_pipelines/__init__.py @@ -57,10 +57,12 @@ "WanBlocks", "Wan22Blocks", "WanImage2VideoAutoBlocks", + "WanVideoToVideoBlocks", "Wan22Image2VideoBlocks", "WanModularPipeline", "Wan22ModularPipeline", "WanImage2VideoModularPipeline", + "WanVideoToVideoModularPipeline", "Wan22Image2VideoModularPipeline", ] _import_structure["helios"] = [ @@ -236,6 +238,8 @@ WanImage2VideoAutoBlocks, WanImage2VideoModularPipeline, WanModularPipeline, + WanVideoToVideoBlocks, + WanVideoToVideoModularPipeline, ) from .wan_animate_2 import ( WanAnimate2Blocks, diff --git a/src/diffusers/modular_pipelines/modular_pipeline.py b/src/diffusers/modular_pipelines/modular_pipeline.py index 1ff8968abc9b..0ab62e4f9420 100644 --- a/src/diffusers/modular_pipelines/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/modular_pipeline.py @@ -107,6 +107,10 @@ def _wan_i2v_map_fn(config_dict=None): return "WanImage2VideoModularPipeline" +def _wan_v2v_map_fn(config_dict=None): + return "WanVideoToVideoModularPipeline" + + def _krea2_map_fn(config_dict=None): if config_dict is None: return "Krea2ModularPipeline" @@ -135,6 +139,7 @@ def _helios_pyramid_map_fn(config_dict=None): ("wan-animate-2", _create_default_map_fn("WanAnimate2ModularPipeline")), ("wan-animate-2-distilled", _create_default_map_fn("WanAnimate2DistilledModularPipeline")), ("wan-i2v", _wan_i2v_map_fn), + ("wan-v2v", _wan_v2v_map_fn), ("flux", _create_default_map_fn("FluxModularPipeline")), ("flux-kontext", _create_default_map_fn("FluxKontextModularPipeline")), ("flux2", _create_default_map_fn("Flux2ModularPipeline")), diff --git a/src/diffusers/modular_pipelines/wan/__init__.py b/src/diffusers/modular_pipelines/wan/__init__.py index 284b6c9fa436..723eeeba16eb 100644 --- a/src/diffusers/modular_pipelines/wan/__init__.py +++ b/src/diffusers/modular_pipelines/wan/__init__.py @@ -25,11 +25,13 @@ _import_structure["modular_blocks_wan22"] = ["Wan22Blocks"] _import_structure["modular_blocks_wan22_i2v"] = ["Wan22Image2VideoBlocks"] _import_structure["modular_blocks_wan_i2v"] = ["WanImage2VideoAutoBlocks"] + _import_structure["modular_blocks_wan_v2v"] = ["WanVideoToVideoBlocks"] _import_structure["modular_pipeline"] = [ "Wan22Image2VideoModularPipeline", "Wan22ModularPipeline", "WanImage2VideoModularPipeline", "WanModularPipeline", + "WanVideoToVideoModularPipeline", ] if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: @@ -43,11 +45,13 @@ from .modular_blocks_wan22 import Wan22Blocks from .modular_blocks_wan22_i2v import Wan22Image2VideoBlocks from .modular_blocks_wan_i2v import WanImage2VideoAutoBlocks + from .modular_blocks_wan_v2v import WanVideoToVideoBlocks from .modular_pipeline import ( Wan22Image2VideoModularPipeline, Wan22ModularPipeline, WanImage2VideoModularPipeline, WanModularPipeline, + WanVideoToVideoModularPipeline, ) else: import sys diff --git a/src/diffusers/modular_pipelines/wan/before_denoise.py b/src/diffusers/modular_pipelines/wan/before_denoise.py index 6b0874037b0d..915bddb7e971 100644 --- a/src/diffusers/modular_pipelines/wan/before_denoise.py +++ b/src/diffusers/modular_pipelines/wan/before_denoise.py @@ -208,7 +208,7 @@ def expected_components(self) -> list[ComponentSpec]: @property def inputs(self) -> list[InputParam]: return [ - InputParam("num_videos_per_prompt", default=1), + InputParam("num_videos_per_prompt", default=1, description="The number of videos to generate per prompt."), InputParam( "prompt_embeds", required=True, @@ -552,3 +552,145 @@ def __call__(self, components: WanModularPipeline, state: PipelineState) -> Pipe self.set_block_state(state, block_state) return components, state + + +class WanVideoToVideoSetTimestepsStep(ModularPipelineBlocks): + model_name = "wan-v2v" + + @property + def expected_components(self) -> list[ComponentSpec]: + return [ + ComponentSpec("scheduler", UniPCMultistepScheduler), + ] + + @property + def description(self) -> str: + return "Set the scheduler timesteps and select the video-to-video denoising schedule from strength." + + @property + def inputs(self) -> list[InputParam]: + return [ + InputParam("num_inference_steps", default=50, description="The number of denoising steps."), + InputParam("timesteps", description="Custom timesteps for the denoising process."), + InputParam("sigmas", description="Custom sigmas for the denoising process."), + InputParam( + "strength", + default=0.8, + type_hint=float, + description="The amount of noise added to the input video latents.", + ), + InputParam("batch_size", required=True, type_hint=int), + InputParam( + "num_videos_per_prompt", + default=1, + type_hint=int, + description="The number of videos to generate per prompt.", + ), + ] + + @property + def intermediate_outputs(self) -> list[OutputParam]: + return [ + OutputParam("timesteps", type_hint=torch.Tensor, description="The selected denoising timesteps."), + OutputParam( + "num_inference_steps", + type_hint=int, + description="The number of selected denoising steps.", + ), + OutputParam( + "latent_timestep", + type_hint=torch.Tensor, + description="The timestep used to add noise to the input video latents.", + ), + ] + + @torch.no_grad() + def __call__(self, components: WanModularPipeline, state: PipelineState) -> PipelineState: + block_state = self.get_block_state(state) + device = components._execution_device + + timesteps, num_inference_steps = retrieve_timesteps( + components.scheduler, + block_state.num_inference_steps, + device, + block_state.timesteps, + block_state.sigmas, + ) + init_timestep = min(int(num_inference_steps * block_state.strength), num_inference_steps) + t_start = max(num_inference_steps - init_timestep, 0) + block_state.timesteps = timesteps[t_start * components.scheduler.order :] + block_state.num_inference_steps = num_inference_steps - t_start + block_state.latent_timestep = block_state.timesteps[:1].repeat( + block_state.batch_size * block_state.num_videos_per_prompt + ) + + self.set_block_state(state, block_state) + return components, state + + +class WanVideoToVideoPrepareLatentsStep(ModularPipelineBlocks): + model_name = "wan-v2v" + + @property + def expected_components(self) -> list[ComponentSpec]: + return [ + ComponentSpec("scheduler", UniPCMultistepScheduler), + ] + + @property + def description(self) -> str: + return "Add noise at the selected timestep to the encoded input video latents." + + @property + def inputs(self) -> list[InputParam]: + return [ + InputParam( + "video_latents", + required=True, + type_hint=torch.Tensor, + description="Normalized VAE latents of the input video.", + ), + InputParam("latent_timestep", required=True, type_hint=torch.Tensor), + InputParam( + "latents", + type_hint=torch.Tensor | None, + description="Pre-generated noisy video latents to use instead of adding noise to the input video.", + ), + InputParam("generator", description="Torch generator for deterministic noise generation."), + ] + + @property + def intermediate_outputs(self) -> list[OutputParam]: + return [ + OutputParam( + "latents", + type_hint=torch.Tensor, + description="Noisy video latents used to start the denoising process.", + ) + ] + + @torch.no_grad() + def __call__(self, components: WanModularPipeline, state: PipelineState) -> PipelineState: + block_state = self.get_block_state(state) + device = components._execution_device + + if block_state.latents is None: + noise = randn_tensor( + block_state.video_latents.shape, + generator=block_state.generator, + device=device, + dtype=torch.float32, + ) + if hasattr(components.scheduler, "add_noise"): + block_state.latents = components.scheduler.add_noise( + block_state.video_latents, noise, block_state.latent_timestep + ) + else: + block_state.latents = components.scheduler.scale_noise( + block_state.video_latents, block_state.latent_timestep, noise + ) + else: + block_state.latents = block_state.latents.to(device) + + self.set_block_state(state, block_state) + return components, state diff --git a/src/diffusers/modular_pipelines/wan/denoise.py b/src/diffusers/modular_pipelines/wan/denoise.py index 63093c37f7ec..1cb1f6e8853f 100644 --- a/src/diffusers/modular_pipelines/wan/denoise.py +++ b/src/diffusers/modular_pipelines/wan/denoise.py @@ -158,7 +158,7 @@ def description(self) -> str: @property def inputs(self) -> list[tuple[str, Any]]: inputs = [ - InputParam("attention_kwargs"), + InputParam("attention_kwargs", description="Additional kwargs for attention processors."), InputParam( "num_inference_steps", required=True, diff --git a/src/diffusers/modular_pipelines/wan/encoders.py b/src/diffusers/modular_pipelines/wan/encoders.py index c7d5df48e7be..992fa4c9f5a2 100644 --- a/src/diffusers/modular_pipelines/wan/encoders.py +++ b/src/diffusers/modular_pipelines/wan/encoders.py @@ -178,9 +178,9 @@ def expected_components(self) -> list[ComponentSpec]: @property def inputs(self) -> list[InputParam]: return [ - InputParam("prompt"), - InputParam("negative_prompt"), - InputParam("max_sequence_length", default=512), + InputParam("prompt", description="The prompt or prompts to guide video generation."), + InputParam("negative_prompt", description="The prompt or prompts not to guide video generation."), + InputParam("max_sequence_length", default=512, description="Maximum sequence length for prompt encoding."), ] @property @@ -563,6 +563,67 @@ def __call__(self, components: WanModularPipeline, state: PipelineState) -> Pipe return components, state +class WanVideoVaeEncoderStep(ModularPipelineBlocks): + model_name = "wan-v2v" + + @property + def description(self) -> str: + return "Preprocess and encode the input video into normalized VAE latents for video-to-video generation." + + @property + def expected_components(self) -> list[ComponentSpec]: + return [ + ComponentSpec("vae", AutoencoderKLWan), + ComponentSpec( + "video_processor", + VideoProcessor, + config=FrozenDict({"vae_scale_factor": 8}), + default_creation_method="from_config", + ), + ] + + @property + def inputs(self) -> list[InputParam]: + return [ + InputParam("video", required=True, description="The input video to transform."), + InputParam("height", type_hint=int, description="The height in pixels of the generated video."), + InputParam("width", type_hint=int, description="The width in pixels of the generated video."), + ] + + @property + def intermediate_outputs(self) -> list[OutputParam]: + return [ + OutputParam( + "video_latents", + type_hint=torch.Tensor, + description="Normalized VAE latents of the input video.", + ) + ] + + @torch.no_grad() + def __call__(self, components: WanModularPipeline, state: PipelineState) -> PipelineState: + block_state = self.get_block_state(state) + + block_state.height = block_state.height or components.default_height + block_state.width = block_state.width or components.default_width + + device = components._execution_device + video = components.video_processor.preprocess_video( + block_state.video, height=block_state.height, width=block_state.width + ).to(device=device, dtype=torch.float32) + block_state.video_latents = encode_vae_image( + video_tensor=video, + vae=components.vae, + generator=None, + device=device, + dtype=components.vae.dtype, + latent_channels=components.num_channels_latents, + ).to(torch.float32) + + self.set_block_state(state, block_state) + return components, state + + class WanPrepareFirstFrameLatentsStep(ModularPipelineBlocks): model_name = "wan" diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_v2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_v2v.py new file mode 100644 index 000000000000..eef0fdd16f83 --- /dev/null +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_v2v.py @@ -0,0 +1,150 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..modular_pipeline import SequentialPipelineBlocks +from ..modular_pipeline_utils import OutputParam +from .before_denoise import ( + WanTextInputStep, + WanVideoToVideoPrepareLatentsStep, + WanVideoToVideoSetTimestepsStep, +) +from .decoders import WanVaeDecoderStep +from .denoise import WanDenoiseStep +from .encoders import WanTextEncoderStep, WanVideoVaeEncoderStep + + +# auto_docstring +class WanVideoToVideoCoreDenoiseStep(SequentialPipelineBlocks): + """ + Denoise the noisy input video latents for video-to-video generation. + + Components: + transformer (`WanTransformer3DModel`) + scheduler (`UniPCMultistepScheduler`) + guider (`ClassifierFreeGuidance`) + + Inputs: + num_videos_per_prompt (`None`, *optional*, defaults to 1): + The number of videos to generate per prompt. + prompt_embeds (`Tensor`): + Pre-generated text embeddings. Can be generated from text_encoder step. + negative_prompt_embeds (`Tensor`, *optional*): + Pre-generated negative text embeddings. Can be generated from text_encoder step. + num_inference_steps (`None`, *optional*, defaults to 50): + The number of denoising steps. + timesteps (`None`, *optional*): + Custom timesteps for the denoising process. + sigmas (`None`, *optional*): + Custom sigmas for the denoising process. + strength (`float`, *optional*, defaults to 0.8): + The amount of noise added to the input video latents. + video_latents (`Tensor`): + Normalized VAE latents of the input video. + latents (`Tensor | NoneType`, *optional*): + Pre-generated noisy video latents to use instead of adding noise to the input video. + generator (`None`, *optional*): + Torch generator for deterministic noise generation. + attention_kwargs (`None`, *optional*): + Additional kwargs for attention processors. + + Outputs: + latents (`Tensor`): + Denoised latents. + """ + + model_name = "wan-v2v" + block_classes = [ + WanTextInputStep, + WanVideoToVideoSetTimestepsStep, + WanVideoToVideoPrepareLatentsStep, + WanDenoiseStep, + ] + block_names = ["input", "set_timesteps", "prepare_latents", "denoise"] + + @property + def description(self): + return "Denoise the noisy input video latents for video-to-video generation." + + @property + def outputs(self): + return [OutputParam.template("latents")] + + +# auto_docstring +class WanVideoToVideoBlocks(SequentialPipelineBlocks): + """ + Modular pipeline blocks for Wan video-to-video generation. + + Components: + text_encoder (`UMT5EncoderModel`) + tokenizer (`AutoTokenizer`) + guider (`ClassifierFreeGuidance`) + vae (`AutoencoderKLWan`) + video_processor (`VideoProcessor`) + transformer (`WanTransformer3DModel`) + scheduler (`UniPCMultistepScheduler`) + + Inputs: + prompt (`None`, *optional*): + The prompt or prompts to guide video generation. + negative_prompt (`None`, *optional*): + The prompt or prompts not to guide video generation. + max_sequence_length (`None`, *optional*, defaults to 512): + Maximum sequence length for prompt encoding. + video (`None`): + The input video to transform. + height (`int`, *optional*): + The height in pixels of the generated video. + width (`int`, *optional*): + The width in pixels of the generated video. + num_videos_per_prompt (`None`, *optional*, defaults to 1): + The number of videos to generate per prompt. + num_inference_steps (`None`, *optional*, defaults to 50): + The number of denoising steps. + timesteps (`None`, *optional*): + Custom timesteps for the denoising process. + sigmas (`None`, *optional*): + Custom sigmas for the denoising process. + strength (`float`, *optional*, defaults to 0.8): + The amount of noise added to the input video latents. + latents (`Tensor | NoneType`, *optional*): + Pre-generated noisy video latents to use instead of adding noise to the input video. + generator (`None`, *optional*): + Torch generator for deterministic noise generation. + attention_kwargs (`None`, *optional*): + Additional kwargs for attention processors. + output_type (`str`, *optional*, defaults to np): + The output type of the decoded videos + + Outputs: + videos (`list`): + The generated videos. + """ + + model_name = "wan-v2v" + block_classes = [ + WanTextEncoderStep, + WanVideoVaeEncoderStep, + WanVideoToVideoCoreDenoiseStep, + WanVaeDecoderStep, + ] + block_names = ["text_encoder", "vae_encoder", "denoise", "decode"] + + @property + def description(self): + return "Modular pipeline blocks for Wan video-to-video generation." + + @property + def outputs(self): + return [OutputParam.template("videos")] diff --git a/src/diffusers/modular_pipelines/wan/modular_pipeline.py b/src/diffusers/modular_pipelines/wan/modular_pipeline.py index a360440c9251..7d69007ca307 100644 --- a/src/diffusers/modular_pipelines/wan/modular_pipeline.py +++ b/src/diffusers/modular_pipelines/wan/modular_pipeline.py @@ -121,6 +121,16 @@ class WanImage2VideoModularPipeline(WanModularPipeline): default_blocks_name = "WanImage2VideoAutoBlocks" +class WanVideoToVideoModularPipeline(WanModularPipeline): + """ + A ModularPipeline for Wan video-to-video. + + > [!WARNING] > This is an experimental feature and is likely to change in the future. + """ + + default_blocks_name = "WanVideoToVideoBlocks" + + class Wan22ModularPipeline(WanModularPipeline): """ A ModularPipeline for Wan2.2 text2video. diff --git a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py index d35c21455ba9..8db601584775 100644 --- a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py +++ b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py @@ -14,9 +14,17 @@ # limitations under the License. import pytest +from PIL import Image -from diffusers.modular_pipelines import WanBlocks, WanModularPipeline +from diffusers import WanVideoToVideoPipeline +from diffusers.modular_pipelines import ( + WanBlocks, + WanModularPipeline, + WanVideoToVideoBlocks, + WanVideoToVideoModularPipeline, +) +from ...testing_utils import assert_tensors_close from ..testing_utils import ( BaseModularPipelineTesterConfig, ModularLoadingTesterMixin, @@ -66,3 +74,53 @@ class TestWanModularPipelineWorkflow(WanModularPipelineTesterConfig, ModularWork class TestWanModularPipelineMemory(WanModularPipelineTesterConfig, ModularMemoryTesterMixin): pass + + +class WanVideoToVideoModularPipelineTesterConfig(BaseModularPipelineTesterConfig): + pipeline_class = WanVideoToVideoModularPipeline + pipeline_blocks_class = WanVideoToVideoBlocks + pretrained_model_name_or_path = "hf-internal-testing/tiny-wan-modular-pipe" + params = frozenset(["prompt", "video", "height", "width", "strength"]) + batch_params = frozenset(["prompt", "video"]) + optional_params = frozenset(["num_inference_steps", "num_videos_per_prompt", "latents", "output_type"]) + output_name = "videos" + + def get_dummy_inputs(self, seed=0): + return { + "prompt": "A painting of a squirrel eating a burger", + "negative_prompt": "", + "video": [Image.new("RGB", (16, 16))] * 9, + "generator": self.get_generator(seed), + "num_inference_steps": 4, + "height": 16, + "width": 16, + "strength": 0.75, + "max_sequence_length": 16, + "output_type": "pt", + } + + +class TestWanVideoToVideoModularPipelineFast(WanVideoToVideoModularPipelineTesterConfig, ModularPipelineTesterMixin): + def test_standard_pipeline_parity(self): + modular_pipeline = self.get_pipeline() + native_pipeline = WanVideoToVideoPipeline( + scheduler=modular_pipeline.scheduler, + text_encoder=modular_pipeline.text_encoder, + tokenizer=modular_pipeline.tokenizer, + transformer=modular_pipeline.transformer, + vae=modular_pipeline.vae, + ) + + modular_output = modular_pipeline(**self.get_dummy_inputs(), output="videos") + native_inputs = self.get_dummy_inputs() + native_output = native_pipeline(guidance_scale=5.0, **native_inputs).frames + + assert_tensors_close(modular_output[0], native_output[0], atol=1e-4) + + +class TestWanVideoToVideoModularPipelineLoading(WanVideoToVideoModularPipelineTesterConfig, ModularLoadingTesterMixin): + pass + + +class TestWanVideoToVideoModularPipelineMemory(WanVideoToVideoModularPipelineTesterConfig, ModularMemoryTesterMixin): + pass From 929ca4b2752e85ca8549064a8bc0ae0160808f65 Mon Sep 17 00:00:00 2001 From: lucasruan1618 Date: Sun, 6 Sep 2026 04:50:15 +0000 Subject: [PATCH 2/2] Fix Wan video-to-video batch encoding and dimension validation --- .../modular_pipelines/wan/encoders.py | 9 ++++++++- .../wan/modular_blocks_wan_v2v.py | 18 ++++++------------ .../wan/test_modular_pipeline_wan.py | 12 ++++++++++++ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/diffusers/modular_pipelines/wan/encoders.py b/src/diffusers/modular_pipelines/wan/encoders.py index 992fa4c9f5a2..39a35306f4b6 100644 --- a/src/diffusers/modular_pipelines/wan/encoders.py +++ b/src/diffusers/modular_pipelines/wan/encoders.py @@ -588,6 +588,7 @@ def inputs(self) -> list[InputParam]: InputParam("video", required=True, description="The input video to transform."), InputParam("height", type_hint=int, description="The height in pixels of the generated video."), InputParam("width", type_hint=int, description="The width in pixels of the generated video."), + InputParam("generator", description="Torch generator for deterministic latent generation."), ] @property @@ -600,12 +601,18 @@ def intermediate_outputs(self) -> list[OutputParam]: ) ] + @staticmethod + def check_inputs(height: int, width: int): + if height % 16 != 0 or width % 16 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.") + @torch.no_grad() def __call__(self, components: WanModularPipeline, state: PipelineState) -> PipelineState: block_state = self.get_block_state(state) block_state.height = block_state.height or components.default_height block_state.width = block_state.width or components.default_width + self.check_inputs(block_state.height, block_state.width) device = components._execution_device video = components.video_processor.preprocess_video( @@ -614,7 +621,7 @@ def __call__(self, components: WanModularPipeline, state: PipelineState) -> Pipe block_state.video_latents = encode_vae_image( video_tensor=video, vae=components.vae, - generator=None, + generator=block_state.generator, device=device, dtype=components.vae.dtype, latent_channels=components.num_channels_latents, diff --git a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_v2v.py b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_v2v.py index eef0fdd16f83..926218c4a65e 100644 --- a/src/diffusers/modular_pipelines/wan/modular_blocks_wan_v2v.py +++ b/src/diffusers/modular_pipelines/wan/modular_blocks_wan_v2v.py @@ -30,9 +30,7 @@ class WanVideoToVideoCoreDenoiseStep(SequentialPipelineBlocks): Denoise the noisy input video latents for video-to-video generation. Components: - transformer (`WanTransformer3DModel`) - scheduler (`UniPCMultistepScheduler`) - guider (`ClassifierFreeGuidance`) + transformer (`WanTransformer3DModel`) scheduler (`UniPCMultistepScheduler`) guider (`ClassifierFreeGuidance`) Inputs: num_videos_per_prompt (`None`, *optional*, defaults to 1): @@ -87,13 +85,9 @@ class WanVideoToVideoBlocks(SequentialPipelineBlocks): Modular pipeline blocks for Wan video-to-video generation. Components: - text_encoder (`UMT5EncoderModel`) - tokenizer (`AutoTokenizer`) - guider (`ClassifierFreeGuidance`) - vae (`AutoencoderKLWan`) - video_processor (`VideoProcessor`) - transformer (`WanTransformer3DModel`) - scheduler (`UniPCMultistepScheduler`) + text_encoder (`UMT5EncoderModel`) tokenizer (`AutoTokenizer`) guider (`ClassifierFreeGuidance`) vae + (`AutoencoderKLWan`) video_processor (`VideoProcessor`) transformer (`WanTransformer3DModel`) scheduler + (`UniPCMultistepScheduler`) Inputs: prompt (`None`, *optional*): @@ -108,6 +102,8 @@ class WanVideoToVideoBlocks(SequentialPipelineBlocks): The height in pixels of the generated video. width (`int`, *optional*): The width in pixels of the generated video. + generator (`None`, *optional*): + Torch generator for deterministic latent generation. num_videos_per_prompt (`None`, *optional*, defaults to 1): The number of videos to generate per prompt. num_inference_steps (`None`, *optional*, defaults to 50): @@ -120,8 +116,6 @@ class WanVideoToVideoBlocks(SequentialPipelineBlocks): The amount of noise added to the input video latents. latents (`Tensor | NoneType`, *optional*): Pre-generated noisy video latents to use instead of adding noise to the input video. - generator (`None`, *optional*): - Torch generator for deterministic noise generation. attention_kwargs (`None`, *optional*): Additional kwargs for attention processors. output_type (`str`, *optional*, defaults to np): diff --git a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py index 8db601584775..3766f8f4b015 100644 --- a/tests/modular_pipelines/wan/test_modular_pipeline_wan.py +++ b/tests/modular_pipelines/wan/test_modular_pipeline_wan.py @@ -101,6 +101,18 @@ def get_dummy_inputs(self, seed=0): class TestWanVideoToVideoModularPipelineFast(WanVideoToVideoModularPipelineTesterConfig, ModularPipelineTesterMixin): + def test_inference_batch_single_identical(self, batch_size=2, expected_max_diff=2e-3): + super().test_inference_batch_single_identical(batch_size=batch_size, expected_max_diff=expected_max_diff) + + @pytest.mark.parametrize(("height", "width"), [(24, 16), (16, 24)]) + def test_height_and_width_must_be_divisible_by_16(self, height, width): + pipeline = self.get_pipeline() + inputs = self.get_dummy_inputs() + inputs.update(height=height, width=width) + + with pytest.raises(ValueError, match="height.*width.*divisible by 16"): + pipeline(**inputs, output="videos") + def test_standard_pipeline_parity(self): modular_pipeline = self.get_pipeline() native_pipeline = WanVideoToVideoPipeline(