diffusers / depth-processor-custom-block

Depth Processor Block

This is a custom block designed to extract depth maps from input images using the Depth Anything Model model. The model can be used as a processor to generate conditioning images for ControlNets.

How to use

import torch
from diffusers.modular_pipelines import ModularPipelineBlocks, SequentialPipelineBlocks
from diffusers.modular_pipelines.stable_diffusion_xl import TEXT2IMAGE_BLOCKS, CONTROLNET_BLOCKS
from diffusers.utils import load_image

# fetch the depth processor block that will create our depth map
depth_processor_block = ModularPipelineBlocks.from_pretrained("diffusers/depth-processor-custom-block", trust_remote_code=true)

my_blocks = TEXT2IMAGE_BLOCKS.copy()
my_blocks.insert("depth_processor", depth_processor_block, 1)

# replace text to image denoise block with controlnet denoise block
my_blocks.sub_blocks["denoise"] = CONTROLNET_BLOCKS["denoise"]

# create our initial set of controlnet blocks
blocks = SequentialPipelineBlocks.from_blocks_dict(my_blocks)

repo_id = "diffusers/modular-stable-diffusion-xl-base-1.0"

# Initialize the pipeline object we can use to run our blocks
pipe = blocks.init_pipeline(repo_id)

# Load model component weights
pipe.load_components(torch_dtype=torch.float16, device_map="cuda")

image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/tasks/car.jpg?download=true")
image = image.resize((1024, 1024))

prompt = ["A red car"]

output = pipe(
    prompt=prompt,
    image=image,
    num_inference_steps=35,
    guidance_scale=7.5,
    output_type="pil",
)