""" """ from typing import Any from typing import Callable from typing import ParamSpec from torchao.quantization import quantize_ from torchao.quantization import Float8DynamicActivationFloat8WeightConfig import spaces import torch from torch.utils._pytree import tree_map P = ParamSpec('P') # Mark only the plain-tensor args dynamic, with Dim.AUTO (dynamic where the # model allows, static where it specializes). image_rotary_emb is left static # (via tree_map) — its nested structure drifts across diffusers versions and # hardcoding it breaks torch.export's tree matching. _AUTO = torch.export.Dim.AUTO TRANSFORMER_DYNAMIC_SHAPES = { 'hidden_states': {1: _AUTO}, 'encoder_hidden_states': {1: _AUTO}, 'encoder_hidden_states_mask': {1: _AUTO}, } INDUCTOR_CONFIGS = { 'conv_1x1_as_mm': True, 'epilogue_fusion': False, 'coordinate_descent_tuning': True, 'coordinate_descent_check_all_directions': True, 'max_autotune': False, 'triton.cudagraphs': False, } def optimize_pipeline_(pipeline: Callable[P, Any], *args: P.args, **kwargs: P.kwargs): @spaces.GPU(duration=750, size="xlarge") def compile_transformer(): with spaces.aoti_capture(pipeline.transformer) as call: pipeline(*args, **kwargs) dynamic_shapes = tree_map(lambda t: None, call.kwargs) # Only mark an arg dynamic if it's actually a Tensor in this call — arg # names AND presence (e.g. encoder_hidden_states_mask=None) drift across # diffusers versions; marking a None arg dynamic errors. for _k, _v in TRANSFORMER_DYNAMIC_SHAPES.items(): if isinstance(call.kwargs.get(_k), torch.Tensor): dynamic_shapes[_k] = _v # quantize_(pipeline.transformer, Float8DynamicActivationFloat8WeightConfig()) exported = torch.export.export( mod=pipeline.transformer, args=call.args, kwargs=call.kwargs, dynamic_shapes=dynamic_shapes, ) return spaces.aoti_compile(exported, INDUCTOR_CONFIGS) spaces.aoti_apply(compile_transformer(), pipeline.transformer)