+
    $Lj9                         ^ RI t ^ RIt^ RIt^ RIHt ^ RIHt ^RIHt ^RI	H
t
 ^RIHt ^RIHt ^RIHt R	 tR
 t ! R R4      t ! R R4      t ! R R]4      t ! R R]
4      t ! R R4      t ! R R4      tR# )    N)deepcopy)optim)AcceleratedOptimizer)AcceleratedScheduler)DistributedType)is_bnb_available)compare_versionsc                   V P                   P                  4        UUu/ uF  w  rVR9   g   K  WbK  	  ppp^ RIHp Tp\	        RRR4      '       d   RVR&   \        V \        P                  4      p\        4       '       dI   V'       gA   ^ RI	Hp \        WP                  VP                  34      '       d    V P                  ^ 8H  pMRpV'       d   RVR&   \	        RRR	4      '       d   \        V \        P                  4      p\        4       '       dF   V'       g>   ^ RI	Hp \        WP                  VP                  34      '       d    V P                  ^ 8H  pV'       d	   ^ R
IHp	 T	p\        RR7      '       dO   \	        RRR4      '       d<   ^ RI	Hp
Hp \        W
V34      '       d!    V P                  ^ 8H  pV'       d	   ^ RIHp TpV! V P,                  3/ VB # u uppi   \         d    T P                  P                  ^ 8H  p EL?i ; i  \         d    T P                  P                  ^ 8H  p Li ; i  \         d    T P                  P                  ^ 8H  p Li ; i)zx
Args:
    optimizer: torch.optim.Optimizer

Returns the DeepSeedCPUOptimizer (deepspeed.ops) version of the optimizer.
)DeepSpeedCPUAdam	deepspeedz>=z0.3.1F
adamw_modeNTz0.5.5)DeepSpeedCPUAdagradz0.38.0)min_versionz0.11.0)Lion	Lion32bit)DeepSpeedCPULion)lrweight_decay)defaultsitemsdeepspeed.ops.adamr   r	   
isinstancer   AdamWr   bitsandbytes.optim
AdamW32bit
optim_bitsAttributeErrorargsAdagradAdagrad32bitdeepspeed.ops.adagradr   r   r   deepspeed.ops.lionr   param_groups)	optimizerkvr   r   optimizer_classis_adawbnb_optis_adar   r   r   is_bnb_32bitsr   s   &             F/app/.local/lib/python3.14/site-packages/accelerate/utils/deepspeed.pymap_pytorch_optim_to_deepspeedr-      s
    "+!3!3!9!9!;[!;qDZ?Z!;H[ 4&O T733!&Y4g0)mmW5G5G%HII>'22b8G  %)H\" T733Iu}}5 f0)oow7K7K%LMM=&11R7F A1O H--2B;PTV^2_2_6i	!233@ ) 4 4 : ?"2911>X>>y \& & >'nn772=G>( & =&^^66"<F= " @ ) 9 9R ?@s@   GG7G! 	H )H6 !$H	H	$H32H36$IIc                   V P                   \        P                  8w  d   \        R4      h\	        V P
                  \        4      '       g   V P
                  # \        R V P
                  P                  4        4       4      # )z
Returns the currently active DeepSpeedPlugin.

Raises:
    ValueError: If DeepSpeed was not enabled and this function is called.
a!  Couldn't retrieve the active `DeepSpeedPlugin` as none were enabled. Please make sure that either `Accelerator` is configured for `deepspeed` or make sure that the desired `DeepSpeedPlugin` has been enabled (`AcceleratorState().select_deepspeed_plugin(name)`) before calling this function.c              3   L   "   T F  qP                   '       g   K  Vx  K  	  R # 5iN)selected).0plugins   & r,   	<genexpr>.get_active_deepspeed_plugin.<locals>.<genexpr>t   s     Y%E6%Es   $
$)	distributed_typer   	DEEPSPEED
ValueErrorr   deepspeed_pluginsdictnextvalues)states   &r,   get_active_deepspeed_pluginr>   d   sk     !:!::,
 	
 e--t44&&&YU%<%<%C%C%EYYY    c                   h   a  ] tR t^wt o RtR tR tR tRR ltRR lt	R t
R	 tR
 tR tR tRtV tR# )HfDeepSpeedConfiga"  
This object contains a DeepSpeed configuration dictionary and can be quickly queried for things like zero stage.

A `weakref` of this object is stored in the module's globals to be able to access the config from areas where
things like the Trainer object is not available (e.g. `from_pretrained` and `_get_resized_embeddings`). Therefore
it's important that this object remains alive while the program is still running.

[`Trainer`] uses the `HfTrainerDeepSpeedConfig` subclass instead. That subclass has logic to sync the configuration
with values of [`TrainingArguments`] by replacing special placeholder values: `"auto"`. Without this special logic
the DeepSpeed configuration is not modified in any way.

Args:
    config_file_or_dict (`Union[str, Dict]`): path to DeepSpeed config file or dict.

c                ~   \        V\        4      '       d   \        V4      pMr\        P                  P                  V4      '       d6   \        VR R7      ;_uu_ 4       p\        P                  ! V4      pRRR4       M  \        P                  ! V4      pXV n        V P%                  4        R#   + '       g   i     L); i  \        P                   d?    \        P                  ! T4      P                  R 4      p\        P                  ! T4      p L~i ; i  \        \        \         3 d    \!        RT 24      hi ; i)zutf-8)encodingNzoExpected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: )r   r:   r   ospathexistsopenjsonloadloadsJSONDecodeErrorbase64urlsafe_b64decodedecodeUnicodeDecodeErrorr   r8   configset_stage_and_offload)selfconfig_file_or_dictrP   fconfig_decodeds   &&   r,   __init__HfDeepSpeedConfig.__init__   s   )400 12FWW^^/00)G<<1 =<8!ZZ(;<F ""$% =< ++ 8%+%=%=>Q%R%Y%YZa%bN!ZZ7F8 '
C   F  GZ  F[  \ s1   B.?C .B>	ADD DD %D<c                L   V P                  R R4      V n        RV n        V P                  4       '       g   V P	                  4       '       dZ   \        RR.4      p\        V P                  R4      V P                  R4      .4      p\        W!,          4      ^ 8  d
   RV n        R# R# R# )	zzero_optimization.stageFcpunvmez*zero_optimization.offload_optimizer.devicez&zero_optimization.offload_param.deviceTN)	get_value_stage_offloadis_zero2is_zero3setlen)rR   offload_devices_validoffload_devicess   &  r,   rQ   'HfDeepSpeedConfig.set_stage_and_offload   s     nn%>C ==??dmmoo$'$8!!NN#OPNN#KLO ?:;a? $ @ .r?   c                    V P                   pVP                  R 4      pVP                  4       pV F  pVP                  V4      pVe   K  RV3u # 	  W$3# ).N)rP   splitpopget)rR   ds_key_longrP   nodesds_keynodes   &&    r,   find_config_node"HfDeepSpeedConfig.find_config_node   sY     !!#&DZZ%F~V|# 
 ~r?   Nc                V    V P                  V4      w  r4Vf   V# VP                  WB4      # )z7
Returns the set value or `default` if no value is set
)ro   rj   )rR   rk   defaultrP   rm   s   &&&  r,   r\   HfDeepSpeedConfig.get_value   s/     ..{;>Nzz&**r?   c                    V P                   pVP                  R4      pV F@  pTpVP                  V4      pVe   K  V'       d   \        RV RV P                    24      h R# 	  Xe   VP	                  X4       R# R# )z{
Deletes a sub-section of the config file if it's found.

Unless `must_exist` is `True` the section doesn't have to exist.
rg   NzCan't find z entry in the config: )rP   rh   rj   r8   ri   )rR   rk   
must_existrP   rl   rn   parent_configs   &&&    r,   del_config_sub_tree%HfDeepSpeedConfig.del_config_sub_tree   s      !!#&D"MZZ%F~${;-?UVZVaVaUb%cdd  $d# %r?   c                F    V P                  V4      pVf   R# \        V4      # )z
Returns `True`/``False` only if the value is set, always `False` otherwise. So use this method to ask the very
specific question of whether the value is set to `True` (and it's not set to `False`` or isn't set).

Fr\   boolrR   rk   values   && r,   is_trueHfDeepSpeedConfig.is_true   s%     {+u64;6r?   c                P    V P                  V4      pVf   R# \        V4      '       * # )z
Returns `True`/``False` only if the value is set, always `False` otherwise. So use this method to ask the very
specific question of whether the value is set to `False` (and it's not set to `True`` or isn't set).
Frz   r|   s   && r,   is_falseHfDeepSpeedConfig.is_false   s'    
 {+u:tE{?:r?   c                     V P                   ^8H  # )   r]   rR   s   &r,   r_   HfDeepSpeedConfig.is_zero2       {{ar?   c                     V P                   ^8H  # )   r   r   s   &r,   r`   HfDeepSpeedConfig.is_zero3   r   r?   c                    V P                   # r0   )r^   r   s   &r,   
is_offloadHfDeepSpeedConfig.is_offload   s    }}r?   )r^   r]   rP   r0   )F)__name__
__module____qualname____firstlineno____doc__rV   rQ   ro   r\   rw   r~   r   r_   r`   r   __static_attributes____classdictcell____classdict__s   @r,   rA   rA   w   sA      %4%&+$.7;   r?   rA   c                   :   a  ] tR t^t o RtR tRR ltR tRtV t	R# )DeepSpeedEngineWrapperz
Internal wrapper for deepspeed.runtime.engine.DeepSpeedEngine. This is used to follow conventional training loop.

Args:
    engine (deepspeed.runtime.engine.DeepSpeedEngine): deepspeed engine to wrap
c                    Wn         R # r0   engine)rR   r   s   &&r,   rV   DeepSpeedEngineWrapper.__init__  s    r?   c                    V P                   P                  VR 7       V P                   P                  ! V3/ VB  V'       d   V P                   P                  4        R# R# ))is_boundaryN)r   "set_gradient_accumulation_boundarybackwardstep)rR   losssync_gradientskwargss   &&&,r,   r   DeepSpeedEngineWrapper.backward  sN     	66>6R 	T,V,  KK r?   c                ~    V P                   P                  4       p\        VR4      '       d   VP                  4       # V# )z3Get the global gradient norm from DeepSpeed engine.item)r   get_global_grad_normhasattrr   )rR   	grad_norms   & r,   r   +DeepSpeedEngineWrapper.get_global_grad_norm  s4    KK446	9f%%>>##r?   r   N)T)
r   r   r   r   r   rV   r   r   r   r   r   s   @r,   r   r      s     , r?   r   c                   V   a a ] tR tRt oRtV 3R ltRR ltR t]R 4       t	Rt
VtV ;t# )	DeepSpeedOptimizerWrapperi'  z
Internal wrapper around a deepspeed optimizer.

Args:
    optimizer (`torch.optim.optimizer.Optimizer`):
        The optimizer to wrap.
c                b   < \         SV `  VR RR7       \        V P                  R4      V n        R# )FN)device_placementscaleroverflow)superrV   r   r$   __has_overflow__)rR   r$   	__class__s   &&r,   rV   "DeepSpeedOptimizerWrapper.__init__0  s*    U4H '
 Cr?   c                    R # r0    )rR   set_to_nones   &&r,   	zero_grad#DeepSpeedOptimizerWrapper.zero_grad4      r?   c                    R # r0   r   r   s   &r,   r   DeepSpeedOptimizerWrapper.step7  r   r?   c                V    V P                   '       d   V P                  P                  # R# )zTWhether or not the optimizer step was done, or skipped because of gradient overflow.F)r   r$   r   r   s   &r,   step_was_skipped*DeepSpeedOptimizerWrapper.step_was_skipped:  s#        >>***r?   )r   r0   )r   r   r   r   r   rV   r   r   propertyr   r   r   __classcell__r   r   s   @@r,   r   r   '  s3     D   r?   r   c                   <   a a ] tR tRt oRtV 3R ltR tRtVtV ;t	# )DeepSpeedSchedulerWrapperiB  z
Internal wrapper around a deepspeed scheduler.

Args:
    scheduler (`torch.optim.lr_scheduler.LambdaLR`):
        The scheduler to wrap.
    optimizers (one or a list of `torch.optim.Optimizer`):
c                &   < \         SV `  W4       R # r0   )r   rV   )rR   	scheduler
optimizersr   s   &&&r,   rV   "DeepSpeedSchedulerWrapper.__init__L  s    /r?   c                    R # r0   r   r   s   &r,   r   DeepSpeedSchedulerWrapper.stepO  r   r?   r   )
r   r   r   r   r   rV   r   r   r   r   r   s   @@r,   r   r   B  s     0 r?   r   c                   .   a  ] tR tRt o RtRR ltRtV tR# )
DummyOptimiS  a  
Dummy optimizer presents model parameters or param groups, this is primarily used to follow conventional training
loop when optimizer config is specified in the deepspeed config file.

Args:
    lr (float):
        Learning rate.
    params (iterable): iterable of parameters to optimize or dicts defining
        parameter groups
    weight_decay (float):
        Weight decay.
    **kwargs (additional keyword arguments, *optional*):
        Other arguments.
c                6    Wn         W n        W0n        W@n        R # r0   )paramsr   r   r   )rR   r   r   r   r   s   &&&&,r,   rV   DummyOptim.__init__c  s    (r?   )r   r   r   r   N)gMbP?r   r   r   r   r   r   rV   r   r   r   s   @r,   r   r   S  s      r?   r   c                   .   a  ] tR tRt o RtRR ltRtV tR# )DummySchedulerij  a  
Dummy scheduler presents model parameters or param groups, this is primarily used to follow conventional training
loop when scheduler config is specified in the deepspeed config file.

Args:
    optimizer (`torch.optim.optimizer.Optimizer`):
        The optimizer to wrap.
    total_num_steps (int, *optional*):
        Total number of steps.
    warmup_num_steps (int, *optional*):
        Number of steps for warmup.
    lr_scheduler_callable (callable, *optional*):
        A callable function that creates an LR Scheduler. It accepts only one argument `optimizer`.
    **kwargs (additional keyword arguments, *optional*):
        Other arguments.
Nc                B    Wn         W n        W0n        W@n        WPn        R # r0   )r$   total_num_stepswarmup_num_stepslr_scheduler_callabler   )rR   r$   r   r   r   r   s   &&&&&,r,   rV   DummyScheduler.__init__|  s    ". 0%:"r?   )r   r   r$   r   r   )Nr   Nr   r   s   @r,   r   r   j  s     " r?   r   )rL   rH   rD   copyr   torchr   r$   r   r   r   dataclassesr   importsr   versionsr	   r-   r>   rA   r   r   r   r   r   r   r?   r,   <module>r      sx      	   , , ( % &D?NZ&C CL' 'T 4 6 4 " . r?   