+
    $Lj6                     r   ^ RI t ^ RIt^ RIt^ RIt^ RIt^ RIHt ^ RIHt ^ RI	H
t
 ^ RIt^RIHt ^RIHtHtHtHtHtHtHtHtHtHtHt ^RIHt ^RIHtHtHt ^R	I H!t! ^R
I"H#t# ^RI$H%t%H&t& . 3R lt'R t(R R lt)R R lt*R R lt+R R lt,R R lt-R!R R llt.R R lt/R R lt0R t1 ! R R 4      t2R# )"    N)literal_eval)which)Any)SageMakerConfig)DynamoBackendPrecisionTypeis_fp8_availableis_hpu_availableis_mlu_availableis_musa_availableis_neuron_availableis_npu_availableis_sdaa_availableis_torch_xla_availableis_xpu_available)DEEPSPEED_MULTINODE_LAUNCHERS)get_free_portis_port_in_usemerge_dicts)compare_versions)parse_flag_from_env)DistributedTypeSageMakerDistributedTypec                    VP                  V4      w  r4\        V 4      P                  4        F1  w  rVV\        V4      P                  4       9   g   K%  \	        W5V4       K3  	  V# )z,
Filters out all `accelerate` specific args
)parse_known_argsvarsitemskeyssetattr)argsparserdefault_argsnew_args_keyvalues   &&&    C/app/.local/lib/python3.14/site-packages/accelerate/utils/launch.py_filter_argsr(   /   sU     )),7KH4j&&(
$x.%%''H5) ) O    c                     R U u. uF  p \        V 4      '       g   K  V NK  	  pp \        V4      ^ 8X  d   \        R4      hV^ ,          p\        P                  ! VR.4      pRV9   d   VRRRR3# VRRR	R
3# u up i )z
Determines the executable and argument names for mpirun, based on the type of install. The supported MPI programs
are: OpenMPI, Intel MPI, or MVAPICH.

Returns: Program name and arg names for hostfile, num processes, and processes per node
z\mpirun or mpiexec were not found. Ensure that Intel MPI, Open MPI, or MVAPICH are installed.z	--versions   Open MPI
--hostfilez-nz
--npernodez	--bind-toz-fz-ppn )mpirunmpiexec)r   lenOSError
subprocesscheck_output)xmpi_appsmpi_appmpirun_versions       r'   _get_mpirun_argsr7   :   s     1=0aE!H0H=
8}tuu qkG,,g{-CDNn$dL+EE dFB.. >s
   A6A6c                f    V ^8  d   QhR\         P                  R\        \        \        3,          /# )   r    current_envargparse	Namespacedictstr)formats   "r'   __annotate__rA   R   s(      ** c3h r)   c                   Rp\        V 4       F  pVP                  R4      '       g   K  \        W4      pVf   K-  VR8X  dT   \        V^ ,          4      WR,           &   \        V^,          4      WR,           &   \        V^,          4      WR,           &   K  \        \        W4      4      W VP	                  4        2&   K  	  V# )z&
Setup the FP8 environment variables.
ACCELERATE_fp8_fp8_override_linear_precisionFP8_OVERRIDE_FPROPFP8_OVERRIDE_DGRADFP8_OVERRIDE_WGRAD)r   
startswithgetattrr?   upper)r    r:   prefixargr&   s   &&   r'   setup_fp8_envrN   R   s     FDz>>&!!D&E 99ADU1XK)= =>ADU1XK)= =>ADU1XK)= =><?@R<SK(399;- 89  r)   c                    V ^8  d   QhR\         P                  R\        \        \        ,          \
        \        \        3,          3,          /# r9   r    returnr<   r=   tuplelistr?   r>   )r@   s   "r'   rA   rA   d   s?     b b(*<*< btCyRVWZ\_W_R`G`Aa br)   c                
   . pV P                   '       d   V P                  '       d   \        R4      h\        V RR4      pV P                  pV P
                  e   \        4       w  rErgp\        V RR4      p	V'       d   V'       d   \        W#,          4      MRp
VVVV P
                  VV
.,          pV'       d   W\        V4      .,          pV'       d   WV	.,          pV P                   '       gC   VP                  \        P                  4       V P                  '       d   VP                  R4       VP                  V P                  4       VP                  V P                  4       \        P                  P!                  4       p\        V P"                  ;'       g    V P$                  4      VR&   V P&                  '       d   R	VR
&   V P(                  R8w  d   V P(                  e   \+        4       '       d   V P(                  VR&   M\-        4       '       d   V P(                  VR&   M\/        4       '       d   V P(                  VR&   M\1        4       '       d   V P(                  VR&   Mo\3        4       '       d   V P(                  VR&   MO\5        4       '       d   V P(                  VR&   M/\7        4       '       d   V P(                  VR&   MV P(                  VR&   V^8  d+   V P8                  f   Q R4       hV P:                  f   Q R4       hVe   V^8  g   V^8  dH   V P8                  e   V P8                  MRVR&   V P:                  e   \        V P:                  4      MRVR&   \=        VR,          R4      '       d   RVR&   \        ^4      VR&    \?        V P@                  PC                  4       4      p\        T4      TR!&   T P@                  PC                  4       R"8X  d'   \G        4       '       g   \I        R#4      h\K        Y4      p \M        T PN                  PQ                  4       4      pTPR                  TR%&   T PT                  TR&&   \        T PV                  4      TR'&   \        T PX                  4      TR(&   \        T PZ                  4      TR)&   \        T P\                  4      TR*&   T P^                  '       d   RTR+&   Y3#   \         d?    \        RT P@                  PC                  4        R\>        PD                  ! 4        R 24      hi ; i  \         d?    \        R$T PN                  PQ                  4        R\L        PD                  ! 4        R 24      hi ; i),zr
Prepares and returns the command list and an environment with the correct simple launcher environment variables.
0--module and --no_python cannot be used togethernum_processesNzbind-tosocket1z-mACCELERATE_USE_CPUtrueACCELERATE_DEBUG_MODEallZE_AFFINITY_MASKMLU_VISIBLE_DEVICESSDAA_VISIBLE_DEVICESMUSA_VISIBLE_DEVICESASCEND_RT_VISIBLE_DEVICESHABANA_VISIBLE_MODULESNEURON_RT_VISIBLE_CORESCUDA_VISIBLE_DEVICESzFWhen using multiple machines, you need to specify the main process IP.zHWhen using multiple machines, you need to specify the main process port.z	127.0.0.1MASTER_ADDR29500MASTER_PORTFzgranularity=fine,compact,1,0KMP_AFFINITYKMP_BLOCKTIMEUnknown mixed_precision mode: . Choose between .ACCELERATE_MIXED_PRECISIONfp8rFP8 is not available on this machine. Please ensure that either Transformer Engine, MSAMP or torchao is installed.Unknown dynamo backend: ACCELERATE_DYNAMO_BACKENDACCELERATE_DYNAMO_MODEACCELERATE_DYNAMO_USE_FULLGRAPHACCELERATE_DYNAMO_USE_DYNAMIC*ACCELERATE_DYNAMO_USE_REGIONAL_COMPILATIONOMP_NUM_THREADSACCELERATE_CPU_AFFINITY)0	no_pythonmodule
ValueErrorrJ   num_machinesmpirun_hostfiler7   r?   appendsys
executabletraining_scriptextendtraining_script_argsosenvironcopycpuuse_cpudebuggpu_idsr   r   r   r   r   r
   r   main_process_ipmain_process_portr   r   mixed_precisionlowerrT   r	   RuntimeErrorrN   r   dynamo_backendrK   r&   dynamo_modedynamo_use_fullgraphdynamo_use_dynamicdynamo_use_regional_compilationnum_cpu_threads_per_processenable_cpu_affinity)r    cmdrW   r|   mpi_app_namehostfile_argnum_proc_argproc_per_node_argbind_to_argbind_tonproc_per_noder:   r   r   s   &             r'   prepare_simple_launcher_cmd_envr   d   s    C~~~$+++KLLD/48M$$L'ScSePL[$	84?LQ]]:;cf  
 	
 #m"455C))C>>>

3>>";;;JJtJJt##$JJt(()**//#K(+DHH,D,D(EK$%zzz/5+,||u!9.2llK*+15K-.  26,,K./  26,,K./7;||K3448LLK01 ""59\\K1226,,K./a##/ 	
T	
/ %%1 	
V	
1 	!ma&7L1<L=A=Q=Q=]T%9%9cnM"DHDZDZDfS)?)?%@lsM";';<eDD&DN#'*1vO$
'(<(<(B(B(DE 14O0DK,-!!#u,!! E  $D6
&t':':'@'@'BC
 0>/C/CK+,,0,<,<K()589R9R5SK1236t7N7N3OK/0@CDDhDh@iK<=%()I)I%JK!"14-.9  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

s   #R> #T
 >A	T
A	Uc                f    V ^8  d   QhR\         P                  R\        \        \        3,          /# rP   r;   )r@   s   "r'   rA   rA      s,     F F 2 2 FtCH~ Fr)   c                8   V P                   ^ 8X  d   \        4       V n         MV P                   f   RV n         V P                  pV P                  pV P                  pV P                   pV^8  d   \        W,          4      V n        \        V4      V n        \        V P                  4      V n
        \        V RR4      '       d"   \        V4      V n        \        V4      V n        M1V RV 2V n        M$\        V4      V n        Ve   \        V4      V n        V^8*  ;'       g    \        V P                  4      ^ 8H  pV'       dJ   \        V4      '       d9   V^8:  d#   RV n        \"        P$                  ! RV R24       M\'        R	V R
24      hV P(                  '       d   V P*                  '       d   \-        R4      hV P(                  '       d	   RV n        MV P*                  '       d   RV n        \.        P0                  P3                  4       pV P4                  '       d   RVR&   \        V RR4      pVR8w  d   V P6                  e   \9        4       '       d   WvR&   M\;        4       '       d   WvR&   Mm\=        4       '       d   WvR&   MX\?        4       '       d   WvR&   MC\A        4       '       d   WvR&   M.\C        4       '       d   WvR&   M\E        4       '       d   WvR&   MWvR&   V PF                  PI                  4       p \K        V4      p\        T4      TR&   T PF                  PI                  4       R8X  d'   \O        4       '       g   \Q        R4      h\S        Y4      p \U        T PV                  PY                  4       4      p	T	PZ                  TR&   T P\                  TR &   \        T P^                  4      TR!&   \        T P`                  4      TR"&   \        T Pb                  4      TR#&   T Pd                  '       EdZ   RTR$&   T Pf                  '       d   T Ph                  '       g   \-        R%4      h\k        T R&4      '       d   \        T Pl                  4      MR'TR(&   \        T Pn                  4      TR)&   \        T Pp                  4      PI                  4       TR*&   \        T Pr                  4      PI                  4       TR+&   \        T Pt                  4      TR,&   T Pv                  e   \        T Pv                  4      TR-&   T Px                  e   \        T Px                  4      TR.&   T Pz                  e   \        T Pz                  4      TR/&   T P|                  e   \        T P|                  4      TR0&   \        T P~                  4      PI                  4       TR1&   \        T P                  4      PI                  4       TR2&   \        T Pf                  4      PI                  4       TR3&   \        T Ph                  4      PI                  4       TR4&   \        T P                  4      PI                  4       TR5&   \        T R6R4      e   \        T P                  4      TR7&   T P                  '       Edt   R8p
RTR9&   \        T P                  4      YjR:,           &   \        T P                  4      YjR;,           &   T P                  e   \        T P                  4      YjR<,           &   T P                  e   \        T P                  4      YjR=,           &   T P                  e   \        T P                  4      YjR>,           &   T P                  e   \        T P                  4      YjR?,           &   T P                  e   \        T P                  4      YjR@,           &   T P                  e   \        T P                  4      YjRA,           &   T P                  e   \        T P                  4      YjRB,           &   \        T P                  4      YjRC,           &   \        T P                  4      YjRD,           &   T P                  e   \        T P                  4      YjRE,           &   T P                  e   \        T P                  4      YjRF,           &   T P                  e   \        T P                  4      YjRG,           &   T P                  e   \        T P                  4      YjRH,           &   T P                  e   \        T P                  4      YjRI,           &   T P                  e   \        T P                  4      YjRJ,           &   T P                  e   \        T P                  4      YjRK,           &   T P                  e   \        T P                  4      YjRL,           &   T P                  e   \        T P                  4      YjRM,           &   T P                  e   \        T P                  4      YjRN,           &   T P                  e   \        T P                  4      YjRO,           &   T P                  e   \        T P                  4      YjRP,           &   T P                  e   \        T P                  4      YjRQ,           &   T P                  e   \        T P                  4      YjRR,           &   T P                  e   \        T P                  4      YjRS,           &   T P                  e   \        T P                  4      YjRT,           &   \        T P                  4      TRU&   T P                  '       d   R'TRV&   T P                  '       d   \        Y4      pT#   \,         d'    \-        RT R\J        PL                  ! 4        R24      hi ; i  \,         d?    \-        RT PV                  PY                  4        R\T        PL                  ! 4        R24      hi ; i)WzW
Prepares and returns an environment with the correct multi-GPU environment variables.
N<s  same_networkF:TPort `  ` is already in use. Accelerate will attempt to launch in a standalone-like mode by finding an open port automatically for this session. If this current attempt fails, or for more control in future runs, please specify a different port (e.g., `--main_process_port <your_chosen_port>`) or use `--main_process_port 0` for automatic selection in your launch command or Accelerate config file.3Tried to launch distributed communication on port `$  `, but another process is utilizing it. Please specify a different port (such as using the `--main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.rV   r[   r\   r   r]   r^   r_   r`   ra   rb   rc   rd   re   rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   rv   ACCELERATE_USE_FSDPzWWhen using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`fsdp_versionrY   FSDP_VERSIONFSDP_SHARDING_STRATEGYFSDP_RESHARD_AFTER_FORWARDFSDP_OFFLOAD_PARAMSFSDP_MIN_NUM_PARAMSFSDP_AUTO_WRAP_POLICYFSDP_TRANSFORMER_CLS_TO_WRAPFSDP_BACKWARD_PREFETCHFSDP_STATE_DICT_TYPEFSDP_FORWARD_PREFETCHFSDP_USE_ORIG_PARAMSFSDP_CPU_RAM_EFFICIENT_LOADINGFSDP_SYNC_MODULE_STATESFSDP_ACTIVATION_CHECKPOINTINGfsdp_ignored_modulesFSDP_IGNORED_MODULESMEGATRON_LM_ACCELERATE_USE_MEGATRON_LM	TP_DEGREEUSE_CUSTOM_FSDPNO_LOAD_OPTIMEOD_MASK_LOSSNO_SAVE_OPTIMOPTIMIZER_CPU_OFFLOADUSE_PRECISION_AWARE_OPTIMIZEROVERLAP_CPU_OPTIMIZER_D2H_H2D DECODER_LAST_PIPELINE_NUM_LAYERS	PP_DEGREEGRADIENT_CLIPPINGNUM_MICRO_BATCHESSEQUENCE_PARALLELISMRECOMPUTE_ACTIVATIONSUSE_DISTRIBUTED_OPTIMIZERRECOMPUTE_GRANULARITYRECOMPUTE_METHODRECOMPUTE_NUM_LAYERSATTENTION_BACKENDEXPERT_MODEL_PARALLEL_SIZECONTEXT_PARALLEL_SIZEATTENTION_DROPOUTHIDDEN_DROPOUTATTENTION_SOFTMAX_IN_FP32EXPERT_TENSOR_PARALLEL_SIZECALCULATE_PER_TOKEN_LOSSUSE_ROTARY_POSITION_EMBEDDINGSrw   rx   )cr   r   rW   r|   r   r?   r   nnodesintmachine_rank	node_rankrJ   master_addrmaster_portrdzv_endpointr   
standalonewarningswarnConnectionErrorrz   ry   r{   r   r   r   r   r   r   r   r   r   r   r
   r   r   r   r   rT   r	   r   rN   r   r   rK   r&   r   r   r   r   use_fsdpfsdp_cpu_ram_efficient_loadingfsdp_sync_module_stateshasattrr   fsdp_sharding_strategyfsdp_reshard_after_forwardfsdp_offload_paramsfsdp_min_num_paramsfsdp_auto_wrap_policy"fsdp_transformer_layer_cls_to_wrapfsdp_backward_prefetchfsdp_state_dict_typefsdp_forward_prefetchfsdp_use_orig_paramsfsdp_activation_checkpointingr   use_megatron_lmmegatron_lm_tp_degreemegatron_lm_use_custom_fsdpmegatron_lm_no_load_optimmegatron_lm_eod_mask_lossmegatron_lm_no_save_optim!megatron_lm_optimizer_cpu_offload)megatron_lm_use_precision_aware_optimizer)megatron_lm_overlap_cpu_optimizer_d2h_h2d,megatron_lm_decoder_last_pipeline_num_layersmegatron_lm_pp_degreemegatron_lm_gradient_clippingmegatron_lm_num_micro_batches megatron_lm_sequence_parallelism!megatron_lm_recompute_activations%megatron_lm_use_distributed_optimizer!megatron_lm_recompute_granularitymegatron_lm_recompute_method megatron_lm_recompute_num_layersmegatron_lm_attention_backend&megatron_lm_expert_model_parallel_size!megatron_lm_context_parallel_sizemegatron_lm_attention_dropoutmegatron_lm_hidden_dropout%megatron_lm_attention_softmax_in_fp32'megatron_lm_expert_tensor_parallel_size$megatron_lm_calculate_per_token_loss*megatron_lm_use_rotary_position_embeddingsr   r   use_parallelism_config%prepare_extend_env_parallelism_config)r    rW   r|   r   r   need_port_checkr:   r   r   r   rL   s   &          r'   prepare_multi_gpu_envr     s9
   
 "!.				'!&&&M$$L**O..a!-"?@,'T../4//"?3D"#45D$3#4A6G5H!ID!-0("#45D #a'FF3t/@/@+AQ+FO>*;<<1"DOMM*+ ,D D "EFWEX Y   {{{t~~~KLL		**//#Kzzz/5+,dIu-G%DLL4.5*+18-.  29./  29./7>344;01 ""5<1229./**002Ou'8 14O0DK,-!!#u,!! E  $D6
&t':':'@'@'BC
 0>/C/CK+,,0,<,<K()589R9R5SK1236t7N7N3OK/0@CDDhDh@iK<=}}}-3)*...t7S7S7Svww@Gn@]@]c$*;*;&<cfN# 14D4O4O0P,-478W8W4X4^4^4`01-01I1I-J-P-P-R)*-01I1I-J)*%%136t7Q7Q3RK/022>:=d>e>e:fK67&&2478S8S4TK01$$025d6O6O2PK.//243M3M/N/T/T/V+,.1$2K2K.L.R.R.T*+8;D<_<_8`8f8f8h4514T5Q5Q1R1X1X1Z-.7:4;];]7^7d7d7f344/6B25d6O6O2PK./4:01,/0J0J,K[()25d6V6V2W../))5478V8V4WK01))5478V8V4WK01))5478V8V4WK0111=<?@f@f<gK!88999EDGHvHvDwK!@@A99EDGHvHvDwK!@@A<<HGJAAHK!CCD -00J0J,K[()478Z8Z4[001--98;D<^<^8_K!44500<;>t?d?d;eK!77811=<?@f@f<gK!88955A@CDDnDn@oK!<<=11=<?@f@f<gK!889,,87:4;\;\7]K!33400<;>t?d?d;eK!778--98;D<^<^8_K!44566BADTEpEpAqK!==>11=<?@f@f<gK!889--98;D<^<^8_K!445**6589X9X5YK!11255A@CDDnDn@oK!<<=77CBEdFrFrBsK!>>?44@?B4ClCl?mK!;;<::FEH??FK!AAB &))I)I%JK!"14-.""";DNm  u9/9JJ[\i\n\n\p[qqrsttu  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

s   4l #m 1mA	nc          
          V ^8  d   QhR\         P                  R\        R\        \        \
        ,          \        \
        \
        3,          3,          /# )r9   r    r:   rQ   )r<   r=   r>   rS   rT   r?   )r@   s   "r'   rA   rA     s@      


+/
49d38n$%r)   c                   RpRVR&   \        V P                  4      WR,           &   \        V P                  4      WR,           &   \        V P                  4      WR,           &   \        V P                  4      WR,           &   \        V P
                  4      WR,           &   \        V P                  4      WR	,           &   \        V P                  4      WR
,           &   V P                  ^8  d   \        V P                  4      WR,           &   V P                  ^8  d[   \        V P                  4      WR,           &   \        V P                  4      WR,           &   \        V P                  4      WR,           &   V# )zN
Extends `current_env` with context parallelism env vars if any have been set
PARALLELISM_CONFIG_r[   !ACCELERATE_USE_PARALLELISM_CONFIGDP_REPLICATE_SIZEDP_SHARD_SIZETP_SIZECP_SIZE
CP_BACKENDSP_SIZE
SP_BACKENDCP_COMM_STRATEGYSP_SEQ_LENGTHSP_SEQ_LENGTH_IS_VARIABLESP_ATTN_IMPLEMENTATION)r?   $parallelism_config_dp_replicate_size parallelism_config_dp_shard_sizeparallelism_config_tp_sizeparallelism_config_cp_sizeparallelism_config_cp_backendparallelism_config_sp_sizeparallelism_config_sp_backend#parallelism_config_cp_comm_strategy parallelism_config_sp_seq_length,parallelism_config_sp_seq_length_is_variable)parallelism_config_sp_attn_implementation)r    r:   rL   s   && r'   r  r    sB    #F7=K3403D4]4]0^K,,-,/0U0U,VK()&)$*I*I&JK"#&)$*I*I&JK"#),T-O-O)PK%&&)$*I*I&JK"#),T-O-O)PK%&&&*36t7_7_3`//0&&*03D4Y4Y0Z_,-<?@q@q<r8899<T=k=k9l556r)   c                    V ^8  d   QhR\         P                  R\        \        \        ,          \
        \        \        3,          3,          /# rP   rR   )r@   s   "r'   rA   rA     s=     a aH$6$6 a5cDQTVYQYNAZ;[ ar)   c                   V P                   ^ 8X  d   \        4       V n         MV P                   f   RV n         V P                  pV P                  pV P                  pV P                   pRpV P
                  f   \        ^ ,          V n        V^8  EdS   V P
                  \        ^,          8w  Ed6   R.pVP                  R\        V P                  4      .4       V P
                  R8X  dG   \        RRR4      '       d   \        R4      hVP                  R	\        V P                  4      R
.4       M'VP                  RR\        V P
                  4      .4       V P                  e(   VP                  R\        V P                  4      .4       MlV P                  e(   VP                  R\        V P                  4      .4       M7VP                  R\        V P                  V P                  ,          4      .4       V'       d   VP                  R\        V4      .4       VP                  R\        V4      .4       V P                  '       d   V P                   '       d   \        R4      hV P                  '       d   VP#                  R4       M#V P                   '       d   VP#                  R4       VP#                  V P$                  4       VP                  V P&                  4       MV^8  d   V P
                  \        ^,          8X  d   \        W,          4      V n        \        V4      V n        \-        V P                  4      V n        \1        V RR4      '       d"   \        V4      V n        \        V4      V n        M1V RV 2V n        M$\        V4      V n        Ve   \        V4      V n        V^8*  ;'       g    \-        V P                  4      ^ 8H  pV'       dJ   \9        V4      '       d9   V^8:  d#   RV n        \<        P>                  ! RV R24       M\A        RV R24      hV P                  '       d   V P                   '       d   \        R4      hV P                  '       d	   RV n        MV P                   '       d   RV n        \B        PD                  PG                  4       pV PH                  '       d   RVR&   \1        V RR 4      pVR 8w  d   V PJ                  e   \M        4       '       d   WR!&   M\O        4       '       d   WR"&   Mm\Q        4       '       d   WR#&   MX\S        4       '       d   WR$&   MC\U        4       '       d   WR%&   M.\W        4       '       d   WR&&   M\Y        4       '       d   WR'&   MWR(&    \[        V P\                  P_                  4       4      p	\c        R,\B        Pd                  Pg                  R+4      4      TR,&   \        T	4      TR-&   T P\                  P_                  4       R.8X  d'   \i        4       '       g   \k        R/4      h\m        Y4      p\        T Pn                  4      P_                  4       TR0&   RTR1&   T Pp                  e   \        T Pp                  4      TR2&   T Pr                  e   \        T Pr                  4      TR3&   T Pt                  e'   \        T Pt                  4      P_                  4       TR4&   T Pv                  e'   \        T Pv                  4      P_                  4       TR5&   T Px                  e'   \        T Px                  4      P_                  4       TR6&   T Pz                  e'   \        T Pz                  4      P_                  4       TR7&   T P|                  e'   \        T P|                  4      P_                  4       TR8&   T P~                  e   \        T P~                  4      TR9&   T P                  '       d   R:TR;&   T P                  e   \        T P                  4      TR<&   T P                  '       d   \        Y4      pYW3#   \         d?    \        R)T P\                  P_                  4        R*\Z        P`                  ! 4        R+24      hi ; i)=zl
Prepares and returns the command list and an environment with the correct DeepSpeed environment variables.
Nr   	deepspeedr+   nossh<z0.14.5z+nossh launcher requires DeepSpeed >= 0.14.5z--node_rankz--no_sshz--no_local_rankz
--launcherz	--excludez	--includez
--num_gpusz--master_addrz--master_portrV   z--modulez--no_pythonr   Fr   Tr   r   r   r   r[   r\   r   r]   r^   r_   r`   ra   rb   rc   rd   re   rk   rl   rm   
PYTHONPATHrn   ro   rp   ACCELERATE_CONFIG_DS_FIELDSACCELERATE_USE_DEEPSPEEDACCELERATE_DEEPSPEED_ZERO_STAGE&ACCELERATE_GRADIENT_ACCUMULATION_STEPSACCELERATE_GRADIENT_CLIPPING-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICEACCELERATE_DEEPSPEED_ZERO3_INIT+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL ACCELERATE_DEEPSPEED_CONFIG_FILErY   rx   (ACCELERATE_DEEPSPEED_MOE_LAYER_CLS_NAMES)Dr   r   rW   r|   r   deepspeed_multinode_launcherr   r   r?   deepspeed_hostfiler   r{   r   deepspeed_exclusion_filterdeepspeed_inclusion_filterrz   ry   r~   r   r   r   r   r   r   rJ   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r
   r   r   r   r   rT   env_var_path_addpathabspathr	   r   rN   'deepspeed_fields_from_accelerate_config
zero_stagegradient_accumulation_stepsgradient_clippingoffload_optimizer_deviceoffload_param_devicezero3_init_flagzero3_save_16bit_modeldeepspeed_config_filer   deepspeed_moe_layer_cls_namesr  r  )
r    rW   r|   r   r   r   r  r:   r   r   s
   &         r'   prepare_deepspeed_cmd_envrN    s   
 "!.				'!&&&M$$L**O..
C ((0,I!,L)aD==A^_`Aaam

L#d&=&=">?@,,7S(;; !NOOJJs4+<+<'=zJKJJ)<T=^=^9_`a**6JJ778 ,,8JJ778 JJc$*<*<@Q@Q*Q&RSTJJ_)=>?

OS):%;<=;;;4>>>OPP[[[JJz"^^^JJ}%

4''(

4,,-		d??C`abCcc!-"?@,'T../4//"?3D"#45D$3#4A6G5H!ID!-0("#45D #a'FF3t/@/@+AQ+FO>*;<<1"DOMM*+ ,D D "EFWEX Y   {{{t~~~KLL		**//#Kzzz/5+,dIu-G%DLL4.5*+18-.  29./  29./7>344;01 ""5<1229./
'(<(<(B(B(DE !1rwws?S TK03O0DK,-!!#u,!! E  $D614T5a5a1b1h1h1jK-..4K*+"9<T__9M56''3@CDDdDd@e<=)69$:P:P6Q6W6W6Y23$$0GJ4KhKhGiGoGoGqCD  ,CFtG`G`CaCgCgCi?@'9<T=Q=Q9R9X9X9Z56"".EHIdIdEeEkEkEmAB!!-:=d>X>X:Y6714-.))5BEdFhFhBi>?""";DNO  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

s   2#` A	ac                    V ^8  d   QhR\         P                  R\        \        \        3,          R\        R\
        \         P                  \        \        \        3,          3,          /# )r9   r    r:   podrQ   )r<   r=   r>   r?   boolrS   )r@   s   "r'   rA   rA   Q  sR      


+/S>@D
8tCH~-.r)   c                
   V P                   R8X  d0   \        RR7      '       d   V P                  '       d   RVR&   MRVR&   V P                  '       d   RVR&   V'       d#   V P                  V n        V P                  V n        W3# )	zQ
Prepares and returns an environment with the correct TPU environment variables.
bf16T)check_is_tpurY   XLA_DOWNCAST_BF16XLA_USE_BF16r[   r\   )r   r   downcast_bf16r   tpu_vmvmtpu_nametpu)r    r:   rP  s   &&&r'   prepare_tpur\  Q  st     v%*@d*S*S/2K+,*-K'zzz/5+,
++==r)   c                h    V ^8  d   QhR\         \        ,          R\        \        \        3,          /# )r9   nargsrQ   )rT   r?   r>   )r@   s   "r'   rA   rA   e  s&     ' '$s) 'S#X 'r)   c                 |   \        V 4      ^ 8  d   / # R p\        P                  ! 4       pVP                  V 4      w  r4\	        V4       F  w  rVVP                  R4      '       g   K  RpV^,           \        V4      8  d1   WE^,           ,          P                  R4      '       d   \        R4      hM\        R4      hVf   VP                  WaR7       K  VP                  WgR7       K  	  VP                  V 4      P                  P                  4        UU	u/ uF  w  rYR9   d   \        V	4      MT	bK  	  up	p# u up	pi )r   c                 v     \        V 4      p V ^,          V 8X  d   \        V 4      # V #   \         d    T u # i ; i)   )floatr   r{   )ss   &r'   _infer_type+_convert_nargs_to_dict.<locals>._infer_typej  s=    	aAAv{1vH 	H	s   #( ( 88Nul   SageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types)type)action)-z--)TrueFalse)r/   r<   ArgumentParserr   	enumeraterI   r{   add_argument
parse_args__dict__r   r   )
r^  rd  r!   r$   unknownindexargumentrg  r%   r&   s
   &         r'   _convert_nargs_to_dictrs  e  s3   
5zA~	 $$&F((/JA$W-{++Fqy3w<'19%00==$ G  > ! C  ~##H#?##H#<# .* !++E2;;AACCJC 	.?%?l5!UJC  s   D8c          
          V ^8  d   QhR\         R\        P                  R\        \        P                  \        \
        \        3,          3,          /# )r9   sagemaker_configr    rQ   )r   r<   r=   rS   r>   r?   r   )r@   s   "r'   rA   rA     sG     s" s"%s"-5-?-?s"
8tCH~-.s"r)   c                 
   \        R 4       V P                  \        P                  R&   V P                  e   V P                  \        P                  R&   MbVP
                  eJ   VP                  e<   VP
                  \        P                  R&   VP                  \        P                  R&   M\        R4      h\        P                  P                  VP                  4      pV'       g   Rp\        P                  P                  VP                  4      pVP                  R4      '       g   \        R	V R
24      h\        R4       \        VP                  4      p \!        VP"                  P%                  4       4      p \)        TP*                  P-                  4       4      pRRR\/        T4      RTP0                  RTP2                  R\/        TP4                  4      R\/        TP6                  4      R\/        TP8                  4      RT P:                  P0                  /pTP"                  P%                  4       R8X  d'   \=        4       '       g   \?        R4      h\A        Y4      pRpT P:                  \B        PD                  8X  d	   RRRR///pRp	T PF                  e   \        RT PF                   R24       / p	\I        T PF                  4      ;_uu_ 4       p
\K        T
4       F?  w  rT^ 8X  d   K  TPM                  R 4      pT^,          PO                  4       Y^ ,          &   KA  	  RRR4       \        R!T	 24       RpT PP                  e   \        R"T PP                   R24       . p\I        T PP                  4      ;_uu_ 4       p
\K        T
4       FR  w  rT^ 8X  d   K  TPM                  R 4      pR#T^ ,          R$T^,          PO                  4       /pTPS                  T4       KT  	  RRR4       \        R%T 24       \        R&4       R'T PT                  R(TR)TR*T PV                  R+T PX                  R,T PZ                  R-T P\                  R.T P^                  R/T P`                  R0T Pb                  R1R2R3TR4TR5TR6T/pT Pd                  e   \g        T Pd                  T4      pY3#   \         d?    \        RTP"                  P%                  4        R\         P&                  ! 4        R24      hi ; i  \         d?    \        RTP*                  P-                  4        R\(        P&                  ! 4        R24      hi ; i  + '       g   i     EL; i  + '       g   i     ELm; i)7z(Configuring Amazon SageMaker environmentAWS_DEFAULT_REGIONNAWS_PROFILEAWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profilerm   z.pyz8Your training script should be a python script and not ""z'Converting Arguments to Hyperparametersrk   rl   rq   ACCELERATE_USE_SAGEMAKERr[   rn   rr   rs   rt   ru   rv   %ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPEro   rp   smdistributeddataparallelenabledTzLoading SageMaker Inputs from z file	zLoaded SageMaker Inputs: zLoading SageMaker Metrics from NameRegexzLoaded SageMaker Metrics: zCreating Estimator	image_urientry_point
source_dirroletransformers_versionpytorch_version
py_versionbase_job_nameinstance_countinstance_typedebugger_hook_configFdistributionhyperparametersenvironmentmetric_definitions)4printregionr   r   profileaws_access_key_idaws_secret_access_keyr0   rB  dirnamer   basenameendswithr{   rs  r   r   r   r   rT   r   r   rK   r?   r&   r   r   r   r   distributed_typer	   r   rN   r   DATA_PARALLELsagemaker_inputs_fileopenrl  splitstripsagemaker_metrics_filer~   r  iam_role_namer  r  r  r  r|   ec2_instance_typeadditional_argsr   )ru  r    r  r  r  r   r   r  r  sagemaker_inputsfileilinelsagemaker_metricsmetric_dicts   &&              r'   prepare_sagemager_args_inputsr    s    

45'7'>'>BJJ#$ +$4$<$<

=!				+0J0J0V*.*@*@

&'.2.H.H

*+uvv !5!56J
''""4#7#78K&&ST_S``abcc	
34,T-F-FGO
'(<(<(B(B(DE
&t':':'@'@'BC 	#F$c/&:#^%9%9 $"2"2)3t/H/H+I'T-D-D)E4c$:^:^6_/1A1R1R1X1X	K !!#u,!! E  $D6L((,D,R,RR'.9d:K)LM --9./?/U/U.VV[\]"8899T$T?6JJt$)*1 1&	 + : 	)*:);<= ..:/0@0W0W/XX]^_"99::d$T?6JJt$AaDQqTZZ\ "((5 + ; 	*+<*=>? 

%//{j .. 0 E E+;;&11)77*77);;?{/D$ ''3+;;TB!!m  
,T-A-A-G-G-I,JJ[\i\n\n\p[qqrs
 	

  
&t':':'@'@'B&CCTUbUgUgUiTjjkl
 	

@ :99 ;::s4   #R >#S& AT2 A"UA	S#&A	T/2U	U	c                   \         P                  P                  V R4      P                  R4       Uu. uF  p\	        V4      ^ 8  g   K  VNK  	  ppVP                  \        V4      4       RP                  V4      # u upi )z
Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the
caller to set it in os.environ.
r,   r   )r   r   getr  r/   r~   r?   join)env_var_namepath_to_addppathss   &&  r'   rA  rA    sf    
 

|R8>>sCRC1s1vPQzQQCER	LL[!"88E? Ss   A<	A<c                   4   a  ] tR tRt o RtRR ltR tRtV tR# )PrepareForLaunchi  aE  
Prepare a function that will launched in a distributed setup.

Args:
    launcher (`Callable`):
        The function to launch.
    distributed_type ([`~state.DistributedType`]):
        The distributed type to prepare for.
    debug (`bool`, *optional*, defaults to `False`):
        Whether or not this is a debug launch.
c                >    Wn         \        V4      V n        W0n        R # )N)launcherr   r  r   )selfr  r  r   s   &&&&r'   __init__PrepareForLaunch.__init__  s      /0@ A
r)   c                   V P                   '       d   \        \        P                  P	                  R 4      4      p\        P                  P	                  R4      p\
        P                  P                  RV\
        P                  P                  WC4      VR7       EMV P                  \        P                  \        P                  \        P                  \        P                  \        P                  \        P                   \        P"                  39   d   \%        V4      \        P                  R&   \        \        P                  P	                  R^4      4      p\        \        P                  P	                  R^ 4      4      p\%        WV,          V,           4      \        P                  R&   \%        ^4      \        P                  R&   V P&                  ! V!   R	# )

WORLD_SIZEACCELERATE_DEBUG_RDV_FILEgloo)rankstore
world_size
LOCAL_RANKNPROC	NODE_RANKRANKFORK_LAUNCHEDN)r   r   r   r   r  torchdistributedinit_process_group	FileStorer  r   	MULTI_GPU	MULTI_MLU
MULTI_MUSA	MULTI_NPU	MULTI_XPU	MULTI_CPUMULTI_NEURONr?   r  )r  rq  r    r  rdv_filenprocr   s   &&*    r'   __call__PrepareForLaunch.__call__!  s>   :::RZZ^^L9:Jzz~~&ABH00''11(G%	 1  ""%%%%&&%%%%%%(('
 
 (+5zBJJ|$

w23EBJJNN;:;I!$U%6%>!?BJJv&)!f

?#tr)   )r   r  r  N)NOF)	__name__
__module____qualname____firstlineno____doc__r  r  __static_attributes____classdictcell__)__classdict__s   @r'   r  r    s     

 r)   r  )F)3r<   r   r1   r   r   astr   shutilr   typingr   r  commands.config.config_argsr   utilsr   r   r	   r
   r   r   r   r   r   r   r   utils.constantsr   utils.otherr   r   r   utils.versionsr   r,   r   dataclassesr   r   r(   r7   rN   r   r  r  rN  r\  rs  r  rA  r   r)   r'   <module>r     s     	  
      9    < D D - ! B -/ /0$bJFR6aH('Ts"l, ,r)   