






'''simple docstring'''




import flax.linen as nn
import jax.numpy as jnp

from .attention_flax import FlaxTransformeraDModel
from .resnet_flax import FlaxDownsampleaD, FlaxResnetBlockaD, FlaxUpsampleaD







class    A_	(nn.Module							):
							"""simple docstring"""




							a__   		=	42
							a__   		=	42
							a__   		=	0.0
							a__   		=	1
							a__   		=	1
							a__   		=	True
							a__   		=	False
							a__   		=	False
							a__   		=	False
							a__   		=	jnp.floataa

							def 							_A	(		self			:Any				)				->  int:


										'''simple docstring'''




										snake_case_				:					Dict   =       []
										snake_case_				:					Union[str, Any]   =       []

										for i in range(self.num_layers				):
													snake_case_				:					Tuple   =       self.in_channels if i == 0 else self.out_channels

													snake_case_				:					Any   =       FlaxResnetBlockaD(
													    in_channels=lowerCAmelCase__       ,							out_channels=self.out_channels       ,							dropout_prob=self.dropout       ,							dtype=self.dtype       ,							)
													resnets.append(lowerCAmelCase__				)

													snake_case_				:					List[str]   =       FlaxTransformeraDModel(
													    in_channels=self.out_channels       ,							n_heads=self.num_attention_heads       ,							d_head=self.out_channels // self.num_attention_heads       ,							depth=1       ,							use_linear_projection=self.use_linear_projection       ,							only_cross_attention=self.only_cross_attention       ,							use_memory_efficient_attention=self.use_memory_efficient_attention       ,							dtype=self.dtype       ,							)
													attentions.append(lowerCAmelCase__				)

										snake_case_				:					Dict   =       resnets
										snake_case_				:					Union[str, Any]   =       attentions

										if self.add_downsample:
													snake_case_				:					Optional[Any]   =       FlaxDownsampleaD(self.out_channels       ,							dtype=self.dtype				)






							def __call__(		self			:Optional[int]       ,							lowerCAmelCase__			:str       ,							lowerCAmelCase__			:Any       ,							lowerCAmelCase__			:List[Any]       ,							lowerCAmelCase__			:List[Any]=True				)				->  List[Any]:


										'''simple docstring'''




										snake_case_				:					int   =       ()

										for resnet, attn in zip(self.resnets       ,							self.attentions				):
													snake_case_				:					List[Any]   =       resnet(lowerCAmelCase__       ,							lowerCAmelCase__       ,							deterministic=lowerCAmelCase__				)
													snake_case_				:					Dict   =       attn(lowerCAmelCase__       ,							lowerCAmelCase__       ,							deterministic=lowerCAmelCase__				)
													output_states += (hidden_states,)

										if self.add_downsample:
													snake_case_				:					Optional[Any]   =       self.downsamplers_a(lowerCAmelCase__				)
													output_states += (hidden_states,)

										return hidden_states, output_states







class    A_	(nn.Module							):
							"""simple docstring"""




							a__   		=	42
							a__   		=	42
							a__   		=	0.0
							a__   		=	1
							a__   		=	True
							a__   		=	jnp.floataa

							def 							_A	(		self			:List[str]				)				->  List[Any]:


										'''simple docstring'''




										snake_case_				:					int   =       []

										for i in range(self.num_layers				):
													snake_case_				:					Any   =       self.in_channels if i == 0 else self.out_channels

													snake_case_				:					int   =       FlaxResnetBlockaD(
													    in_channels=lowerCAmelCase__       ,							out_channels=self.out_channels       ,							dropout_prob=self.dropout       ,							dtype=self.dtype       ,							)
													resnets.append(lowerCAmelCase__				)
										snake_case_				:					Union[str, Any]   =       resnets

										if self.add_downsample:
													snake_case_				:					Tuple   =       FlaxDownsampleaD(self.out_channels       ,							dtype=self.dtype				)






							def __call__(		self			:Dict       ,							lowerCAmelCase__			:Dict       ,							lowerCAmelCase__			:Any       ,							lowerCAmelCase__			:Union[str, Any]=True				)				->  List[Any]:


										'''simple docstring'''




										snake_case_				:					Dict   =       ()

										for resnet in self.resnets:
													snake_case_				:					Any   =       resnet(lowerCAmelCase__       ,							lowerCAmelCase__       ,							deterministic=lowerCAmelCase__				)
													output_states += (hidden_states,)

										if self.add_downsample:
													snake_case_				:					Union[str, Any]   =       self.downsamplers_a(lowerCAmelCase__				)
													output_states += (hidden_states,)

										return hidden_states, output_states







class    A_	(nn.Module							):
							"""simple docstring"""




							a__   		=	42
							a__   		=	42
							a__   		=	42
							a__   		=	0.0
							a__   		=	1
							a__   		=	1
							a__   		=	True
							a__   		=	False
							a__   		=	False
							a__   		=	False
							a__   		=	jnp.floataa

							def 							_A	(		self			:Tuple				)				->  str:


										'''simple docstring'''




										snake_case_				:					Dict   =       []
										snake_case_				:					List[Any]   =       []

										for i in range(self.num_layers				):
													snake_case_				:					List[str]   =       self.in_channels if (i == self.num_layers - 1) else self.out_channels
													snake_case_				:					Optional[int]   =       self.prev_output_channel if i == 0 else self.out_channels

													snake_case_				:					int   =       FlaxResnetBlockaD(
													    in_channels=resnet_in_channels + res_skip_channels       ,							out_channels=self.out_channels       ,							dropout_prob=self.dropout       ,							dtype=self.dtype       ,							)
													resnets.append(lowerCAmelCase__				)

													snake_case_				:					Optional[int]   =       FlaxTransformeraDModel(
													    in_channels=self.out_channels       ,							n_heads=self.num_attention_heads       ,							d_head=self.out_channels // self.num_attention_heads       ,							depth=1       ,							use_linear_projection=self.use_linear_projection       ,							only_cross_attention=self.only_cross_attention       ,							use_memory_efficient_attention=self.use_memory_efficient_attention       ,							dtype=self.dtype       ,							)
													attentions.append(lowerCAmelCase__				)

										snake_case_				:					Optional[Any]   =       resnets
										snake_case_				:					List[str]   =       attentions

										if self.add_upsample:
													snake_case_				:					Optional[Any]   =       FlaxUpsampleaD(self.out_channels       ,							dtype=self.dtype				)






							def __call__(		self			:List[str]       ,							lowerCAmelCase__			:List[str]       ,							lowerCAmelCase__			:str       ,							lowerCAmelCase__			:int       ,							lowerCAmelCase__			:int       ,							lowerCAmelCase__			:Tuple=True				)				->  int:


										'''simple docstring'''




										for resnet, attn in zip(self.resnets       ,							self.attentions				):
													# pop res hidden states
													snake_case_				:					Any   =       res_hidden_states_tuple[-1]
													snake_case_				:					str   =       res_hidden_states_tuple[:-1]
													snake_case_				:					List[Any]   =       jnp.concatenate((hidden_states, res_hidden_states)       ,							axis=-1				)

													snake_case_				:					Union[str, Any]   =       resnet(lowerCAmelCase__       ,							lowerCAmelCase__       ,							deterministic=lowerCAmelCase__				)
													snake_case_				:					Optional[int]   =       attn(lowerCAmelCase__       ,							lowerCAmelCase__       ,							deterministic=lowerCAmelCase__				)

										if self.add_upsample:
													snake_case_				:					List[Any]   =       self.upsamplers_a(lowerCAmelCase__				)

										return hidden_states







class    A_	(nn.Module							):
							"""simple docstring"""




							a__   		=	42
							a__   		=	42
							a__   		=	42
							a__   		=	0.0
							a__   		=	1
							a__   		=	True
							a__   		=	jnp.floataa

							def 							_A	(		self			:List[Any]				)				->  Optional[int]:


										'''simple docstring'''




										snake_case_				:					int   =       []

										for i in range(self.num_layers				):
													snake_case_				:					List[Any]   =       self.in_channels if (i == self.num_layers - 1) else self.out_channels
													snake_case_				:					List[str]   =       self.prev_output_channel if i == 0 else self.out_channels

													snake_case_				:					Any   =       FlaxResnetBlockaD(
													    in_channels=resnet_in_channels + res_skip_channels       ,							out_channels=self.out_channels       ,							dropout_prob=self.dropout       ,							dtype=self.dtype       ,							)
													resnets.append(lowerCAmelCase__				)

										snake_case_				:					List[str]   =       resnets

										if self.add_upsample:
													snake_case_				:					Optional[Any]   =       FlaxUpsampleaD(self.out_channels       ,							dtype=self.dtype				)






							def __call__(		self			:List[Any]       ,							lowerCAmelCase__			:List[Any]       ,							lowerCAmelCase__			:Optional[int]       ,							lowerCAmelCase__			:Union[str, Any]       ,							lowerCAmelCase__			:int=True				)				->  Tuple:


										'''simple docstring'''




										for resnet in self.resnets:
													# pop res hidden states
													snake_case_				:					Optional[Any]   =       res_hidden_states_tuple[-1]
													snake_case_				:					int   =       res_hidden_states_tuple[:-1]
													snake_case_				:					Optional[int]   =       jnp.concatenate((hidden_states, res_hidden_states)       ,							axis=-1				)

													snake_case_				:					int   =       resnet(lowerCAmelCase__       ,							lowerCAmelCase__       ,							deterministic=lowerCAmelCase__				)

										if self.add_upsample:
													snake_case_				:					List[Any]   =       self.upsamplers_a(lowerCAmelCase__				)

										return hidden_states







class    A_	(nn.Module							):
							"""simple docstring"""




							a__   		=	42
							a__   		=	0.0
							a__   		=	1
							a__   		=	1
							a__   		=	False
							a__   		=	False
							a__   		=	jnp.floataa

							def 							_A	(		self			:List[Any]				)				->  Any:


										'''simple docstring'''




										snake_case_				:					Union[str, Any]   =       [
										    FlaxResnetBlockaD(
										        in_channels=self.in_channels       ,							out_channels=self.in_channels       ,							dropout_prob=self.dropout       ,							dtype=self.dtype       ,							)
										]

										snake_case_				:					str   =       []

										for _ in range(self.num_layers				):
													snake_case_				:					Union[str, Any]   =       FlaxTransformeraDModel(
													    in_channels=self.in_channels       ,							n_heads=self.num_attention_heads       ,							d_head=self.in_channels // self.num_attention_heads       ,							depth=1       ,							use_linear_projection=self.use_linear_projection       ,							use_memory_efficient_attention=self.use_memory_efficient_attention       ,							dtype=self.dtype       ,							)
													attentions.append(lowerCAmelCase__				)

													snake_case_				:					Tuple   =       FlaxResnetBlockaD(
													    in_channels=self.in_channels       ,							out_channels=self.in_channels       ,							dropout_prob=self.dropout       ,							dtype=self.dtype       ,							)
													resnets.append(lowerCAmelCase__				)

										snake_case_				:					Union[str, Any]   =       resnets
										snake_case_				:					List[str]   =       attentions






							def __call__(		self			:Union[str, Any]       ,							lowerCAmelCase__			:Any       ,							lowerCAmelCase__			:Union[str, Any]       ,							lowerCAmelCase__			:Any       ,							lowerCAmelCase__			:int=True				)				->  int:


										'''simple docstring'''




										snake_case_				:					List[str]   =       self.resnets[0](lowerCAmelCase__       ,							lowerCAmelCase__				)
										for attn, resnet in zip(self.attentions       ,							self.resnets[1:]				):
													snake_case_				:					Optional[Any]   =       attn(lowerCAmelCase__       ,							lowerCAmelCase__       ,							deterministic=lowerCAmelCase__				)
													snake_case_				:					Optional[Any]   =       resnet(lowerCAmelCase__       ,							lowerCAmelCase__       ,							deterministic=lowerCAmelCase__				)

										return hidden_states












'''simple docstring'''




import os
import sys
import tempfile

import torch

from .state import AcceleratorState
from .utils import PrecisionType, PrepareForLaunch, is_mps_available, patch_environment

def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__=()     ,__magic_name__=None     ,__magic_name__="no"     ,__magic_name__="29500"						)->    Optional[int]:



			"""simple docstring"""



			snake_case_				:					str   =       False
			snake_case_				:					int   =       False
			if any(key.startswith("KAGGLE"						) for key in os.environ.keys()						):
						snake_case_				:					Any   =       True
			elif "IPython" in sys.modules:
						snake_case_				:					Union[str, Any]   =       "google.colab" in str(sys.modules["IPython"].get_ipython()						)

			try:
						snake_case_				:					Any   =       PrecisionType(mixed_precision.lower()						)
			except ValueError:
						raise ValueError(
						    F'''Unknown mixed_precision mode: {args.mixed_precision.lower()}. Choose between {PrecisionType.list()}.'''						)

			if (in_colab or in_kaggle) and (os.environ.get("TPU_NAME"     ,__magic_name__						) is not None):
						# TPU launch
						import torch_xla.distributed.xla_multiprocessing as xmp

						if len(AcceleratorState._shared_state						) > 0:
									raise ValueError(
									    "To train on TPU in Colab or Kaggle Kernel, the `Accelerator` should only be initialized inside "
									    "your training function. Restart your notebook and make sure no cells initializes an "
									    "`Accelerator`."						)
						if num_processes is None:
									snake_case_				:					Tuple   =       8

						snake_case_				:					Optional[int]   =       PrepareForLaunch(__magic_name__     ,distributed_type="TPU"						)
						print(F'''Launching a training on {num_processes} TPU cores.'''						)
						xmp.spawn(__magic_name__     ,args=__magic_name__     ,nprocs=__magic_name__     ,start_method="fork"						)
			elif in_colab:
						# No need for a distributed launch otherwise as it's either CPU or one GPU.
						if torch.cuda.is_available():
									print("Launching training on one GPU."						)
						else:
									print("Launching training on one CPU."						)
						function(*__magic_name__						)
			else:
						if num_processes is None:
									raise ValueError(
									    "You have to specify the number of GPUs you would like to use, add `num_processes=...` to your call."						)

						if num_processes > 1:
									# Multi-GPU launch
									from torch.multiprocessing import start_processes
									from torch.multiprocessing.spawn import ProcessRaisedException

									if len(AcceleratorState._shared_state						) > 0:
												raise ValueError(
												    "To launch a multi-GPU training from your notebook, the `Accelerator` should only be initialized "
												    "inside your training function. Restart your notebook and make sure no cells initializes an "
												    "`Accelerator`."						)

									if torch.cuda.is_initialized():
												raise ValueError(
												    "To launch a multi-GPU training from your notebook, you need to avoid running any instruction "
												    "using `torch.cuda` in any cell. Restart your notebook and make sure no cells use any CUDA "
												    "function."						)

									# torch.distributed will expect a few environment variable to be here. We set the ones common to each
									# process here (the other ones will be set be the launcher).
									with patch_environment(
									    world_size=__magic_name__     ,master_addr="127.0.01"     ,master_port=__magic_name__     ,mixed_precision=__magic_name__						):
												snake_case_				:					Optional[int]   =       PrepareForLaunch(__magic_name__     ,distributed_type="MULTI_GPU"						)
												print(F'''Launching training on {num_processes} GPUs.'''						)
												try:
															start_processes(__magic_name__     ,args=__magic_name__     ,nprocs=__magic_name__     ,start_method="fork"						)
												except ProcessRaisedException as e:
															if "Cannot re-initialize CUDA in forked subprocess" in e.args[0]:
																		raise RuntimeError(
																		    "CUDA has been initialized before the `notebook_launcher` could create a forked subprocess. "
																		    "This likely stems from an outside import causing issues once the `notebook_launcher()` is called. "
																		    "Please review your imports and test them when running the `notebook_launcher()` to identify "
																		    "which one is problematic."						) from e

						else:
									# No need for a distributed launch otherwise as it's either CPU, GPU or MPS.
									if is_mps_available():
												snake_case_				:					Any   =       "1"
												print("Launching training on MPS."						)
									elif torch.cuda.is_available():
												print("Launching training on one GPU."						)
									else:
												print("Launching training on CPU."						)
									function(*__magic_name__						)






def 	__UpperCAmelCase       ( __magic_name__     ,__magic_name__=()     ,__magic_name__=2						)->    Dict:



			"""simple docstring"""



			from torch.multiprocessing import start_processes

			with tempfile.NamedTemporaryFile() as tmp_file:
						# torch.distributed will expect a few environment variable to be here. We set the ones common to each
						# process here (the other ones will be set be the launcher).
						with patch_environment(
						    world_size=__magic_name__     ,master_addr="127.0.01"     ,master_port="29500"     ,accelerate_mixed_precision="no"     ,accelerate_debug_rdv_file=tmp_file.name     ,accelerate_use_cpu="yes"     ,):
									snake_case_				:					Any   =       PrepareForLaunch(__magic_name__     ,debug=__magic_name__						)
									start_processes(__magic_name__     ,args=__magic_name__     ,nprocs=__magic_name__     ,start_method="fork"						)




