



import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version(">=", FSDP_PYTORCH_VERSION):
       import torch.distributed.checkpoint as dist_cp
       from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
       from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
       from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
       from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


a__      :				int   	=			get_logger(__name__)

def   _lowerCAmelCase   (						A__						,  A__						,  A__						,  A__						,  A__=0	):
      os.makedirs(A__						,  exist_ok=A__	)
      with FSDP.state_dict_type(
          A__						,  fsdp_plugin.state_dict_type						,  fsdp_plugin.state_dict_config						,  fsdp_plugin.optim_state_dict_config	):
            lowercase__     =  model.state_dict()
            if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                  lowercase__     =  F'''{MODEL_NAME}.bin''' if model_index == 0 else F'''{MODEL_NAME}_{model_index}.bin'''
                  lowercase__     =  os.path.join(A__						,  A__	)
                  if accelerator.process_index == 0:
                        logger.info(F'''Saving model to {output_model_file}'''	)
                        torch.save(A__						,  A__	)
                        logger.info(F'''Model saved to {output_model_file}'''	)
            elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
                  lowercase__     =  (
                      F'''{MODEL_NAME}_rank{accelerator.process_index}.bin'''
                      if model_index == 0
                      else F'''{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'''
                  )
                  lowercase__     =  os.path.join(A__						,  A__	)
                  logger.info(F'''Saving model to {output_model_file}'''	)
                  torch.save(A__						,  A__	)
                  logger.info(F'''Model saved to {output_model_file}'''	)
            elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
                  lowercase__     =  os.path.join(A__						,  F'''{MODEL_NAME}_{model_index}'''	)
                  os.makedirs(A__						,  exist_ok=A__	)
                  logger.info(F'''Saving model to {ckpt_dir}'''	)
                  lowercase__     =  {'model': state_dict}

                  dist_cp.save_state_dict(
                      state_dict=A__						,  storage_writer=dist_cp.FileSystemWriter(A__	)						,  planner=DefaultSavePlanner()						,  )
                  logger.info(F'''Model saved to {ckpt_dir}'''	)

def   _lowerCAmelCase   (						A__						,  A__						,  A__						,  A__						,  A__=0	):
      accelerator.wait_for_everyone()
      with FSDP.state_dict_type(
          A__						,  fsdp_plugin.state_dict_type						,  fsdp_plugin.state_dict_config						,  fsdp_plugin.optim_state_dict_config	):
            if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                  if type(A__	) != FSDP and accelerator.process_index != 0:
                        if not fsdp_plugin.sync_module_states:
                              raise ValueError(
                                  'Set the `sync_module_states` flag to `True` so that model states are synced across processes when '
                                  'initializing FSDP object'	)
                        return
                  lowercase__     =  F'''{MODEL_NAME}.bin''' if model_index == 0 else F'''{MODEL_NAME}_{model_index}.bin'''
                  lowercase__     =  os.path.join(A__						,  A__	)
                  logger.info(F'''Loading model from {input_model_file}'''	)
                  lowercase__     =  torch.load(A__	)
                  logger.info(F'''Model loaded from {input_model_file}'''	)
            elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
                  lowercase__     =  (
                      F'''{MODEL_NAME}_rank{accelerator.process_index}.bin'''
                      if model_index == 0
                      else F'''{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'''
                  )
                  lowercase__     =  os.path.join(A__						,  A__	)
                  logger.info(F'''Loading model from {input_model_file}'''	)
                  lowercase__     =  torch.load(A__	)
                  logger.info(F'''Model loaded from {input_model_file}'''	)
            elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
                  lowercase__     =  (
                      os.path.join(A__						,  F'''{MODEL_NAME}_{model_index}'''	)
                      if F'''{MODEL_NAME}''' not in input_dir
                      else input_dir
                  )
                  logger.info(F'''Loading model from {ckpt_dir}'''	)
                  lowercase__     =  {'model': model.state_dict()}
                  dist_cp.load_state_dict(
                      state_dict=A__						,  storage_reader=dist_cp.FileSystemReader(A__	)						,  planner=DefaultLoadPlanner()						,  )
                  lowercase__     =  state_dict['model']
                  logger.info(F'''Model loaded from {ckpt_dir}'''	)
            model.load_state_dict(A__	)

def   _lowerCAmelCase   (						A__						,  A__						,  A__						,  A__						,  A__						,  A__=0	):
      os.makedirs(A__						,  exist_ok=A__	)
      with FSDP.state_dict_type(
          A__						,  fsdp_plugin.state_dict_type						,  fsdp_plugin.state_dict_config						,  fsdp_plugin.optim_state_dict_config	):
            lowercase__     =  FSDP.optim_state_dict(A__						,  A__	)
            if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                  if accelerator.process_index == 0:
                        lowercase__     =  (
                            F'''{OPTIMIZER_NAME}.bin''' if optimizer_index == 0 else F'''{OPTIMIZER_NAME}_{optimizer_index}.bin'''
                        )
                        lowercase__     =  os.path.join(A__						,  A__	)
                        logger.info(F'''Saving Optimizer state to {output_optimizer_file}'''	)
                        torch.save(A__						,  A__	)
                        logger.info(F'''Optimizer state saved in {output_optimizer_file}'''	)
            else:
                  lowercase__     =  os.path.join(A__						,  F'''{OPTIMIZER_NAME}_{optimizer_index}'''	)
                  os.makedirs(A__						,  exist_ok=A__	)
                  logger.info(F'''Saving Optimizer state to {ckpt_dir}'''	)
                  dist_cp.save_state_dict(
                      state_dict={'optimizer': optim_state}						,  storage_writer=dist_cp.FileSystemWriter(A__	)						,  planner=DefaultSavePlanner()						,  )
                  logger.info(F'''Optimizer state saved in {ckpt_dir}'''	)

def   _lowerCAmelCase   (						A__						,  A__						,  A__						,  A__						,  A__						,  A__=0	):
      accelerator.wait_for_everyone()
      with FSDP.state_dict_type(
          A__						,  fsdp_plugin.state_dict_type						,  fsdp_plugin.state_dict_config						,  fsdp_plugin.optim_state_dict_config	):
            if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
                  lowercase__     =  None
                  # below check should work but currently it isn't working (mostly opytorch issue),
                  # in the meantime disabling it at the cost of excess memory usage
                  # if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
                  lowercase__     =  (
                      F'''{OPTIMIZER_NAME}.bin''' if optimizer_index == 0 else F'''{OPTIMIZER_NAME}_{optimizer_index}.bin'''
                  )
                  lowercase__     =  os.path.join(A__						,  A__	)
                  logger.info(F'''Loading Optimizer state from {input_optimizer_file}'''	)
                  lowercase__     =  torch.load(A__	)
                  logger.info(F'''Optimizer state loaded from {input_optimizer_file}'''	)
            else:
                  lowercase__     =  (
                      os.path.join(A__						,  F'''{OPTIMIZER_NAME}_{optimizer_index}'''	)
                      if F'''{OPTIMIZER_NAME}''' not in input_dir
                      else input_dir
                  )
                  logger.info(F'''Loading Optimizer from {ckpt_dir}'''	)
                  lowercase__     =  load_sharded_optimizer_state_dict(
                      model_state_dict=model.state_dict()						,  optimizer_key='optimizer'						,  storage_reader=dist_cp.FileSystemReader(A__	)						,  )
                  lowercase__     =  optim_state['optimizer']
                  logger.info(F'''Optimizer loaded from {ckpt_dir}'''	)
            lowercase__     =  FSDP.optim_state_dict_to_load(A__						,  A__						,  A__	)
            optimizer.load_state_dict(A__	)





import os
from typing import List, Optional, Union

from ...tokenization_utils import PreTrainedTokenizer
from ...tokenization_utils_base import AddedToken
from ...utils import logging


a__      :				int   	=			logging.get_logger(__name__)

a__      :				Tuple   	=			{"vocab_file": "vocab.txt"}

a__      :				int   	=			{
    "vocab_file": {
        "facebook/esm2_t6_8M_UR50D": "https://huggingface.co/facebook/esm2_t6_8M_UR50D/resolve/main/vocab.txt",
        "facebook/esm2_t12_35M_UR50D": "https://huggingface.co/facebook/esm2_t12_35M_UR50D/resolve/main/vocab.txt",
    },
}

a__      :				Dict   	=			{
    "facebook/esm2_t6_8M_UR50D": 10_24,
    "facebook/esm2_t12_35M_UR50D": 10_24,
}

def   _lowerCAmelCase   (						A__	):
      with open(A__						,  'r'	) as f:
            lowercase__     =  f.read().splitlines()
            return [l.strip() for l in lines]



class       UpperCAmelCase__(							lowerCamelCase	):

   '''simple docstring'''




   A    :  Union[str, Any]       				=    VOCAB_FILES_NAMES
   A    :  str       				=    PRETRAINED_VOCAB_FILES_MAP
   A    :  int       				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A    :  List[Any]       				=    ["input_ids", "attention_mask"]


   def __init__(						self  :			Union[str, Any]			,				lowerCAmelCase  :			Any			,				lowerCAmelCase  :			Optional[int]="<unk>"			,				lowerCAmelCase  :			Dict="<cls>"			,				lowerCAmelCase  :			List[str]="<pad>"			,				lowerCAmelCase  :			Union[str, Any]="<mask>"			,				lowerCAmelCase  :			Optional[Any]="<eos>"			,				**lowerCAmelCase  :			Any			,				)	->				Optional[Any]:




         """simple docstring"""

         super().__init__(**lowerCAmelCase)
         lowercase__     =  load_vocab_file(lowerCAmelCase)
         lowercase__     =  dict(enumerate(self.all_tokens))
         lowercase__     =  {tok: ind for ind, tok in enumerate(self.all_tokens)}
         lowercase__     =  unk_token
         lowercase__     =  cls_token
         lowercase__     =  pad_token
         lowercase__     =  mask_token
         lowercase__     =  eos_token
         lowercase__     =  self.all_tokens
         self._create_trie(self.unique_no_split_tokens)


   def 			UpperCAmelCase		(						self  :			List[Any]			,				lowerCAmelCase  :			int)	->				str:




         """simple docstring"""

         return self._id_to_token.get(lowerCAmelCase			,				self.unk_token)


   def 			UpperCAmelCase		(						self  :			Dict			,				lowerCAmelCase  :			str)	->				int:




         """simple docstring"""

         return self._token_to_id.get(lowerCAmelCase			,				self._token_to_id.get(self.unk_token))


   def 			UpperCAmelCase		(						self  :			Optional[Any]			,				lowerCAmelCase  :			str			,				**lowerCAmelCase  :			Union[str, Any])	->				Dict:




         """simple docstring"""

         return text.split()


   def 			UpperCAmelCase		(						self  :			Union[str, Any]			,				lowerCAmelCase  :			Any=False)	->				Union[str, Any]:




         """simple docstring"""

         return len(self._id_to_token)


   def 			UpperCAmelCase		(						self  :			Tuple)	->				int:




         """simple docstring"""

         return {token: i for i, token in enumerate(self.all_tokens)}


   def 			UpperCAmelCase		(						self  :			Optional[Any]			,				lowerCAmelCase  :			str)	->				int:




         """simple docstring"""

         return self._token_to_id.get(lowerCAmelCase			,				self._token_to_id.get(self.unk_token))


   def 			UpperCAmelCase		(						self  :			Dict			,				lowerCAmelCase  :			int)	->				str:




         """simple docstring"""

         return self._id_to_token.get(lowerCAmelCase			,				self.unk_token)


   def 			UpperCAmelCase		(						self  :			Any			,				lowerCAmelCase  :			List[int]			,				lowerCAmelCase  :			Optional[List[int]] = None)	->				List[int]:




         """simple docstring"""

         lowercase__     =  [self.cls_token_id]
         lowercase__     =  [self.eos_token_id]  # No sep token in ESM vocabulary
         if token_ids_a is None:
               if self.eos_token_id is None:
                     return cls + token_ids_a
               else:
                     return cls + token_ids_a + sep
         elif self.eos_token_id is None:
               raise ValueError('Cannot tokenize multiple sequences when EOS token is not set!')
         return cls + token_ids_a + sep + token_ids_a + sep  # Multiple inputs always have an EOS token


   def 			UpperCAmelCase		(						self  :			Union[str, Any]			,				lowerCAmelCase  :			List			,				lowerCAmelCase  :			Optional[List] = None			,				lowerCAmelCase  :			bool = False)	->				List[int]:




         """simple docstring"""

         if already_has_special_tokens:
               if token_ids_a is not None:
                     raise ValueError(
                         'You should not supply a second sequence if the provided sequence of '
                         'ids is already formatted with special tokens for the model.')

               return [1 if token in self.all_special_ids else 0 for token in token_ids_a]
         lowercase__     =  [1] + ([0] * len(lowerCAmelCase)) + [1]
         if token_ids_a is not None:
               mask += [0] * len(lowerCAmelCase) + [1]
         return mask


   def 			UpperCAmelCase		(						self  :			Union[str, Any]			,				lowerCAmelCase  :			Tuple			,				lowerCAmelCase  :			Optional[int])	->				Dict:




         """simple docstring"""

         lowercase__     =  os.path.join(lowerCAmelCase			,				(filename_prefix + '-' if filename_prefix else '') + 'vocab.txt')
         with open(lowerCAmelCase			,				'w') as f:
               f.write('\n'.join(self.all_tokens))
         return (vocab_file,)


   @property
   def 			UpperCAmelCase		(						self  :			Optional[int])	->				int:




         """simple docstring"""

         return self.get_vocab_size(with_added_tokens=lowerCAmelCase)



   def 			UpperCAmelCase		(						self  :			Optional[Any]			,				lowerCAmelCase  :			Union[List[str], List[AddedToken]]			,				lowerCAmelCase  :			bool = False)	->				int:




         """simple docstring"""

         return super()._add_tokens(lowerCAmelCase			,				special_tokens=lowerCAmelCase)
