






import json
import os
from typing import Optional

import numpy as np

from ...feature_extraction_utils import BatchFeature
from ...processing_utils import ProcessorMixin
from ...utils import logging
from ...utils.hub import get_file_from_repo
from ..auto import AutoTokenizer


snake_case__      :	int       	=      logging.get_logger(__name__)


class 						_a       (  UpperCAmelCase__    ):






  """simple docstring"""





  A_						=   """AutoTokenizer"""
  A_						=   ["""tokenizer"""]

  A_						=   {
      """semantic_prompt""": 1,
      """coarse_prompt""": 2,
      """fine_prompt""": 2,
  }




  def __init__(     self		,		_UpperCAmelCase		,		_UpperCAmelCase=None )			-> int:
     super().__init__(_UpperCAmelCase )

     UpperCamelCase_									=     speaker_embeddings




  @classmethod
  def    _UpperCAmelCase     (     cls		,		_UpperCAmelCase		,		_UpperCAmelCase="speaker_embeddings_path.json"		,		**_UpperCAmelCase )			-> Union[str, Any]:

     if speaker_embeddings_dict_path is not None:
        UpperCamelCase_									=     get_file_from_repo(
            _UpperCAmelCase		,		_UpperCAmelCase		,		subfolder=kwargs.pop('subfolder'		,		_UpperCAmelCase )		,		cache_dir=kwargs.pop('cache_dir'		,		_UpperCAmelCase )		,		force_download=kwargs.pop('force_download'		,		_UpperCAmelCase )		,		proxies=kwargs.pop('proxies'		,		_UpperCAmelCase )		,		resume_download=kwargs.pop('resume_download'		,		_UpperCAmelCase )		,		local_files_only=kwargs.pop('local_files_only'		,		_UpperCAmelCase )		,		use_auth_token=kwargs.pop('use_auth_token'		,		_UpperCAmelCase )		,		revision=kwargs.pop('revision'		,		_UpperCAmelCase )		,		)
        if speaker_embeddings_path is None:
           logger.warning(
               f"""`{os.path.join(_UpperCAmelCase		,		_UpperCAmelCase )}` does not exists
                    , no preloaded speaker embeddings will be used - Make sure to provide a correct path to the json
                    dictionnary if wanted, otherwise set `speaker_embeddings_dict_path=None`.""" )
           UpperCamelCase_									=     None
        else:
           with open(_UpperCAmelCase ) as speaker_embeddings_json:
              UpperCamelCase_									=     json.load(_UpperCAmelCase )
     else:
        UpperCamelCase_									=     None

     UpperCamelCase_									=     AutoTokenizer.from_pretrained(_UpperCAmelCase		,		**_UpperCAmelCase )

     return cls(tokenizer=_UpperCAmelCase		,		speaker_embeddings=_UpperCAmelCase )




  def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		_UpperCAmelCase="speaker_embeddings_path.json"		,		_UpperCAmelCase="speaker_embeddings"		,		_UpperCAmelCase = False		,		**_UpperCAmelCase		,		)			-> List[str]:
     if self.speaker_embeddings is not None:
        os.makedirs(os.path.join(_UpperCAmelCase		,		_UpperCAmelCase		,		'v2' )		,		exist_ok=_UpperCAmelCase )

        UpperCamelCase_									=     {}

        UpperCamelCase_									=     save_directory

        for prompt_key in self.speaker_embeddings:
           if prompt_key != "repo_or_path":
              UpperCamelCase_									=     self._load_voice_preset(_UpperCAmelCase )

              UpperCamelCase_									=     {}
              for key in self.speaker_embeddings[prompt_key]:
                 np.save(
                     os.path.join(
                         embeddings_dict['repo_or_path']		,		_UpperCAmelCase		,		f"""{prompt_key}_{key}""" )		,		voice_preset[key]		,		allow_pickle=_UpperCAmelCase		,		)
                 UpperCamelCase_									=     os.path.join(_UpperCAmelCase		,		f"""{prompt_key}_{key}.npy""" )

              UpperCamelCase_									=     tmp_dict

        with open(os.path.join(_UpperCAmelCase		,		_UpperCAmelCase )		,		'w' ) as fp:
           json.dump(_UpperCAmelCase		,		_UpperCAmelCase )

     super().save_pretrained(_UpperCAmelCase		,		_UpperCAmelCase		,		**_UpperCAmelCase )




  def    _UpperCAmelCase     (     self		,		_UpperCAmelCase = None		,		**_UpperCAmelCase )			-> int:
     UpperCamelCase_									=     self.speaker_embeddings[voice_preset]

     UpperCamelCase_									=     {}
     for key in ["semantic_prompt", "coarse_prompt", "fine_prompt"]:
        if key not in voice_preset_paths:
           raise ValueError(
               f"""Voice preset unrecognized, missing {key} as a key in self.speaker_embeddings[{voice_preset}].""" )

        UpperCamelCase_									=     get_file_from_repo(
            self.speaker_embeddings.get('repo_or_path'		,		'/' )		,		voice_preset_paths[key]		,		subfolder=kwargs.pop('subfolder'		,		_UpperCAmelCase )		,		cache_dir=kwargs.pop('cache_dir'		,		_UpperCAmelCase )		,		force_download=kwargs.pop('force_download'		,		_UpperCAmelCase )		,		proxies=kwargs.pop('proxies'		,		_UpperCAmelCase )		,		resume_download=kwargs.pop('resume_download'		,		_UpperCAmelCase )		,		local_files_only=kwargs.pop('local_files_only'		,		_UpperCAmelCase )		,		use_auth_token=kwargs.pop('use_auth_token'		,		_UpperCAmelCase )		,		revision=kwargs.pop('revision'		,		_UpperCAmelCase )		,		)
        if path is None:
           raise ValueError(
               f"""`{os.path.join(self.speaker_embeddings.get("repo_or_path"		,		"/" )		,		voice_preset_paths[key] )}` does not exists
                    , no preloaded voice preset will be used - Make sure to provide correct paths to the {voice_preset}
                    embeddings.""" )

        UpperCamelCase_									=     np.load(_UpperCAmelCase )

     return voice_preset_dict




  def    _UpperCAmelCase     (     self		,		_UpperCAmelCase = None )			-> Optional[Any]:
     for key in ["semantic_prompt", "coarse_prompt", "fine_prompt"]:
        if key not in voice_preset:
           raise ValueError(f"""Voice preset unrecognized, missing {key} as a key.""" )

        if not isinstance(voice_preset[key]		,		np.ndarray ):
           raise ValueError(f"""{key} voice preset must be a {str(self.preset_shape[key] )}D ndarray.""" )

        if len(voice_preset[key].shape ) != self.preset_shape[key]:
           raise ValueError(f"""{key} voice preset must be a {str(self.preset_shape[key] )}D ndarray.""" )




  def __call__(     self		,		_UpperCAmelCase=None		,		_UpperCAmelCase=None		,		_UpperCAmelCase="pt"		,		_UpperCAmelCase=256		,		_UpperCAmelCase=False		,		_UpperCAmelCase=True		,		_UpperCAmelCase=False		,		**_UpperCAmelCase		,		)			-> Tuple:
     if voice_preset is not None and not isinstance(_UpperCAmelCase		,		_UpperCAmelCase ):
        if (
            isinstance(_UpperCAmelCase		,		_UpperCAmelCase )
            and self.speaker_embeddings is not None
            and voice_preset in self.speaker_embeddings
        ):
           UpperCamelCase_									=     self._load_voice_preset(_UpperCAmelCase )

        else:
           if isinstance(_UpperCAmelCase		,		_UpperCAmelCase ) and not voice_preset.endswith('.npz' ):
              UpperCamelCase_									=     voice_preset + '.npz'

           UpperCamelCase_									=     np.load(_UpperCAmelCase )

     if voice_preset is not None:
        self._validate_voice_preset_dict(_UpperCAmelCase		,		**_UpperCAmelCase )
        UpperCamelCase_									=     BatchFeature(data=_UpperCAmelCase		,		tensor_type=_UpperCAmelCase )

     UpperCamelCase_									=     self.tokenizer(
         _UpperCAmelCase		,		return_tensors=_UpperCAmelCase		,		padding='max_length'		,		max_length=_UpperCAmelCase		,		return_attention_mask=_UpperCAmelCase		,		return_token_type_ids=_UpperCAmelCase		,		add_special_tokens=_UpperCAmelCase		,		**_UpperCAmelCase		,		)

     if voice_preset is not None:
        UpperCamelCase_									=     voice_preset

     return encoded_text













import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


snake_case__      :	List[str]       	=      logging.get_logger(__name__)

snake_case__      :	Optional[int]       	=      {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case__      :	Optional[Any]       	=      {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

snake_case__      :	List[str]       	=      {
    """google/realm-cc-news-pretrained-embedder""": 5_1_2,
    """google/realm-cc-news-pretrained-encoder""": 5_1_2,
    """google/realm-cc-news-pretrained-scorer""": 5_1_2,
    """google/realm-cc-news-pretrained-openqa""": 5_1_2,
    """google/realm-orqa-nq-openqa""": 5_1_2,
    """google/realm-orqa-nq-reader""": 5_1_2,
    """google/realm-orqa-wq-openqa""": 5_1_2,
    """google/realm-orqa-wq-reader""": 5_1_2,
}

snake_case__      :	Tuple       	=      {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}


class 						_a       (  UpperCAmelCase__    ):






  """simple docstring"""





  A_						=   VOCAB_FILES_NAMES
  A_						=   PRETRAINED_VOCAB_FILES_MAP
  A_						=   PRETRAINED_INIT_CONFIGURATION
  A_						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  A_						=   RealmTokenizer




  def __init__(     self		,		_UpperCAmelCase=None		,		_UpperCAmelCase=None		,		_UpperCAmelCase=True		,		_UpperCAmelCase="[UNK]"		,		_UpperCAmelCase="[SEP]"		,		_UpperCAmelCase="[PAD]"		,		_UpperCAmelCase="[CLS]"		,		_UpperCAmelCase="[MASK]"		,		_UpperCAmelCase=True		,		_UpperCAmelCase=None		,		**_UpperCAmelCase		,		)			-> Optional[Any]:
     super().__init__(
         _UpperCAmelCase		,		tokenizer_file=_UpperCAmelCase		,		do_lower_case=_UpperCAmelCase		,		unk_token=_UpperCAmelCase		,		sep_token=_UpperCAmelCase		,		pad_token=_UpperCAmelCase		,		cls_token=_UpperCAmelCase		,		mask_token=_UpperCAmelCase		,		tokenize_chinese_chars=_UpperCAmelCase		,		strip_accents=_UpperCAmelCase		,		**_UpperCAmelCase		,		)

     UpperCamelCase_									=     json.loads(self.backend_tokenizer.normalizer.__getstate__() )
     if (
         normalizer_state.get('lowercase'		,		_UpperCAmelCase ) != do_lower_case
         or normalizer_state.get('strip_accents'		,		_UpperCAmelCase ) != strip_accents
         or normalizer_state.get('handle_chinese_chars'		,		_UpperCAmelCase ) != tokenize_chinese_chars
     ):
        UpperCamelCase_									=     getattr(_UpperCAmelCase		,		normalizer_state.pop('type' ) )
        UpperCamelCase_									=     do_lower_case
        UpperCamelCase_									=     strip_accents
        UpperCamelCase_									=     tokenize_chinese_chars
        UpperCamelCase_									=     normalizer_class(**_UpperCAmelCase )

     UpperCamelCase_									=     do_lower_case




  def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		**_UpperCAmelCase )			-> List[Any]:
     UpperCamelCase_									=     PaddingStrategy.MAX_LENGTH

     UpperCamelCase_									=     text
     UpperCamelCase_									=     kwargs.pop('text_pair'		,		_UpperCAmelCase )
     UpperCamelCase_									=     kwargs.pop('return_tensors'		,		_UpperCAmelCase )

     UpperCamelCase_									=     {
         'input_ids': [],
         'attention_mask': [],
         'token_type_ids': [],
     }

     for idx, candidate_text in enumerate(_UpperCAmelCase ):
        if batch_text_pair is not None:
           UpperCamelCase_									=     batch_text_pair[idx]
        else:
           UpperCamelCase_									=     None

        UpperCamelCase_									=     super().__call__(_UpperCAmelCase		,		_UpperCAmelCase		,		return_tensors=_UpperCAmelCase		,		**_UpperCAmelCase )

        UpperCamelCase_									=     encoded_candidates.get('input_ids' )
        UpperCamelCase_									=     encoded_candidates.get('attention_mask' )
        UpperCamelCase_									=     encoded_candidates.get('token_type_ids' )

        if encoded_input_ids is not None:
           output_data["input_ids"].append(_UpperCAmelCase )
        if encoded_attention_mask is not None:
           output_data["attention_mask"].append(_UpperCAmelCase )
        if encoded_token_type_ids is not None:
           output_data["token_type_ids"].append(_UpperCAmelCase )

     UpperCamelCase_									=     {key: item for key, item in output_data.items() if len(_UpperCAmelCase ) != 0}

     return BatchEncoding(_UpperCAmelCase		,		tensor_type=_UpperCAmelCase )




  def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		_UpperCAmelCase=None )			-> Union[str, Any]:
     UpperCamelCase_									=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

     return output




  def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		_UpperCAmelCase = None )			-> List[int]:
     UpperCamelCase_									=     [self.sep_token_id]
     UpperCamelCase_									=     [self.cls_token_id]
     if token_ids_a is None:
        return len(cls + token_ids_a + sep ) * [0]
     return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]




  def    _UpperCAmelCase     (     self		,		_UpperCAmelCase		,		_UpperCAmelCase = None )			-> Tuple[str]:
     UpperCamelCase_									=     self._tokenizer.model.save(_UpperCAmelCase		,		name=_UpperCAmelCase )
     return tuple(_UpperCAmelCase )





