



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case__					:     str    		=   logging.get_logger(__name__)
snake_case__					:     Optional[Any]    		=   {"vocab_file": "spiece.model"}

snake_case__					:     str    		=   {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

snake_case__					:     Optional[Any]    		=   {
    "albert-base-v1": 5_12,
    "albert-large-v1": 5_12,
    "albert-xlarge-v1": 5_12,
    "albert-xxlarge-v1": 5_12,
    "albert-base-v2": 5_12,
    "albert-large-v2": 5_12,
    "albert-xlarge-v2": 5_12,
    "albert-xxlarge-v2": 5_12,
}

snake_case__					:     Union[str, Any]    		=   "▁"

class   _A   ( __UpperCAmelCase	):


						'''simple docstring'''





						_snake_case			:     Any      							=      VOCAB_FILES_NAMES
						_snake_case			:     int      							=      PRETRAINED_VOCAB_FILES_MAP
						_snake_case			:     Optional[int]      							=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



						def __init__(	self	:							Any							,      lowerCamelCase	:							Optional[int]							,      lowerCamelCase	:							Tuple=True							,      lowerCamelCase	:							List[str]=True							,      lowerCamelCase	:							Union[str, Any]=False							,      lowerCamelCase	:							str="[CLS]"							,      lowerCamelCase	:							int="[SEP]"							,      lowerCamelCase	:							Dict="<unk>"							,      lowerCamelCase	:							Dict="[SEP]"							,      lowerCamelCase	:							Any="<pad>"							,      lowerCamelCase	:							Optional[int]="[CLS]"							,      lowerCamelCase	:							Any="[MASK]"							,      lowerCamelCase	:							Optional[Dict[str, Any]] = None							,      **lowerCamelCase	:							str							,      ):






							'''simple docstring'''



							__lowercase		       =			(
							    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_							,      normalized=lowerCAmelCase_	)
							    if isinstance(lowerCAmelCase_							,      lowerCAmelCase_	)
							    else mask_token
							)

							__lowercase		       =			{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=lowerCAmelCase_							,      remove_space=lowerCAmelCase_							,      keep_accents=lowerCAmelCase_							,      bos_token=lowerCAmelCase_							,      eos_token=lowerCAmelCase_							,      unk_token=lowerCAmelCase_							,      sep_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      cls_token=lowerCAmelCase_							,      mask_token=lowerCAmelCase_							,      sp_model_kwargs=self.sp_model_kwargs							,      **lowerCAmelCase_							,      )

							__lowercase		       =			do_lower_case
							__lowercase		       =			remove_space
							__lowercase		       =			keep_accents
							__lowercase		       =			vocab_file

							__lowercase		       =			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
							self.sp_model.Load(lowerCAmelCase_	)



						@property
						def     _snake_case	(	self	:							Optional[Any]	):






							'''simple docstring'''



							return len(self.sp_model	)



						def     _snake_case	(	self	:							List[Any]	):






							'''simple docstring'''



							__lowercase		       =			{self.convert_ids_to_tokens(lowerCAmelCase_	): i for i in range(self.vocab_size	)}
							vocab.update(self.added_tokens_encoder	)
							return vocab



						def __getstate__(	self	:							Tuple	):






							'''simple docstring'''



							__lowercase		       =			self.__dict__.copy()
							__lowercase		       =			None
							return state



						def __setstate__(	self	:							Optional[Any]							,      lowerCamelCase	:							int	):






							'''simple docstring'''



							__lowercase		       =			d

							# for backward compatibility
							if not hasattr(self							,      "sp_model_kwargs"	):
								__lowercase		       =			{}

							__lowercase		       =			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
							self.sp_model.Load(self.vocab_file	)



						def     _snake_case	(	self	:							Optional[int]							,      lowerCamelCase	:							Optional[int]	):






							'''simple docstring'''



							if self.remove_space:
								__lowercase		       =			" ".join(inputs.strip().split()	)
							else:
								__lowercase		       =			inputs
							__lowercase		       =			outputs.replace("``"							,      "\""	).replace("''"							,      "\""	)

							if not self.keep_accents:
								__lowercase		       =			unicodedata.normalize("NFKD"							,      lowerCAmelCase_	)
								__lowercase		       =			"".join([c for c in outputs if not unicodedata.combining(lowerCAmelCase_	)]	)
							if self.do_lower_case:
								__lowercase		       =			outputs.lower()

							return outputs



						def     _snake_case	(	self	:							Optional[Any]							,      lowerCamelCase	:							str	):






							'''simple docstring'''



							__lowercase		       =			self.preprocess_text(lowerCAmelCase_	)
							__lowercase		       =			self.sp_model.encode(lowerCAmelCase_							,      out_type=lowerCAmelCase_	)
							__lowercase		       =			[]
							for piece in pieces:
								if len(lowerCAmelCase_	) > 1 and piece[-1] == str(","	) and piece[-2].isdigit():
									__lowercase		       =			self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase_							,      ""	)	)
									if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
										if len(cur_pieces[0]	) == 1:
											__lowercase		       =			cur_pieces[1:]
										else:
											__lowercase		       =			cur_pieces[0][1:]
									cur_pieces.append(piece[-1]	)
									new_pieces.extend(lowerCAmelCase_	)
								else:
									new_pieces.append(lowerCAmelCase_	)

							return new_pieces



						def     _snake_case	(	self	:							List[Any]							,      lowerCamelCase	:							int	):






							'''simple docstring'''



							return self.sp_model.PieceToId(lowerCAmelCase_	)



						def     _snake_case	(	self	:							Dict							,      lowerCamelCase	:							Dict	):






							'''simple docstring'''



							return self.sp_model.IdToPiece(lowerCAmelCase_	)



						def     _snake_case	(	self	:							Optional[int]							,      lowerCamelCase	:							int	):






							'''simple docstring'''



							__lowercase		       =			[]
							__lowercase		       =			""
							__lowercase		       =			False
							for token in tokens:
								# make sure that special tokens are not decoded using sentencepiece model
								if token in self.all_special_tokens:
									if not prev_is_special:
										out_string += " "
									out_string += self.sp_model.decode(lowerCAmelCase_	) + token
									__lowercase		       =			True
									__lowercase		       =			[]
								else:
									current_sub_tokens.append(lowerCAmelCase_	)
									__lowercase		       =			False
							out_string += self.sp_model.decode(lowerCAmelCase_	)
							return out_string.strip()



						def     _snake_case	(	self	:							Dict							,      lowerCamelCase	:							List[int]							,      lowerCamelCase	:							Optional[List[int]] = None	):






							'''simple docstring'''



							__lowercase		       =			[self.sep_token_id]
							__lowercase		       =			[self.cls_token_id]
							if token_ids_a is None:
								return cls + token_ids_a + sep
							return cls + token_ids_a + sep + token_ids_a + sep



						def     _snake_case	(	self	:							Tuple							,      lowerCamelCase	:							List[int]							,      lowerCamelCase	:							Optional[List[int]] = None							,      lowerCamelCase	:							bool = False	):






							'''simple docstring'''



							if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=lowerCAmelCase_							,      token_ids_a=lowerCAmelCase_							,      already_has_special_tokens=lowerCAmelCase_	)

							if token_ids_a is not None:
								return [1] + ([0] * len(lowerCAmelCase_	)) + [1] + ([0] * len(lowerCAmelCase_	)) + [1]
							return [1] + ([0] * len(lowerCAmelCase_	)) + [1]



						def     _snake_case	(	self	:							List[str]							,      lowerCamelCase	:							List[int]							,      lowerCamelCase	:							Optional[List[int]] = None	):






							'''simple docstring'''



							__lowercase		       =			[self.sep_token_id]
							__lowercase		       =			[self.cls_token_id]

							if token_ids_a is None:
								return len(cls + token_ids_a + sep	) * [0]
							return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]



						def     _snake_case	(	self	:							Dict							,      lowerCamelCase	:							str							,      lowerCamelCase	:							Optional[str] = None	):






							'''simple docstring'''



							if not os.path.isdir(lowerCAmelCase_	):
								logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""	)
								return
							__lowercase		       =			os.path.join(
							    lowerCAmelCase_							,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)

							if os.path.abspath(self.vocab_file	) != os.path.abspath(lowerCAmelCase_	) and os.path.isfile(self.vocab_file	):
								copyfile(self.vocab_file							,      lowerCAmelCase_	)
							elif not os.path.isfile(self.vocab_file	):
								with open(lowerCAmelCase_							,      "wb"	) as fi:
									__lowercase		       =			self.sp_model.serialized_model_proto()
									fi.write(lowerCAmelCase_	)

							return (out_vocab_file,)








# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.whisper import WhisperForConditionalGeneration, WhisperProcessor
from .base import PipelineTool

class   _A   ( _lowercase	):


						'''simple docstring'''





						_snake_case			:     Dict      							=      """openai/whisper-base"""
						_snake_case			:     Union[str, Any]      							=      (
						    """This is a tool that transcribes an audio into text. It takes an input named `audio` and returns the """
						    """transcribed text."""
						)
						_snake_case			:     Any      							=      """transcriber"""
						_snake_case			:     Any      							=      WhisperProcessor
						_snake_case			:     Optional[int]      							=      WhisperForConditionalGeneration

						_snake_case			:     str      							=      ["""audio"""]
						_snake_case			:     Optional[int]      							=      ["""text"""]



						def     _snake_case	(	self	:							List[str]							,      lowerCamelCase	:							Optional[int]	):






							'''simple docstring'''



							return self.pre_processor(lowerCamelCase							,      return_tensors="pt"	).input_features



						def     _snake_case	(	self	:							str							,      lowerCamelCase	:							List[Any]	):






							'''simple docstring'''



							return self.model.generate(inputs=lowerCamelCase	)



						def     _snake_case	(	self	:							List[str]							,      lowerCamelCase	:							Optional[Any]	):






							'''simple docstring'''



							return self.pre_processor.batch_decode(lowerCamelCase							,      skip_special_tokens=lowerCamelCase	)[0]



