

"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


A_							:			int											= logging.get_logger(__name__)


A_							:			List[Any]											= {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all BART models at https://huggingface.co/models?filter=bart
A_							:			Optional[int]											= {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json""",
    },
}

A_							:			Dict											= {
    """facebook/bart-base""": 1_024,
    """facebook/bart-large""": 1_024,
    """facebook/bart-large-mnli""": 1_024,
    """facebook/bart-large-cnn""": 1_024,
    """facebook/bart-large-xsum""": 1_024,
    """yjernite/bart_eli5""": 1_024,
}





class    lowerCamelCase						(A__			):
			lowerCamelCase__   :		Dict							=							VOCAB_FILES_NAMES
			lowerCamelCase__   :		Optional[Any]							=							PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__   :		Dict							=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__   :		Optional[int]							=							["input_ids", "attention_mask"]
			lowerCamelCase__   :		Any							=							BartTokenizer

			def __init__(					self				:						List[Any]    ,				__UpperCAmelCase				:						Tuple=None    ,				__UpperCAmelCase				:						List[str]=None    ,				__UpperCAmelCase				:						str=None    ,				__UpperCAmelCase				:						str="replace"    ,				__UpperCAmelCase				:						Dict="<s>"    ,				__UpperCAmelCase				:						List[Any]="</s>"    ,				__UpperCAmelCase				:						List[Any]="</s>"    ,				__UpperCAmelCase				:						List[str]="<s>"    ,				__UpperCAmelCase				:						int="<unk>"    ,				__UpperCAmelCase				:						int="<pad>"    ,				__UpperCAmelCase				:						Union[str, Any]="<mask>"    ,				__UpperCAmelCase				:						List[str]=False    ,				__UpperCAmelCase				:						int=True    ,				**__UpperCAmelCase				:						Optional[int]    ,				)   ->    Any:
						super().__init__(
						    UpperCamelCase__    ,				UpperCamelCase__    ,				tokenizer_file=UpperCamelCase__    ,				errors=UpperCamelCase__    ,				bos_token=UpperCamelCase__    ,				eos_token=UpperCamelCase__    ,				sep_token=UpperCamelCase__    ,				cls_token=UpperCamelCase__    ,				unk_token=UpperCamelCase__    ,				pad_token=UpperCamelCase__    ,				mask_token=UpperCamelCase__    ,				add_prefix_space=UpperCamelCase__    ,				trim_offsets=UpperCamelCase__    ,				**UpperCamelCase__    ,				)

						SCREAMING_SNAKE_CASE__								=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()       )
						if pre_tok_state.get("""add_prefix_space"""    ,				UpperCamelCase__       ) != add_prefix_space:
									SCREAMING_SNAKE_CASE__								=      getattr(UpperCamelCase__    ,				pre_tok_state.pop("""type"""       )       )
									SCREAMING_SNAKE_CASE__								=      add_prefix_space
									SCREAMING_SNAKE_CASE__								=      pre_tok_class(**UpperCamelCase__       )

						SCREAMING_SNAKE_CASE__								=      add_prefix_space

						# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
						SCREAMING_SNAKE_CASE__								=      """post_processor"""
						SCREAMING_SNAKE_CASE__								=      getattr(self.backend_tokenizer    ,				UpperCamelCase__    ,				UpperCamelCase__       )
						if tokenizer_component_instance:
									SCREAMING_SNAKE_CASE__								=      json.loads(tokenizer_component_instance.__getstate__()       )

									# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
									if "sep" in state:
												SCREAMING_SNAKE_CASE__								=      tuple(state["""sep"""]       )
									if "cls" in state:
												SCREAMING_SNAKE_CASE__								=      tuple(state["""cls"""]       )

									SCREAMING_SNAKE_CASE__								=      False

									if state.get("""add_prefix_space"""    ,				UpperCamelCase__       ) != add_prefix_space:
												SCREAMING_SNAKE_CASE__								=      add_prefix_space
												SCREAMING_SNAKE_CASE__								=      True

									if state.get("""trim_offsets"""    ,				UpperCamelCase__       ) != trim_offsets:
												SCREAMING_SNAKE_CASE__								=      trim_offsets
												SCREAMING_SNAKE_CASE__								=      True

									if changes_to_apply:
												SCREAMING_SNAKE_CASE__								=      getattr(UpperCamelCase__    ,				state.pop("""type"""       )       )
												SCREAMING_SNAKE_CASE__								=      component_class(**UpperCamelCase__       )
												setattr(self.backend_tokenizer    ,				UpperCamelCase__    ,				UpperCamelCase__       )

			@property
			def 					SCREAMING_SNAKE_CASE				(					self				:						Tuple       )   ->    str:





						if self._mask_token is None:
									if self.verbose:
												logger.error("""Using mask_token, but it is not set yet."""       )
									return None
						return str(self._mask_token       )

			@mask_token.setter
			def 					SCREAMING_SNAKE_CASE				(					self				:						int    ,				__UpperCAmelCase				:						Tuple       )   ->    Tuple:
						SCREAMING_SNAKE_CASE__								=      AddedToken(UpperCamelCase__    ,				lstrip=UpperCamelCase__    ,				rstrip=UpperCamelCase__       ) if isinstance(UpperCamelCase__    ,				UpperCamelCase__       ) else value
						SCREAMING_SNAKE_CASE__								=      value

			def 					SCREAMING_SNAKE_CASE				(					self				:						Union[str, Any]    ,				*__UpperCAmelCase				:						int    ,				**__UpperCAmelCase				:						Dict       )   ->    BatchEncoding:
						SCREAMING_SNAKE_CASE__								=      kwargs.get("""is_split_into_words"""    ,				UpperCamelCase__       )

						if is_split_into_words and not self.add_prefix_space:
									raise ValueError(
									    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
									    """to use it with pretokenized inputs."""       )

						return super()._batch_encode_plus(*UpperCamelCase__    ,				**UpperCamelCase__       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						Tuple    ,				*__UpperCAmelCase				:						Any    ,				**__UpperCAmelCase				:						Dict       )   ->    BatchEncoding:
						SCREAMING_SNAKE_CASE__								=      kwargs.get("""is_split_into_words"""    ,				UpperCamelCase__       )

						if is_split_into_words and not self.add_prefix_space:
									raise ValueError(
									    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
									    """to use it with pretokenized inputs."""       )

						return super()._encode_plus(*UpperCamelCase__    ,				**UpperCamelCase__       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						int    ,				__UpperCAmelCase				:						List[str]    ,				__UpperCAmelCase				:						List[str] = None       )   ->    Tuple[str]:
						SCREAMING_SNAKE_CASE__								=      self._tokenizer.model.save(UpperCamelCase__    ,				name=UpperCamelCase__       )
						return tuple(UpperCamelCase__       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						str    ,				__UpperCAmelCase				:						str    ,				__UpperCAmelCase				:						str=None       )   ->    List[Any]:
						SCREAMING_SNAKE_CASE__								=      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
						if token_ids_a is None:
									return output

						return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





			def 					SCREAMING_SNAKE_CASE				(					self				:						str    ,				__UpperCAmelCase				:						Union[str, Any]    ,				__UpperCAmelCase				:						Optional[int] = None       )   ->    List[int]:
						SCREAMING_SNAKE_CASE__								=      [self.sep_token_id]
						SCREAMING_SNAKE_CASE__								=      [self.cls_token_id]

						if token_ids_a is None:
									return len(cls + token_ids_a + sep       ) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]





"""simple docstring"""




import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_							:			List[Any]											= logging.get_logger(__name__)

A_							:			Dict											= {
    "microsoft/unispeech-large-1500h-cv": (
        "https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json"
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}





class    lowerCamelCase						(A__			):
			lowerCamelCase__   :		int							=							'unispeech'

			def __init__(					self				:						Union[str, Any]    ,				__UpperCAmelCase				:						List[Any]=3_2    ,				__UpperCAmelCase				:						Union[str, Any]=7_6_8    ,				__UpperCAmelCase				:						Tuple=1_2    ,				__UpperCAmelCase				:						Dict=1_2    ,				__UpperCAmelCase				:						Optional[Any]=3_0_7_2    ,				__UpperCAmelCase				:						Optional[Any]="gelu"    ,				__UpperCAmelCase				:						Union[str, Any]=0.1    ,				__UpperCAmelCase				:						List[Any]=0.1    ,				__UpperCAmelCase				:						List[str]=0.1    ,				__UpperCAmelCase				:						List[str]=0.0    ,				__UpperCAmelCase				:						Tuple=0.0    ,				__UpperCAmelCase				:						List[Any]=0.1    ,				__UpperCAmelCase				:						Union[str, Any]=0.1    ,				__UpperCAmelCase				:						Union[str, Any]=0.02    ,				__UpperCAmelCase				:						Union[str, Any]=1e-5    ,				__UpperCAmelCase				:						List[Any]="group"    ,				__UpperCAmelCase				:						Union[str, Any]="gelu"    ,				__UpperCAmelCase				:						Any=(5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2, 5_1_2)    ,				__UpperCAmelCase				:						List[str]=(5, 2, 2, 2, 2, 2, 2)    ,				__UpperCAmelCase				:						int=(1_0, 3, 3, 3, 3, 2, 2)    ,				__UpperCAmelCase				:						str=False    ,				__UpperCAmelCase				:						Any=1_2_8    ,				__UpperCAmelCase				:						str=1_6    ,				__UpperCAmelCase				:						Optional[Any]=False    ,				__UpperCAmelCase				:						Optional[int]=True    ,				__UpperCAmelCase				:						Union[str, Any]=0.05    ,				__UpperCAmelCase				:						str=1_0    ,				__UpperCAmelCase				:						Optional[Any]=2    ,				__UpperCAmelCase				:						List[Any]=0.0    ,				__UpperCAmelCase				:						Tuple=1_0    ,				__UpperCAmelCase				:						Tuple=0    ,				__UpperCAmelCase				:						Tuple=3_2_0    ,				__UpperCAmelCase				:						Any=2    ,				__UpperCAmelCase				:						Optional[int]=0.1    ,				__UpperCAmelCase				:						Any=1_0_0    ,				__UpperCAmelCase				:						str=2_5_6    ,				__UpperCAmelCase				:						Dict=2_5_6    ,				__UpperCAmelCase				:						Dict=0.1    ,				__UpperCAmelCase				:						List[str]="mean"    ,				__UpperCAmelCase				:						Optional[Any]=False    ,				__UpperCAmelCase				:						List[Any]=False    ,				__UpperCAmelCase				:						str=2_5_6    ,				__UpperCAmelCase				:						Dict=8_0    ,				__UpperCAmelCase				:						List[Any]=0    ,				__UpperCAmelCase				:						int=1    ,				__UpperCAmelCase				:						Optional[int]=2    ,				__UpperCAmelCase				:						Any=0.5    ,				**__UpperCAmelCase				:						List[str]    ,				)   ->    Tuple:
						super().__init__(**__UpperCAmelCase    ,				pad_token_id=__UpperCAmelCase    ,				bos_token_id=__UpperCAmelCase    ,				eos_token_id=__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      hidden_size
						SCREAMING_SNAKE_CASE__								=      feat_extract_norm
						SCREAMING_SNAKE_CASE__								=      feat_extract_activation
						SCREAMING_SNAKE_CASE__								=      list(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      list(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      list(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      conv_bias
						SCREAMING_SNAKE_CASE__								=      num_conv_pos_embeddings
						SCREAMING_SNAKE_CASE__								=      num_conv_pos_embedding_groups
						SCREAMING_SNAKE_CASE__								=      len(self.conv_dim       )
						SCREAMING_SNAKE_CASE__								=      num_hidden_layers
						SCREAMING_SNAKE_CASE__								=      intermediate_size
						SCREAMING_SNAKE_CASE__								=      hidden_act
						SCREAMING_SNAKE_CASE__								=      num_attention_heads
						SCREAMING_SNAKE_CASE__								=      hidden_dropout
						SCREAMING_SNAKE_CASE__								=      attention_dropout
						SCREAMING_SNAKE_CASE__								=      activation_dropout
						SCREAMING_SNAKE_CASE__								=      feat_proj_dropout
						SCREAMING_SNAKE_CASE__								=      final_dropout
						SCREAMING_SNAKE_CASE__								=      layerdrop
						SCREAMING_SNAKE_CASE__								=      layer_norm_eps
						SCREAMING_SNAKE_CASE__								=      initializer_range
						SCREAMING_SNAKE_CASE__								=      num_ctc_classes
						SCREAMING_SNAKE_CASE__								=      vocab_size
						SCREAMING_SNAKE_CASE__								=      do_stable_layer_norm
						SCREAMING_SNAKE_CASE__								=      use_weighted_layer_sum
						SCREAMING_SNAKE_CASE__								=      classifier_proj_size

						if (
						    (len(self.conv_stride       ) != self.num_feat_extract_layers)
						    or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
						    or (len(self.conv_dim       ) != self.num_feat_extract_layers)
						):
									raise ValueError(
									    """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
									    """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
									    F""" {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,"""
									    F""" `len(config.conv_kernel) = {len(self.conv_kernel       )}`."""       )

						# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
						SCREAMING_SNAKE_CASE__								=      apply_spec_augment
						SCREAMING_SNAKE_CASE__								=      mask_time_prob
						SCREAMING_SNAKE_CASE__								=      mask_time_length
						SCREAMING_SNAKE_CASE__								=      mask_time_min_masks
						SCREAMING_SNAKE_CASE__								=      mask_feature_prob
						SCREAMING_SNAKE_CASE__								=      mask_feature_length
						SCREAMING_SNAKE_CASE__								=      mask_feature_min_masks

						# parameters for pretraining with codevector quantized representations
						SCREAMING_SNAKE_CASE__								=      num_codevectors_per_group
						SCREAMING_SNAKE_CASE__								=      num_codevector_groups
						SCREAMING_SNAKE_CASE__								=      contrastive_logits_temperature
						SCREAMING_SNAKE_CASE__								=      feat_quantizer_dropout
						SCREAMING_SNAKE_CASE__								=      num_negatives
						SCREAMING_SNAKE_CASE__								=      codevector_dim
						SCREAMING_SNAKE_CASE__								=      proj_codevector_dim
						SCREAMING_SNAKE_CASE__								=      diversity_loss_weight

						# ctc loss
						SCREAMING_SNAKE_CASE__								=      ctc_loss_reduction
						SCREAMING_SNAKE_CASE__								=      ctc_zero_infinity

						# pretraining loss
						SCREAMING_SNAKE_CASE__								=      replace_prob





			@property
			def 					SCREAMING_SNAKE_CASE				(					self				:						int       )   ->    Optional[Any]:
						return functools.reduce(operator.mul    ,				self.conv_stride    ,				1       )


