


from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


a					  =     logging.get_logger(__name__)

a					  =     {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

a					  =     {
    'vocab_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'
    },
    'merges_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'
    },
}

a					  =     {'allegro/herbert-base-cased': 5_1_4}
a					  =     {}







class        UpperCamelCase__   (	__magic_name__   ):
					__SCREAMING_SNAKE_CASE       :  List[str]						=   VOCAB_FILES_NAMES
					__SCREAMING_SNAKE_CASE       :  Dict						=   PRETRAINED_VOCAB_FILES_MAP
					__SCREAMING_SNAKE_CASE       :  List[Any]						=   PRETRAINED_INIT_CONFIGURATION
					__SCREAMING_SNAKE_CASE       :  Dict						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__SCREAMING_SNAKE_CASE       :  str						=   HerbertTokenizer

					def __init__(     self       :   int							,					UpperCamelCase__       :   Dict=None							,					UpperCamelCase__       :   str=None							,					UpperCamelCase__       :   str=None							,					UpperCamelCase__       :   List[Any]="<s>"							,					UpperCamelCase__       :   Union[str, Any]="<unk>"							,					UpperCamelCase__       :   Union[str, Any]="<pad>"							,					UpperCamelCase__       :   Union[str, Any]="<mask>"							,					UpperCamelCase__       :   List[str]="</s>"							,					**UpperCamelCase__       :   List[Any]							,					):






										'''simple docstring'''






										super().__init__(
										    UpperCamelCase__							,					UpperCamelCase__							,					tokenizer_file=UpperCamelCase__							,					cls_token=UpperCamelCase__							,					unk_token=UpperCamelCase__							,					pad_token=UpperCamelCase__							,					mask_token=UpperCamelCase__							,					sep_token=UpperCamelCase__							,					**UpperCamelCase__							,					)

					def   UpperCAmelCase__       (     self       :   List[Any]							,					UpperCamelCase__       :   List[int]							,					UpperCamelCase__       :   Optional[List[int]] = None       ):






										'''simple docstring'''






										lowercase_  							=			[self.cls_token_id]
										lowercase_  							=			[self.sep_token_id]
										if token_ids_a is None:
															return cls + token_ids_a + sep

										return cls + token_ids_a + sep + token_ids_a + sep

					def   UpperCAmelCase__       (     self       :   Optional[Any]							,					UpperCamelCase__       :   List[int]							,					UpperCamelCase__       :   Optional[List[int]] = None							,					UpperCamelCase__       :   bool = False       ):






										'''simple docstring'''






										if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=UpperCamelCase__							,					token_ids_a=UpperCamelCase__							,					already_has_special_tokens=UpperCamelCase__       )

										if token_ids_a is None:
															return [1] + ([0] * len(UpperCamelCase__       )) + [1]
										return [1] + ([0] * len(UpperCamelCase__       )) + [1] + ([0] * len(UpperCamelCase__       )) + [1]

					def   UpperCAmelCase__       (     self       :   List[str]							,					UpperCamelCase__       :   List[int]							,					UpperCamelCase__       :   Optional[List[int]] = None       ):






										'''simple docstring'''






										lowercase_  							=			[self.sep_token_id]
										lowercase_  							=			[self.cls_token_id]

										if token_ids_a is None:
															return len(cls + token_ids_a + sep       ) * [0]
										return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







					def   UpperCAmelCase__       (     self       :   Optional[Any]							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   Optional[str] = None       ):






										'''simple docstring'''






										lowercase_  							=			self._tokenizer.model.save(UpperCamelCase__							,					name=UpperCamelCase__       )
										return tuple(UpperCamelCase__       )






import torch
import torch.nn as nn
from transformers.modeling_utils import ModuleUtilsMixin
from transformers.models.ta.modeling_ta import TaBlock, TaConfig, TaLayerNorm

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class        UpperCamelCase__   (	__magic_name__							,				__magic_name__							,				__magic_name__   ):

					@register_to_config
					def __init__(     self       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   float							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   int							,					UpperCamelCase__       :   str							,					UpperCamelCase__       :   bool = False							,					):






										'''simple docstring'''






										super().__init__()

										lowercase_  							=			nn.Embedding(UpperCamelCase__							,					UpperCamelCase__       )

										lowercase_  							=			nn.Embedding(UpperCamelCase__							,					UpperCamelCase__       )
										lowercase_  							=			False

										lowercase_  							=			nn.Dropout(p=UpperCamelCase__       )

										lowercase_  							=			TaConfig(
										    vocab_size=UpperCamelCase__							,					d_model=UpperCamelCase__							,					num_heads=UpperCamelCase__							,					d_kv=UpperCamelCase__							,					d_ff=UpperCamelCase__							,					dropout_rate=UpperCamelCase__							,					feed_forward_proj=UpperCamelCase__							,					is_decoder=UpperCamelCase__							,					is_encoder_decoder=UpperCamelCase__							,					)

										lowercase_  							=			nn.ModuleList()
										for lyr_num in range(UpperCamelCase__       ):
															lowercase_  							=			TaBlock(UpperCamelCase__       )
															self.encoders.append(UpperCamelCase__       )

										lowercase_  							=			TaLayerNorm(UpperCamelCase__       )
										lowercase_  							=			nn.Dropout(p=UpperCamelCase__       )







					def   UpperCAmelCase__       (     self       :   Optional[Any]							,					UpperCamelCase__       :   Optional[int]							,					UpperCamelCase__       :   str       ):






										'''simple docstring'''






										lowercase_  							=			self.token_embedder(UpperCamelCase__       )

										lowercase_  							=			encoder_input_tokens.shape[1]
										lowercase_  							=			torch.arange(UpperCamelCase__							,					device=encoder_input_tokens.device       )
										x += self.position_encoding(UpperCamelCase__       )

										lowercase_  							=			self.dropout_pre(UpperCamelCase__       )

										# inverted the attention mask
										lowercase_  							=			encoder_input_tokens.size()
										lowercase_  							=			self.get_extended_attention_mask(UpperCamelCase__							,					UpperCamelCase__       )

										for lyr in self.encoders:
															lowercase_  							=			lyr(UpperCamelCase__							,					UpperCamelCase__       )[0]
										lowercase_  							=			self.layer_norm(UpperCamelCase__       )

										return self.dropout_post(UpperCamelCase__       ), encoder_inputs_mask


