



from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


a_			=					logging.get_logger(__name__)

a_			=					{
    """Helsinki-NLP/opus-mt-en-de""": """https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/config.json""",
    # See all Marian models at https://huggingface.co/models?filter=marian
}



class  __lowerCAmelCase     (	lowerCAmelCase__   ):
 lowerCAmelCase__       					=  """marian"""
 lowerCAmelCase__       					=  ["""past_key_values"""]
 lowerCAmelCase__       					=  {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}




 def __init__(						self	,				__UpperCAmelCase=58101	,				__UpperCAmelCase=None	,				__UpperCAmelCase=1024	,				__UpperCAmelCase=12	,				__UpperCAmelCase=4096	,				__UpperCAmelCase=16	,				__UpperCAmelCase=12	,				__UpperCAmelCase=4096	,				__UpperCAmelCase=16	,				__UpperCAmelCase=0.0	,				__UpperCAmelCase=0.0	,				__UpperCAmelCase=True	,				__UpperCAmelCase=True	,				__UpperCAmelCase="gelu"	,				__UpperCAmelCase=1024	,				__UpperCAmelCase=0.1	,				__UpperCAmelCase=0.0	,				__UpperCAmelCase=0.0	,				__UpperCAmelCase=0.02	,				__UpperCAmelCase=58100	,				__UpperCAmelCase=False	,				__UpperCAmelCase=58100	,				__UpperCAmelCase=0	,				__UpperCAmelCase=0	,				__UpperCAmelCase=True	,				**__UpperCAmelCase	,				):



    '''simple docstring'''
    __lowerCamelCase													=    vocab_size
    __lowerCamelCase													=    decoder_vocab_size or vocab_size
    __lowerCamelCase													=    max_position_embeddings
    __lowerCamelCase													=    d_model
    __lowerCamelCase													=    encoder_ffn_dim
    __lowerCamelCase													=    encoder_layers
    __lowerCamelCase													=    encoder_attention_heads
    __lowerCamelCase													=    decoder_ffn_dim
    __lowerCamelCase													=    decoder_layers
    __lowerCamelCase													=    decoder_attention_heads
    __lowerCamelCase													=    dropout
    __lowerCamelCase													=    attention_dropout
    __lowerCamelCase													=    activation_dropout
    __lowerCamelCase													=    activation_function
    __lowerCamelCase													=    init_std
    __lowerCamelCase													=    encoder_layerdrop
    __lowerCamelCase													=    decoder_layerdrop
    __lowerCamelCase													=    use_cache
    __lowerCamelCase													=    encoder_layers
    __lowerCamelCase													=    scale_embedding  # scale factor will be sqrt(d_model) if True
    __lowerCamelCase													=    share_encoder_decoder_embeddings
    super().__init__(
        pad_token_id=__UpperCAmelCase	,				eos_token_id=__UpperCAmelCase	,				is_encoder_decoder=__UpperCAmelCase	,				decoder_start_token_id=__UpperCAmelCase	,				forced_eos_token_id=__UpperCAmelCase	,				**__UpperCAmelCase	,				)



class  __lowerCAmelCase     (	lowerCAmelCase__   ):




 @property
 # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.inputs
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    if self.task in ["default", "seq2seq-lm"]:
       __lowerCamelCase													=    OrderedDict(
           [
               ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
           ]				)

       if self.use_past:
          __lowerCamelCase													=    {0: '''batch'''}
          __lowerCamelCase													=    {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
       else:
          __lowerCamelCase													=    {0: '''batch''', 1: '''decoder_sequence'''}
          __lowerCamelCase													=    {0: '''batch''', 1: '''decoder_sequence'''}

       if self.use_past:
          self.fill_with_past_key_values_(__UpperCAmelCase	,				direction='''inputs'''				)
    elif self.task == "causal-lm":
       # TODO: figure this case out.
       __lowerCamelCase													=    OrderedDict(
           [
               ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
           ]				)
       if self.use_past:
          __lowerCamelCase							,__lowerCamelCase													=    self.num_layers
          for i in range(__UpperCAmelCase				):
             __lowerCamelCase													=    {0: '''batch''', 2: '''past_sequence + sequence'''}
             __lowerCamelCase													=    {0: '''batch''', 2: '''past_sequence + sequence'''}
    else:
       __lowerCamelCase													=    OrderedDict(
           [
               ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
               ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
           ]				)

    return common_inputs




 @property
 # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    if self.task in ["default", "seq2seq-lm"]:
       __lowerCamelCase													=    super().outputs
    else:
       __lowerCamelCase													=    super(__UpperCAmelCase	,				self				).outputs
       if self.use_past:
          __lowerCamelCase							,__lowerCamelCase													=    self.num_layers
          for i in range(__UpperCAmelCase				):
             __lowerCamelCase													=    {0: '''batch''', 2: '''past_sequence + sequence'''}
             __lowerCamelCase													=    {0: '''batch''', 2: '''past_sequence + sequence'''}
    return common_outputs




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = -1	,				__UpperCAmelCase = -1	,				__UpperCAmelCase = False	,				__UpperCAmelCase = None	,				):



    '''simple docstring'''
    __lowerCamelCase													=    self._generate_dummy_inputs_for_encoder_and_decoder(
        __UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase				)

    # Generate decoder inputs
    __lowerCamelCase													=    seq_length if not self.use_past else 1
    __lowerCamelCase													=    self._generate_dummy_inputs_for_encoder_and_decoder(
        __UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase				)
    __lowerCamelCase													=    {F"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
    __lowerCamelCase													=    dict(**__UpperCAmelCase	,				**__UpperCAmelCase				)

    if self.use_past:
       if not is_torch_available():
          raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''				)
       else:
          import torch
       __lowerCamelCase							,__lowerCamelCase													=    common_inputs['''input_ids'''].shape
       __lowerCamelCase													=    common_inputs['''decoder_input_ids'''].shape[1]
       __lowerCamelCase							,__lowerCamelCase													=    self.num_attention_heads
       __lowerCamelCase													=    (
           batch,
           num_encoder_attention_heads,
           encoder_seq_length,
           self._config.hidden_size // num_encoder_attention_heads,
       )
       __lowerCamelCase													=    decoder_seq_length + 3
       __lowerCamelCase													=    (
           batch,
           num_decoder_attention_heads,
           decoder_past_length,
           self._config.hidden_size // num_decoder_attention_heads,
       )

       __lowerCamelCase													=    torch.cat(
           [common_inputs['''decoder_attention_mask'''], torch.ones(__UpperCAmelCase	,				__UpperCAmelCase				)]	,				dim=1				)

       __lowerCamelCase													=    []
       # If the number of encoder and decoder layers are present in the model configuration, both are considered
       __lowerCamelCase							,__lowerCamelCase													=    self.num_layers
       __lowerCamelCase													=    min(__UpperCAmelCase	,				__UpperCAmelCase				)
       __lowerCamelCase													=    max(__UpperCAmelCase	,				__UpperCAmelCase				) - min_num_layers
       __lowerCamelCase													=    '''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

       for _ in range(__UpperCAmelCase				):
          common_inputs["past_key_values"].append(
              (
                  torch.zeros(__UpperCAmelCase				),
                  torch.zeros(__UpperCAmelCase				),
                  torch.zeros(__UpperCAmelCase				),
                  torch.zeros(__UpperCAmelCase				),
              )				)
       # TODO: test this.
       __lowerCamelCase													=    encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
       for _ in range(__UpperCAmelCase	,				__UpperCAmelCase				):
          common_inputs["past_key_values"].append((torch.zeros(__UpperCAmelCase				), torch.zeros(__UpperCAmelCase				))				)
    return common_inputs




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = -1	,				__UpperCAmelCase = -1	,				__UpperCAmelCase = False	,				__UpperCAmelCase = None	,				):



    '''simple docstring'''
    __lowerCamelCase													=    self._generate_dummy_inputs_for_encoder_and_decoder(
        __UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase				)

    if self.use_past:
       if not is_torch_available():
          raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''				)
       else:
          import torch
       __lowerCamelCase							,__lowerCamelCase													=    common_inputs['''input_ids'''].shape
       # Not using the same length for past_key_values
       __lowerCamelCase													=    seqlen + 2
       __lowerCamelCase							,__lowerCamelCase													=    self.num_layers
       __lowerCamelCase							,__lowerCamelCase													=    self.num_attention_heads
       __lowerCamelCase													=    (
           batch,
           num_encoder_attention_heads,
           past_key_values_length,
           self._config.hidden_size // num_encoder_attention_heads,
       )

       __lowerCamelCase													=    common_inputs['''attention_mask'''].dtype
       __lowerCamelCase													=    torch.cat(
           [common_inputs['''attention_mask'''], torch.ones(__UpperCAmelCase	,				__UpperCAmelCase	,				dtype=__UpperCAmelCase				)]	,				dim=1				)
       __lowerCamelCase													=    [
           (torch.zeros(__UpperCAmelCase				), torch.zeros(__UpperCAmelCase				)) for _ in range(__UpperCAmelCase				)
       ]
    return common_inputs




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = -1	,				__UpperCAmelCase = -1	,				__UpperCAmelCase = False	,				__UpperCAmelCase = None	,				):



    '''simple docstring'''
    # Copied from OnnxConfig.generate_dummy_inputs
    # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
    # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
    __lowerCamelCase													=    compute_effective_axis_dimension(
        __UpperCAmelCase	,				fixed_dimension=OnnxConfig.default_fixed_batch	,				num_token_to_add=0				)

    # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
    __lowerCamelCase													=    tokenizer.num_special_tokens_to_add(__UpperCAmelCase				)
    __lowerCamelCase													=    compute_effective_axis_dimension(
        __UpperCAmelCase	,				fixed_dimension=OnnxConfig.default_fixed_sequence	,				num_token_to_add=__UpperCAmelCase				)

    # Generate dummy inputs according to compute batch and sequence
    __lowerCamelCase													=    [''' '''.join([tokenizer.unk_token]				) * seq_length] * batch_size
    __lowerCamelCase													=    dict(tokenizer(__UpperCAmelCase	,				return_tensors=__UpperCAmelCase				)				)
    return common_inputs




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase = -1	,				__UpperCAmelCase = -1	,				__UpperCAmelCase = False	,				__UpperCAmelCase = None	,				):



    '''simple docstring'''
    if self.task in ["default", "seq2seq-lm"]:
       __lowerCamelCase													=    self._generate_dummy_inputs_for_default_and_seqaseq_lm(
           __UpperCAmelCase	,				batch_size=__UpperCAmelCase	,				seq_length=__UpperCAmelCase	,				is_pair=__UpperCAmelCase	,				framework=__UpperCAmelCase				)

    else:
       __lowerCamelCase													=    self._generate_dummy_inputs_for_causal_lm(
           __UpperCAmelCase	,				batch_size=__UpperCAmelCase	,				seq_length=__UpperCAmelCase	,				is_pair=__UpperCAmelCase	,				framework=__UpperCAmelCase				)

    return common_inputs




 def        lowerCamelCase      (						self	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase				):



    '''simple docstring'''
    if self.task in ["default", "seq2seq-lm"]:
       __lowerCamelCase													=    super()._flatten_past_key_values_(__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase				)
    else:
       __lowerCamelCase													=    super(__UpperCAmelCase	,				self				)._flatten_past_key_values_(
           __UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase	,				__UpperCAmelCase				)




 @property
 def        lowerCamelCase      (						self				):



    '''simple docstring'''
    return 1E-4











import unicodedata
from dataclasses import dataclass
from typing import Optional, Union

import numpy as np

from transformers.data.data_collator import DataCollatorMixin
from transformers.file_utils import PaddingStrategy
from transformers.tokenization_utils_base import PreTrainedTokenizerBase





def 		a__			(		_UpperCamelCase			: Tuple ,_UpperCamelCase			: Any ,_UpperCamelCase			: Union[str, Any] ,_UpperCamelCase			: Any							):
   if isinstance(_UpperCamelCase ,_UpperCamelCase							):
      __lowerCamelCase													=    np.full((len(_UpperCamelCase							), sequence_length, 2) ,_UpperCamelCase							)
   else:
      __lowerCamelCase													=    np.full((len(_UpperCamelCase							), sequence_length) ,_UpperCamelCase							)

   for i, tensor in enumerate(_UpperCamelCase							):
      if padding_side == "right":
         if isinstance(_UpperCamelCase ,_UpperCamelCase							):
            __lowerCamelCase													=    tensor[:sequence_length]
         else:
            __lowerCamelCase													=    tensor[:sequence_length]
      else:
         if isinstance(_UpperCamelCase ,_UpperCamelCase							):
            __lowerCamelCase													=    tensor[:sequence_length]
         else:
            __lowerCamelCase													=    tensor[:sequence_length]

   return out_tensor.tolist()





def 		a__			(		_UpperCamelCase			: Dict							):
   __lowerCamelCase													=    ord(_UpperCamelCase							)
   if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 1_23 and cp <= 1_26):
      return True
   __lowerCamelCase													=    unicodedata.category(_UpperCamelCase							)
   if cat.startswith('''P'''							):
      return True
   return False





@dataclass
class  __lowerCAmelCase     (	lowerCAmelCase__   ):
 lowerCAmelCase__       					=  42
 lowerCAmelCase__       					=  True
 lowerCAmelCase__       					=  None
 lowerCAmelCase__       					=  None
 lowerCAmelCase__       					=  -1_0_0
 lowerCAmelCase__       					=  "pt"




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    import torch

    __lowerCamelCase													=    '''label''' if '''label''' in features[0].keys() else '''labels'''
    __lowerCamelCase													=    [feature[label_name] for feature in features] if label_name in features[0].keys() else None
    __lowerCamelCase													=    self.tokenizer.pad(
        __UpperCAmelCase	,				padding=self.padding	,				max_length=self.max_length	,				pad_to_multiple_of=self.pad_to_multiple_of	,				return_tensors='''pt''' if labels is None else None	,				)

    if labels is None:
       return batch

    __lowerCamelCase													=    torch.tensor(batch['''entity_ids''']				).shape[1]
    __lowerCamelCase													=    self.tokenizer.padding_side
    if padding_side == "right":
       __lowerCamelCase													=    [
           list(__UpperCAmelCase				) + [self.label_pad_token_id] * (sequence_length - len(__UpperCAmelCase				)) for label in labels
       ]
    else:
       __lowerCamelCase													=    [
           [self.label_pad_token_id] * (sequence_length - len(__UpperCAmelCase				)) + list(__UpperCAmelCase				) for label in labels
       ]

    __lowerCamelCase													=    [feature['''ner_tags'''] for feature in features]
    __lowerCamelCase													=    padding_tensor(__UpperCAmelCase	,				-1	,				__UpperCAmelCase	,				__UpperCAmelCase				)
    __lowerCamelCase													=    [feature['''original_entity_spans'''] for feature in features]
    __lowerCamelCase													=    padding_tensor(__UpperCAmelCase	,				(-1, -1)	,				__UpperCAmelCase	,				__UpperCAmelCase				)
    __lowerCamelCase													=    {k: torch.tensor(__UpperCAmelCase	,				dtype=torch.intaa				) for k, v in batch.items()}

    return batch






