


"""simple docstring"""


import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class   _a      (      SCREAMING_SNAKE_CASE_		,		unittest.TestCase ):
    a_    :				Tuple             =    LEDTokenizer
    a_    :				Optional[Any]             =    LEDTokenizerFast
    a_    :				Optional[int]             =    True
    def      _UpperCamelCase       ( self							:					Optional[int]				):
          super().setUp()
          lowerCamelCase__           =		[
              'l',
              'o',
              'w',
              'e',
              'r',
              's',
              't',
              'i',
              'd',
              'n',
              '\u0120',
              '\u0120l',
              '\u0120n',
              '\u0120lo',
              '\u0120low',
              'er',
              '\u0120lowest',
              '\u0120newer',
              '\u0120wider',
              '<unk>',
          ]
          lowerCamelCase__           =		dict(zip(SCREAMING_SNAKE_CASE__							,					range(len(SCREAMING_SNAKE_CASE__				)				)				)				)
          lowerCamelCase__           =		['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
          lowerCamelCase__           =		{'unk_token': '<unk>'}

          lowerCamelCase__           =		os.path.join(self.tmpdirname							,					VOCAB_FILES_NAMES['vocab_file']				)
          lowerCamelCase__           =		os.path.join(self.tmpdirname							,					VOCAB_FILES_NAMES['merges_file']				)
          with open(self.vocab_file							,					'w'							,					encoding='utf-8'				) as fp:
                fp.write(json.dumps(SCREAMING_SNAKE_CASE__				) + '\n'				)
          with open(self.merges_file							,					'w'							,					encoding='utf-8'				) as fp:
                fp.write('\n'.join(SCREAMING_SNAKE_CASE__				)				)
    def      _UpperCamelCase       ( self							:					str							,					**SCREAMING_SNAKE_CASE__							:					Any				):
          kwargs.update(self.special_tokens_map				)
          return self.tokenizer_class.from_pretrained(self.tmpdirname							,					**SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					Any							,					**SCREAMING_SNAKE_CASE__							:					str				):
          kwargs.update(self.special_tokens_map				)
          return self.rust_tokenizer_class.from_pretrained(self.tmpdirname							,					**SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					Tuple				):
          return "lower newer", "lower newer"
    @cached_property
    def      _UpperCamelCase       ( self							:					Optional[int]				):
          return LEDTokenizer.from_pretrained('allenai/led-base-16384'				)
    @cached_property
    def      _UpperCamelCase       ( self							:					int				):
          return LEDTokenizerFast.from_pretrained('allenai/led-base-16384'				)
    @require_torch
    def      _UpperCamelCase       ( self							:					Tuple				):
          lowerCamelCase__           =		['A long paragraph for summarization.', 'Another paragraph for summarization.']
          lowerCamelCase__           =		[0, 2_50, 2_51, 1_78_18, 13, 3_91_86, 19_38, 4, 2]

          for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                lowerCamelCase__           =		tokenizer(SCREAMING_SNAKE_CASE__							,					max_length=len(SCREAMING_SNAKE_CASE__				)							,					padding=SCREAMING_SNAKE_CASE__							,					return_tensors='pt'				)
                self.assertIsInstance(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)

                self.assertEqual((2, 9)							,					batch.input_ids.shape				)
                self.assertEqual((2, 9)							,					batch.attention_mask.shape				)
                lowerCamelCase__           =		batch.input_ids.tolist()[0]
                self.assertListEqual(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)
    @require_torch
    def      _UpperCamelCase       ( self							:					List[Any]				):
          lowerCamelCase__           =		['A long paragraph for summarization.', 'Another paragraph for summarization.']
          for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                lowerCamelCase__           =		tokenizer(SCREAMING_SNAKE_CASE__							,					padding=SCREAMING_SNAKE_CASE__							,					return_tensors='pt'				)
                self.assertIn('input_ids'							,					SCREAMING_SNAKE_CASE__				)
                self.assertIn('attention_mask'							,					SCREAMING_SNAKE_CASE__				)
                self.assertNotIn('labels'							,					SCREAMING_SNAKE_CASE__				)
                self.assertNotIn('decoder_attention_mask'							,					SCREAMING_SNAKE_CASE__				)
    @require_torch
    def      _UpperCamelCase       ( self							:					Tuple				):
          lowerCamelCase__           =		[
              'Summary of the text.',
              'Another summary.',
          ]
          for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                lowerCamelCase__           =		tokenizer(text_target=SCREAMING_SNAKE_CASE__							,					max_length=32							,					padding='max_length'							,					return_tensors='pt'				)
                self.assertEqual(32							,					targets['input_ids'].shape[1]				)
    @require_torch
    def      _UpperCamelCase       ( self							:					Optional[int]				):
          for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                lowerCamelCase__           =		tokenizer(
                    ['I am a small frog' * 10_24, 'I am a small frog']							,					padding=SCREAMING_SNAKE_CASE__							,					truncation=SCREAMING_SNAKE_CASE__							,					return_tensors='pt'				)
                self.assertIsInstance(SCREAMING_SNAKE_CASE__							,					SCREAMING_SNAKE_CASE__				)
                self.assertEqual(batch.input_ids.shape							,					(2, 51_22)				)
    @require_torch
    def      _UpperCamelCase       ( self							:					Optional[Any]				):
          lowerCamelCase__           =		['A long paragraph for summarization.']
          lowerCamelCase__           =		[
              'Summary of the text.',
          ]
          for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                lowerCamelCase__           =		tokenizer(SCREAMING_SNAKE_CASE__							,					return_tensors='pt'				)
                lowerCamelCase__           =		tokenizer(text_target=SCREAMING_SNAKE_CASE__							,					return_tensors='pt'				)
                lowerCamelCase__           =		inputs['input_ids']
                lowerCamelCase__           =		targets['input_ids']
                self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()				)
                self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()				)
                self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()				)
                self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()				)
    @require_torch
    def      _UpperCamelCase       ( self							:					int				):
          for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                lowerCamelCase__           =		['Summary of the text.', 'Another summary.']
                lowerCamelCase__           =		[[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

                lowerCamelCase__           =		tokenizer(SCREAMING_SNAKE_CASE__							,					padding=SCREAMING_SNAKE_CASE__				)
                lowerCamelCase__           =		[[0] * len(SCREAMING_SNAKE_CASE__				) for x in encoded_output['input_ids']]
                lowerCamelCase__           =		tokenizer.pad(SCREAMING_SNAKE_CASE__				)
                self.assertSequenceEqual(outputs['global_attention_mask']							,					SCREAMING_SNAKE_CASE__				)
    def      _UpperCamelCase       ( self							:					List[Any]				):
          pass


    def      _UpperCamelCase       ( self							:					Tuple				):
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
                      lowerCamelCase__           =		self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__				)
                      lowerCamelCase__           =		self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__				)
                      lowerCamelCase__           =		'A, <mask> AllenNLP sentence.'
                      lowerCamelCase__           =		tokenizer_r.encode_plus(SCREAMING_SNAKE_CASE__							,					add_special_tokens=SCREAMING_SNAKE_CASE__							,					return_token_type_ids=SCREAMING_SNAKE_CASE__				)
                      lowerCamelCase__           =		tokenizer_p.encode_plus(SCREAMING_SNAKE_CASE__							,					add_special_tokens=SCREAMING_SNAKE_CASE__							,					return_token_type_ids=SCREAMING_SNAKE_CASE__				)
                      self.assertEqual(sum(tokens_r['token_type_ids']				)							,					sum(tokens_p['token_type_ids']				)				)
                      self.assertEqual(
                          sum(tokens_r['attention_mask']				) / len(tokens_r['attention_mask']				)							,					sum(tokens_p['attention_mask']				) / len(tokens_p['attention_mask']				)							,					)

                      lowerCamelCase__           =		tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']				)
                      lowerCamelCase__           =		tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']				)
                      self.assertSequenceEqual(tokens_p['input_ids']							,					[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]				)
                      self.assertSequenceEqual(tokens_r['input_ids']							,					[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]				)

                      self.assertSequenceEqual(
                          SCREAMING_SNAKE_CASE__							,					['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']				)
                      self.assertSequenceEqual(
                          SCREAMING_SNAKE_CASE__							,					['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']				)







"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_snake_case          =							logging.get_logger(__name__)

_snake_case          =							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_snake_case          =							[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

_snake_case          =							{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
_snake_case          =							{f"""funnel-transformer/{name}""": 512 for name in _model_names}
_snake_case          =							{f"""funnel-transformer/{name}""": {"do_lower_case": True} for name in _model_names}



class   _a      (      SCREAMING_SNAKE_CASE_ ):
    a_    :				List[str]             =    VOCAB_FILES_NAMES
    a_    :				List[str]             =    PRETRAINED_VOCAB_FILES_MAP
    a_    :				Optional[int]             =    PRETRAINED_INIT_CONFIGURATION
    a_    :				List[str]             =    FunnelTokenizer
    a_    :				Dict             =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a_    :				int             =    2
    def __init__( self							:					int							,					SCREAMING_SNAKE_CASE__							:					Optional[int]=None							,					SCREAMING_SNAKE_CASE__							:					Union[str, Any]=None							,					SCREAMING_SNAKE_CASE__							:					int=True							,					SCREAMING_SNAKE_CASE__							:					Any="<unk>"							,					SCREAMING_SNAKE_CASE__							:					List[Any]="<sep>"							,					SCREAMING_SNAKE_CASE__							:					int="<pad>"							,					SCREAMING_SNAKE_CASE__							:					Tuple="<cls>"							,					SCREAMING_SNAKE_CASE__							:					Tuple="<mask>"							,					SCREAMING_SNAKE_CASE__							:					Any="<s>"							,					SCREAMING_SNAKE_CASE__							:					Tuple="</s>"							,					SCREAMING_SNAKE_CASE__							:					str=True							,					SCREAMING_SNAKE_CASE__							:					Any=True							,					SCREAMING_SNAKE_CASE__							:					str=None							,					SCREAMING_SNAKE_CASE__							:					int="##"							,					**SCREAMING_SNAKE_CASE__							:					Any							,					):
          super().__init__(
              SCREAMING_SNAKE_CASE__							,					tokenizer_file=SCREAMING_SNAKE_CASE__							,					do_lower_case=SCREAMING_SNAKE_CASE__							,					unk_token=SCREAMING_SNAKE_CASE__							,					sep_token=SCREAMING_SNAKE_CASE__							,					pad_token=SCREAMING_SNAKE_CASE__							,					cls_token=SCREAMING_SNAKE_CASE__							,					mask_token=SCREAMING_SNAKE_CASE__							,					bos_token=SCREAMING_SNAKE_CASE__							,					eos_token=SCREAMING_SNAKE_CASE__							,					clean_text=SCREAMING_SNAKE_CASE__							,					tokenize_chinese_chars=SCREAMING_SNAKE_CASE__							,					strip_accents=SCREAMING_SNAKE_CASE__							,					wordpieces_prefix=SCREAMING_SNAKE_CASE__							,					**SCREAMING_SNAKE_CASE__							,					)

          lowerCamelCase__           =		json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
          if (
              normalizer_state.get('lowercase'							,					SCREAMING_SNAKE_CASE__				) != do_lower_case
              or normalizer_state.get('strip_accents'							,					SCREAMING_SNAKE_CASE__				) != strip_accents
              or normalizer_state.get('handle_chinese_chars'							,					SCREAMING_SNAKE_CASE__				) != tokenize_chinese_chars
          ):
                lowerCamelCase__           =		getattr(SCREAMING_SNAKE_CASE__							,					normalizer_state.pop('type'				)				)
                lowerCamelCase__           =		do_lower_case
                lowerCamelCase__           =		strip_accents
                lowerCamelCase__           =		tokenize_chinese_chars
                lowerCamelCase__           =		normalizer_class(**SCREAMING_SNAKE_CASE__				)

          lowerCamelCase__           =		do_lower_case
    def      _UpperCamelCase       ( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					Tuple							,					SCREAMING_SNAKE_CASE__							:					Optional[Any]=None				):
          lowerCamelCase__           =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

          return output
    def      _UpperCamelCase       ( self							:					Dict							,					SCREAMING_SNAKE_CASE__							:					List[int]							,					SCREAMING_SNAKE_CASE__							:					Optional[List[int]] = None				):
          lowerCamelCase__           =		[self.sep_token_id]
          lowerCamelCase__           =		[self.cls_token_id]
          if token_ids_a is None:
                return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0]
          return len(cls				) * [self.cls_token_type_id] + len(token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]


    def      _UpperCamelCase       ( self							:					Optional[int]							,					SCREAMING_SNAKE_CASE__							:					str							,					SCREAMING_SNAKE_CASE__							:					Optional[str] = None				):
          lowerCamelCase__           =		self._tokenizer.model.save(SCREAMING_SNAKE_CASE__							,					name=SCREAMING_SNAKE_CASE__				)
          return tuple(SCREAMING_SNAKE_CASE__				)



