

import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class       __snake_case  (    lowerCAmelCase__     , unittest.TestCase       ):
  __lowerCAmelCase     :	Any						 =  BertTokenizer
  __lowerCAmelCase     :	List[Any]						 =  BertTokenizerFast
  __lowerCAmelCase     :	Union[str, Any]						 =  True
  __lowerCAmelCase     :	int						 =  True
  __lowerCAmelCase     :	Optional[int]						 =  filter_non_english

  def 						lowerCAmelCase__     (					self):
      super().setUp()

      SCREAMING_SNAKE_CASE_									=						[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      SCREAMING_SNAKE_CASE_									=						os.path.join(self.tmpdirname						,		VOCAB_FILES_NAMES['vocab_file'])
      with open(self.vocab_file						,		'w'						,		encoding='utf-8') as vocab_writer:
          vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))

  def 						lowerCAmelCase__     (					self						,		_A):
      SCREAMING_SNAKE_CASE_									=						'UNwant\u00E9d,running'
      SCREAMING_SNAKE_CASE_									=						'unwanted, running'
      return input_text, output_text

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.tokenizer_class(self.vocab_file)

      SCREAMING_SNAKE_CASE_									=						tokenizer.tokenize('UNwant\u00E9d,running')
      self.assertListEqual(_A						,		['un', '##want', '##ed', ',', 'runn', '##ing'])
      self.assertListEqual(tokenizer.convert_tokens_to_ids(_A)						,		[9, 6, 7, 12, 10, 11])

  def 						lowerCAmelCase__     (					self):
      if not self.test_rust_tokenizer:
          return

      SCREAMING_SNAKE_CASE_									=						self.get_tokenizer()
      SCREAMING_SNAKE_CASE_									=						self.get_rust_tokenizer()

      SCREAMING_SNAKE_CASE_									=						'UNwant\u00E9d,running'

      SCREAMING_SNAKE_CASE_									=						tokenizer.tokenize(_A)
      SCREAMING_SNAKE_CASE_									=						rust_tokenizer.tokenize(_A)
      self.assertListEqual(_A						,		_A)

      SCREAMING_SNAKE_CASE_									=						tokenizer.encode(_A						,		add_special_tokens=_A)
      SCREAMING_SNAKE_CASE_									=						rust_tokenizer.encode(_A						,		add_special_tokens=_A)
      self.assertListEqual(_A						,		_A)

      SCREAMING_SNAKE_CASE_									=						self.get_rust_tokenizer()
      SCREAMING_SNAKE_CASE_									=						tokenizer.encode(_A)
      SCREAMING_SNAKE_CASE_									=						rust_tokenizer.encode(_A)
      self.assertListEqual(_A						,		_A)

      # With lower casing
      SCREAMING_SNAKE_CASE_									=						self.get_tokenizer(do_lower_case=_A)
      SCREAMING_SNAKE_CASE_									=						self.get_rust_tokenizer(do_lower_case=_A)

      SCREAMING_SNAKE_CASE_									=						'UNwant\u00E9d,running'

      SCREAMING_SNAKE_CASE_									=						tokenizer.tokenize(_A)
      SCREAMING_SNAKE_CASE_									=						rust_tokenizer.tokenize(_A)
      self.assertListEqual(_A						,		_A)

      SCREAMING_SNAKE_CASE_									=						tokenizer.encode(_A						,		add_special_tokens=_A)
      SCREAMING_SNAKE_CASE_									=						rust_tokenizer.encode(_A						,		add_special_tokens=_A)
      self.assertListEqual(_A						,		_A)

      SCREAMING_SNAKE_CASE_									=						self.get_rust_tokenizer()
      SCREAMING_SNAKE_CASE_									=						tokenizer.encode(_A)
      SCREAMING_SNAKE_CASE_									=						rust_tokenizer.encode(_A)
      self.assertListEqual(_A						,		_A)

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')						,		['ah', '\u535A', '\u63A8', 'zz'])

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						BasicTokenizer(do_lower_case=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')						,		['hello', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')						,		['hello'])

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						BasicTokenizer(do_lower_case=_A						,		strip_accents=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')						,		['hällo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')						,		['h\u00E9llo'])

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						BasicTokenizer(do_lower_case=_A						,		strip_accents=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')						,		['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')						,		['hello'])

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						BasicTokenizer(do_lower_case=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')						,		['hallo', '!', 'how', 'are', 'you', '?'])
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo')						,		['hello'])

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						BasicTokenizer(do_lower_case=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')						,		['HeLLo', '!', 'how', 'Are', 'yoU', '?'])

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						BasicTokenizer(do_lower_case=_A						,		strip_accents=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')						,		['HäLLo', '!', 'how', 'Are', 'yoU', '?'])

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						BasicTokenizer(do_lower_case=_A						,		strip_accents=_A)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')						,		['HaLLo', '!', 'how', 'Are', 'yoU', '?'])

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						BasicTokenizer(do_lower_case=_A						,		never_split=['[UNK]'])

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')						,		['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						BasicTokenizer()
      SCREAMING_SNAKE_CASE_									=						'a\n\'ll !!to?\'d of, can\'t.'
      SCREAMING_SNAKE_CASE_									=						['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
      self.assertListEqual(tokenizer.tokenize(_A)						,		_A)

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      SCREAMING_SNAKE_CASE_									=						{}
      for i, token in enumerate(_A):
          SCREAMING_SNAKE_CASE_									=						i
      SCREAMING_SNAKE_CASE_									=						WordpieceTokenizer(vocab=_A						,		unk_token='[UNK]')

      self.assertListEqual(tokenizer.tokenize('')						,		[])

      self.assertListEqual(tokenizer.tokenize('unwanted running')						,		['un', '##want', '##ed', 'runn', '##ing'])

      self.assertListEqual(tokenizer.tokenize('unwantedX running')						,		['[UNK]', 'runn', '##ing'])

  def 						lowerCAmelCase__     (					self):
      self.assertTrue(_is_whitespace(' '))
      self.assertTrue(_is_whitespace('\t'))
      self.assertTrue(_is_whitespace('\r'))
      self.assertTrue(_is_whitespace('\n'))
      self.assertTrue(_is_whitespace('\u00A0'))

      self.assertFalse(_is_whitespace('A'))
      self.assertFalse(_is_whitespace('-'))

  def 						lowerCAmelCase__     (					self):
      self.assertTrue(_is_control('\u0005'))

      self.assertFalse(_is_control('A'))
      self.assertFalse(_is_control(' '))
      self.assertFalse(_is_control('\t'))
      self.assertFalse(_is_control('\r'))

  def 						lowerCAmelCase__     (					self):
      self.assertTrue(_is_punctuation('-'))
      self.assertTrue(_is_punctuation('$'))
      self.assertTrue(_is_punctuation('`'))
      self.assertTrue(_is_punctuation('.'))

      self.assertFalse(_is_punctuation('A'))
      self.assertFalse(_is_punctuation(' '))

  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.get_tokenizer()
      SCREAMING_SNAKE_CASE_									=						self.get_rust_tokenizer()

      # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
      self.assertListEqual([tokenizer.tokenize(_A) for t in ['Test', '\xad', 'test']]						,		[['[UNK]'], [], ['[UNK]']])

      self.assertListEqual(
          [rust_tokenizer.tokenize(_A) for t in ['Test', '\xad', 'test']]						,		[['[UNK]'], [], ['[UNK]']])

  @slow
  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						self.tokenizer_class.from_pretrained('bert-base-uncased')

      SCREAMING_SNAKE_CASE_									=						tokenizer.encode('sequence builders'						,		add_special_tokens=_A)
      SCREAMING_SNAKE_CASE_									=						tokenizer.encode('multi-sequence build'						,		add_special_tokens=_A)

      SCREAMING_SNAKE_CASE_									=						tokenizer.build_inputs_with_special_tokens(_A)
      SCREAMING_SNAKE_CASE_									=						tokenizer.build_inputs_with_special_tokens(_A						,		_A)

      assert encoded_sentence == [101] + text + [102]
      assert encoded_pair == [101] + text + [102] + text_a + [102]

  def 						lowerCAmelCase__     (					self):
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
              SCREAMING_SNAKE_CASE_									=						self.rust_tokenizer_class.from_pretrained(_A						,		**_A)

              SCREAMING_SNAKE_CASE_									=						f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
              SCREAMING_SNAKE_CASE_									=						tokenizer_r.encode_plus(
                  _A						,		return_attention_mask=_A						,		return_token_type_ids=_A						,		return_offsets_mapping=_A						,		add_special_tokens=_A						,		)

              SCREAMING_SNAKE_CASE_									=						tokenizer_r.do_lower_case if hasattr(_A						,		'do_lower_case') else False
              SCREAMING_SNAKE_CASE_									=						(
                  [
                      ((0, 0), tokenizer_r.cls_token),
                      ((0, 1), 'A'),
                      ((1, 2), ','),
                      ((3, 5), 'na'),
                      ((5, 6), '##ï'),
                      ((6, 8), '##ve'),
                      ((9, 15), tokenizer_r.mask_token),
                      ((16, 21), 'Allen'),
                      ((21, 23), '##NL'),
                      ((23, 24), '##P'),
                      ((25, 33), 'sentence'),
                      ((33, 34), '.'),
                      ((0, 0), tokenizer_r.sep_token),
                  ]
                  if not do_lower_case
                  else [
                      ((0, 0), tokenizer_r.cls_token),
                      ((0, 1), 'a'),
                      ((1, 2), ','),
                      ((3, 8), 'naive'),
                      ((9, 15), tokenizer_r.mask_token),
                      ((16, 21), 'allen'),
                      ((21, 23), '##nl'),
                      ((23, 24), '##p'),
                      ((25, 33), 'sentence'),
                      ((33, 34), '.'),
                      ((0, 0), tokenizer_r.sep_token),
                  ]
              )

              self.assertEqual(
                  [e[1] for e in expected_results]						,		tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
              self.assertEqual([e[0] for e in expected_results]						,		tokens['offset_mapping'])


  def 						lowerCAmelCase__     (					self):
      SCREAMING_SNAKE_CASE_									=						['的', '人', '有']
      SCREAMING_SNAKE_CASE_									=						''.join(_A)
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
              SCREAMING_SNAKE_CASE_									=						True
              SCREAMING_SNAKE_CASE_									=						self.tokenizer_class.from_pretrained(_A						,		**_A)
              SCREAMING_SNAKE_CASE_									=						self.rust_tokenizer_class.from_pretrained(_A						,		**_A)

              SCREAMING_SNAKE_CASE_									=						tokenizer_p.encode(_A						,		add_special_tokens=_A)
              SCREAMING_SNAKE_CASE_									=						tokenizer_r.encode(_A						,		add_special_tokens=_A)

              SCREAMING_SNAKE_CASE_									=						tokenizer_r.convert_ids_to_tokens(_A)
              SCREAMING_SNAKE_CASE_									=						tokenizer_p.convert_ids_to_tokens(_A)

              # it is expected that each Chinese character is not preceded by "##"
              self.assertListEqual(_A						,		_A)
              self.assertListEqual(_A						,		_A)

              SCREAMING_SNAKE_CASE_									=						False
              SCREAMING_SNAKE_CASE_									=						self.rust_tokenizer_class.from_pretrained(_A						,		**_A)
              SCREAMING_SNAKE_CASE_									=						self.tokenizer_class.from_pretrained(_A						,		**_A)

              SCREAMING_SNAKE_CASE_									=						tokenizer_r.encode(_A						,		add_special_tokens=_A)
              SCREAMING_SNAKE_CASE_									=						tokenizer_p.encode(_A						,		add_special_tokens=_A)

              SCREAMING_SNAKE_CASE_									=						tokenizer_r.convert_ids_to_tokens(_A)
              SCREAMING_SNAKE_CASE_									=						tokenizer_p.convert_ids_to_tokens(_A)

              # it is expected that only the first Chinese character is not preceded by "##".
              SCREAMING_SNAKE_CASE_									=						[
                  f"""##{token}""" if idx != 0 else token for idx, token in enumerate(_A)
              ]
              self.assertListEqual(_A						,		_A)
              self.assertListEqual(_A						,		_A)





import pytest

import datasets


# Import fixture modules as plugins
UpperCamelCase__  :      Union[str, Any]			=    ["tests.fixtures.files", "tests.fixtures.hub", "tests.fixtures.fsspec"]


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Optional[Any] ,   _SCREAMING_SNAKE_CASE					:		Dict       ):





    """simple docstring"""


    for item in items:
        if any(marker in item.keywords for marker in ['integration', 'unit']       ):
            continue
        item.add_marker(pytest.mark.unit       )


def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Optional[int]       ):





    """simple docstring"""


    config.addinivalue_line('markers' ,   'torchaudio_latest: mark test to run with torchaudio>=0.12'       )


@pytest.fixture(autouse=_SCREAMING_SNAKE_CASE       )
def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Any ,   _SCREAMING_SNAKE_CASE					:		str       ):





    """simple docstring"""


    SCREAMING_SNAKE_CASE_									=						tmp_path_factory.getbasetemp() / 'cache'
    SCREAMING_SNAKE_CASE_									=						test_hf_cache_home / 'datasets'
    SCREAMING_SNAKE_CASE_									=						test_hf_cache_home / 'metrics'
    SCREAMING_SNAKE_CASE_									=						test_hf_cache_home / 'modules'
    monkeypatch.setattr('datasets.config.HF_DATASETS_CACHE' ,   str(_SCREAMING_SNAKE_CASE       )       )
    monkeypatch.setattr('datasets.config.HF_METRICS_CACHE' ,   str(_SCREAMING_SNAKE_CASE       )       )
    monkeypatch.setattr('datasets.config.HF_MODULES_CACHE' ,   str(_SCREAMING_SNAKE_CASE       )       )
    SCREAMING_SNAKE_CASE_									=						test_hf_datasets_cache / 'downloads'
    monkeypatch.setattr('datasets.config.DOWNLOADED_DATASETS_PATH' ,   str(_SCREAMING_SNAKE_CASE       )       )
    SCREAMING_SNAKE_CASE_									=						test_hf_datasets_cache / 'downloads' / 'extracted'
    monkeypatch.setattr('datasets.config.EXTRACTED_DATASETS_PATH' ,   str(_SCREAMING_SNAKE_CASE       )       )


@pytest.fixture(autouse=_SCREAMING_SNAKE_CASE ,   scope='session'       )
def 					_UpperCAmelCase       (  ):





    """simple docstring"""


    datasets.disable_progress_bar()


@pytest.fixture(autouse=_SCREAMING_SNAKE_CASE       )
def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Any       ):





    """simple docstring"""


    monkeypatch.setattr('datasets.config.HF_UPDATE_DOWNLOAD_COUNTS' ,   _SCREAMING_SNAKE_CASE       )





@pytest.fixture
def 					_UpperCAmelCase       (  _SCREAMING_SNAKE_CASE					:		Tuple       ):





    """simple docstring"""


    monkeypatch.setattr('sqlalchemy.util.deprecations.SILENCE_UBER_WARNING' ,   _SCREAMING_SNAKE_CASE       )


