

'''simple docstring'''






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class     lowercase_			(_a       , unittest.TestCase   ):




  """simple docstring"""




  SCREAMING_SNAKE_CASE					:						Tuple           =    RoCBertTokenizer
  SCREAMING_SNAKE_CASE					:						Dict           =    None
  SCREAMING_SNAKE_CASE					:						Optional[Any]           =    False
  SCREAMING_SNAKE_CASE					:						Union[str, Any]           =    True
  SCREAMING_SNAKE_CASE					:						List[Any]           =    filter_non_english






  def       SCREAMING_SNAKE_CASE   (   self     :  Optional[Any]    ):
   super().setUp()

   __lowercase													=				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
   __lowercase													=				{}
   __lowercase													=				{}
   for i, value in enumerate(_A    ):
    __lowercase													=				i
    __lowercase													=				i
   __lowercase													=				os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['''vocab_file''']    )
   __lowercase													=				os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['''word_shape_file''']    )
   __lowercase													=				os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES['''word_pronunciation_file''']    )
   with open(self.vocab_file      ,'''w'''      ,encoding='''utf-8'''    ) as vocab_writer:
    vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]    )    )
   with open(self.word_shape_file      ,'''w'''      ,encoding='''utf-8'''    ) as word_shape_writer:
    json.dump(_A      ,_A      ,ensure_ascii=_A    )
   with open(self.word_pronunciation_file      ,'''w'''      ,encoding='''utf-8'''    ) as word_pronunciation_writer:
    json.dump(_A      ,_A      ,ensure_ascii=_A    )






  def       SCREAMING_SNAKE_CASE   (   self     :  int    ):
   __lowercase													=				self.tokenizer_class(self.vocab_file      ,self.word_shape_file      ,self.word_pronunciation_file    )

   __lowercase													=				tokenizer.tokenize('''你好[SEP]你是谁'''    )
   self.assertListEqual(_A      ,['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']    )
   self.assertListEqual(tokenizer.convert_tokens_to_ids(_A    )      ,[5, 6, 2, 5, 7, 8]    )
   self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_A    )      ,[5, 6, 2, 5, 7, 8]    )
   self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_A    )      ,[5, 6, 2, 5, 7, 8]    )






  def       SCREAMING_SNAKE_CASE   (   self     :  List[str]    ):
   __lowercase													=				RoCBertBasicTokenizer()

   self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''    )      ,['''ah''', '''\u535A''', '''\u63A8''', '''zz''']    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Any    ):
   __lowercase													=				RoCBertBasicTokenizer(do_lower_case=_A    )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )      ,['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )      ,['''hello''']    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Any    ):
   __lowercase													=				RoCBertBasicTokenizer(do_lower_case=_A      ,strip_accents=_A    )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )      ,['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )      ,['''h\u00E9llo''']    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Any    ):
   __lowercase													=				RoCBertBasicTokenizer(do_lower_case=_A      ,strip_accents=_A    )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )      ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )      ,['''hello''']    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Union[str, Any]    ):
   __lowercase													=				RoCBertBasicTokenizer(do_lower_case=_A    )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )      ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']    )
   self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''    )      ,['''hello''']    )






  def       SCREAMING_SNAKE_CASE   (   self     :  List[str]    ):
   __lowercase													=				RoCBertBasicTokenizer(do_lower_case=_A    )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''    )      ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Dict    ):
   __lowercase													=				RoCBertBasicTokenizer(do_lower_case=_A      ,strip_accents=_A    )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )      ,['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Tuple    ):
   __lowercase													=				RoCBertBasicTokenizer(do_lower_case=_A      ,strip_accents=_A    )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''    )      ,['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Dict    ):
   __lowercase													=				RoCBertBasicTokenizer(do_lower_case=_A      ,never_split=['''[UNK]''']    )

   self.assertListEqual(
       tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''    )      ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']    )






  def       SCREAMING_SNAKE_CASE   (   self     :  List[Any]    ):
   __lowercase													=				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

   __lowercase													=				{}
   for i, token in enumerate(_A    ):
    __lowercase													=				i
   __lowercase													=				RoCBertWordpieceTokenizer(vocab=_A      ,unk_token='''[UNK]'''    )

   self.assertListEqual(tokenizer.tokenize(''''''    )      ,[]    )

   self.assertListEqual(tokenizer.tokenize('''unwanted running'''    )      ,['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']    )

   self.assertListEqual(tokenizer.tokenize('''unwantedX running'''    )      ,['''[UNK]''', '''runn''', '''##ing''']    )






  def       SCREAMING_SNAKE_CASE   (   self     :  List[Any]    ):
   self.assertTrue(_is_whitespace(''' '''    )    )
   self.assertTrue(_is_whitespace('''\t'''    )    )
   self.assertTrue(_is_whitespace('''\r'''    )    )
   self.assertTrue(_is_whitespace('''\n'''    )    )
   self.assertTrue(_is_whitespace('''\u00A0'''    )    )

   self.assertFalse(_is_whitespace('''A'''    )    )
   self.assertFalse(_is_whitespace('''-'''    )    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Optional[int]    ):
   self.assertTrue(_is_control('''\u0005'''    )    )

   self.assertFalse(_is_control('''A'''    )    )
   self.assertFalse(_is_control(''' '''    )    )
   self.assertFalse(_is_control('''\t'''    )    )
   self.assertFalse(_is_control('''\r'''    )    )






  def       SCREAMING_SNAKE_CASE   (   self     :  str    ):
   self.assertTrue(_is_punctuation('''-'''    )    )
   self.assertTrue(_is_punctuation('''$'''    )    )
   self.assertTrue(_is_punctuation('''`'''    )    )
   self.assertTrue(_is_punctuation('''.'''    )    )

   self.assertFalse(_is_punctuation('''A'''    )    )
   self.assertFalse(_is_punctuation(''' '''    )    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Any    ):
   __lowercase													=				self.get_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(_A    ) for t in ['''Test''', '''\xad''', '''test''']]      ,[['''[UNK]'''], [], ['''[UNK]''']]    )

   if self.test_rust_tokenizer:
    __lowercase													=				self.get_rust_tokenizer()
    self.assertListEqual(
        [rust_tokenizer.tokenize(_A    ) for t in ['''Test''', '''\xad''', '''test''']]      ,[['''[UNK]'''], [], ['''[UNK]''']]    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Tuple    ):
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
     __lowercase													=				self.rust_tokenizer_class.from_pretrained(_A      ,**_A    )

     __lowercase													=				F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
     __lowercase													=				tokenizer_r.encode_plus(
         _A      ,return_attention_mask=_A      ,return_token_type_ids=_A      ,return_offsets_mapping=_A      ,add_special_tokens=_A      ,)

     __lowercase													=				tokenizer_r.do_lower_case if hasattr(_A      ,'''do_lower_case'''    ) else False
     __lowercase													=				(
         [
             ((0, 0), tokenizer_r.cls_token),
             ((0, 1), '''A'''),
             ((1, 2), ''','''),
             ((3, 5), '''na'''),
             ((5, 6), '''##ï'''),
             ((6, 8), '''##ve'''),
             ((9, 1_5), tokenizer_r.mask_token),
             ((1_6, 2_1), '''Allen'''),
             ((2_1, 2_3), '''##NL'''),
             ((2_3, 2_4), '''##P'''),
             ((2_5, 3_3), '''sentence'''),
             ((3_3, 3_4), '''.'''),
             ((0, 0), tokenizer_r.sep_token),
         ]
         if not do_lower_case
         else [
             ((0, 0), tokenizer_r.cls_token),
             ((0, 1), '''a'''),
             ((1, 2), ''','''),
             ((3, 8), '''naive'''),
             ((9, 1_5), tokenizer_r.mask_token),
             ((1_6, 2_1), '''allen'''),
             ((2_1, 2_3), '''##nl'''),
             ((2_3, 2_4), '''##p'''),
             ((2_5, 3_3), '''sentence'''),
             ((3_3, 3_4), '''.'''),
             ((0, 0), tokenizer_r.sep_token),
         ]
     )

     self.assertEqual(
         [e[1] for e in expected_results]      ,tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']    )    )
     self.assertEqual([e[0] for e in expected_results]      ,tokens['''offset_mapping''']    )






  def       SCREAMING_SNAKE_CASE   (   self     :  Tuple    ):
   __lowercase													=				['''的''', '''人''', '''有''']
   __lowercase													=				''''''.join(_A    )
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
     __lowercase													=				True
     __lowercase													=				self.tokenizer_class.from_pretrained(_A      ,**_A    )
     __lowercase													=				self.rust_tokenizer_class.from_pretrained(_A      ,**_A    )

     __lowercase													=				tokenizer_p.encode(_A      ,add_special_tokens=_A    )
     __lowercase													=				tokenizer_r.encode(_A      ,add_special_tokens=_A    )

     __lowercase													=				tokenizer_r.convert_ids_to_tokens(_A    )
     __lowercase													=				tokenizer_p.convert_ids_to_tokens(_A    )

     # it is expected that each Chinese character is not preceded by "##"
     self.assertListEqual(_A      ,_A    )
     self.assertListEqual(_A      ,_A    )

     __lowercase													=				False
     __lowercase													=				self.rust_tokenizer_class.from_pretrained(_A      ,**_A    )
     __lowercase													=				self.tokenizer_class.from_pretrained(_A      ,**_A    )

     __lowercase													=				tokenizer_r.encode(_A      ,add_special_tokens=_A    )
     __lowercase													=				tokenizer_p.encode(_A      ,add_special_tokens=_A    )

     __lowercase													=				tokenizer_r.convert_ids_to_tokens(_A    )
     __lowercase													=				tokenizer_p.convert_ids_to_tokens(_A    )

     # it is expected that only the first Chinese character is not preceded by "##".
     __lowercase													=				[
         F"##{token}" if idx != 0 else token for idx, token in enumerate(_A    )
     ]
     self.assertListEqual(_A      ,_A    )
     self.assertListEqual(_A      ,_A    )






  @slow
  def       SCREAMING_SNAKE_CASE   (   self     :  str    ):
   __lowercase													=				self.tokenizer_class(self.vocab_file      ,self.word_shape_file      ,self.word_pronunciation_file    )

   __lowercase													=				tokenizer.encode('''你好'''      ,add_special_tokens=_A    )
   __lowercase													=				tokenizer.encode('''你是谁'''      ,add_special_tokens=_A    )

   __lowercase													=				tokenizer.build_inputs_with_special_tokens(_A    )
   __lowercase													=				tokenizer.build_inputs_with_special_tokens(_A      ,_A    )

   assert encoded_sentence == [1] + text + [2]
   assert encoded_pair == [1] + text + [2] + text_a + [2]






  def       SCREAMING_SNAKE_CASE   (   self     :  List[Any]    ):
   __lowercase													=				self.get_tokenizers(do_lower_case=_A    )
   for tokenizer in tokenizers:
    with self.subTest(F"{tokenizer.__class__.__name__}"    ):
     __lowercase													=				'''你好，你是谁'''
     __lowercase													=				tokenizer.tokenize(_A    )
     __lowercase													=				tokenizer.convert_tokens_to_ids(_A    )
     __lowercase													=				tokenizer.convert_tokens_to_shape_ids(_A    )
     __lowercase													=				tokenizer.convert_tokens_to_pronunciation_ids(_A    )
     __lowercase													=				tokenizer.prepare_for_model(
         _A      ,_A      ,_A      ,add_special_tokens=_A    )

     __lowercase													=				tokenizer.encode_plus(_A      ,add_special_tokens=_A    )

     self.assertEqual(_A      ,_A    )





'''simple docstring'''






import re


def 	_A  (					A__   ):






 """simple docstring"""



 __lowercase													=				re.compile(
     R'''^(?:0|94|\+94|0{2}94)''' R'''7(0|1|2|4|5|6|7|8)''' R'''(-| |)''' R'''\d{7}$'''   )

 return bool(re.search(A__  ,       A__   )   )


if __name__ == "__main__":
 lowerCAmelCase__      		=			'''0094702343221'''

 print(is_sri_lankan_phone_number(phone))

