import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFAutoModel, is_tensorflow_text_available, is_tf_available
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.testing_utils import require_tensorflow_text, require_tf, slow


if is_tf_available():
    import tensorflow as tf

if is_tensorflow_text_available():
    from transformers.models.bert import TFBertTokenizer


_lowercase							:      Optional[int]      							=['''bert-base-uncased''', '''bert-base-cased''']
_lowercase							:      Any      							='''hf-internal-testing/tiny-bert-tf-only'''

if is_tf_available():



    class  SCREAMING_SNAKE_CASE_ (					tf.keras.Model			):
      '''simple docstring'''


      def __init__(				self    :					Dict      ,	SCREAMING_SNAKE_CASE__    :					Any     )    ->				List[str]:
             super().__init__()
             A						:     Optional[Any]         =tokenizer
             A						:     Any         =AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE__     )
             A						:     int         =TFAutoModel.from_config(SCREAMING_SNAKE_CASE__     )







      def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					Optional[int]     )    ->				List[Any]:
             A						:     List[str]         =self.tokenizer(SCREAMING_SNAKE_CASE__     )
             A						:     Tuple         =self.bert(**SCREAMING_SNAKE_CASE__     )
             return out["pooler_output"]



@require_tf
@require_tensorflow_text
class  SCREAMING_SNAKE_CASE_ (					unittest.TestCase			):
  '''simple docstring'''


  def 		SCREAMING_SNAKE_CASE_		(				self    :					str     )    ->				Tuple:
         super().setUp()

         A						:     List[Any]         =[
             BertTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__     ) for checkpoint in (TOKENIZER_CHECKPOINTS * 2)
         ]  # repeat for when fast_bert_tokenizer=false
         A						:     Dict         =[TFBertTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__     ) for checkpoint in TOKENIZER_CHECKPOINTS] + [
             TFBertTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__      ,	use_fast_bert_tokenizer=SCREAMING_SNAKE_CASE__     )
             for checkpoint in TOKENIZER_CHECKPOINTS
         ]
         assert len(self.tokenizers     ) == len(self.tf_tokenizers     )

         A						:     Union[str, Any]         =[
             'This is a straightforward English test sentence.',
             'This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.',
             'Now we\'re going to add some Chinese: 一 二 三 一二三',
             'And some much more rare Chinese: 齉 堃 齉堃',
             'Je vais aussi écrire en français pour tester les accents',
             'Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ',
         ]
         A						:     str         =list(zip(self.test_sentences      ,	self.test_sentences[::-1]     )     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Any     )    ->				List[Any]:
         for tokenizer, tf_tokenizer in zip(self.tokenizers      ,	self.tf_tokenizers     ):
                for test_inputs in (self.test_sentences, self.paired_sentences):
                       A						:     Union[str, Any]         =tokenizer(SCREAMING_SNAKE_CASE__      ,	return_tensors='tf'      ,	padding='longest'     )
                       A						:     Dict         =tf_tokenizer(SCREAMING_SNAKE_CASE__     )

                       for key in python_outputs.keys():
                              self.assertTrue(tf.reduce_all(python_outputs[key].shape == tf_outputs[key].shape     )     )
                              self.assertTrue(tf.reduce_all(tf.cast(python_outputs[key]      ,	tf.intaa     ) == tf_outputs[key]     )     )







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Union[str, Any]     )    ->				Any:
         for tf_tokenizer in self.tf_tokenizers:
                A						:     List[str]         =tf_tokenizer(self.paired_sentences     )
                A						:     Tuple         =tf_tokenizer(
                    text=[sentence[0] for sentence in self.paired_sentences]      ,	text_pair=[sentence[1] for sentence in self.paired_sentences]      ,	)
                for key in merged_outputs.keys():
                       self.assertTrue(tf.reduce_all(tf.cast(merged_outputs[key]      ,	tf.intaa     ) == separated_outputs[key]     )     )







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					int     )    ->				str:
         for tf_tokenizer in self.tf_tokenizers:
                A						:     Union[str, Any]         =tf.function(SCREAMING_SNAKE_CASE__     )
                for test_inputs in (self.test_sentences, self.paired_sentences):
                       A						:     Union[str, Any]         =tf.constant(SCREAMING_SNAKE_CASE__     )
                       A						:     Union[str, Any]         =compiled_tokenizer(SCREAMING_SNAKE_CASE__     )
                       A						:     int         =tf_tokenizer(SCREAMING_SNAKE_CASE__     )

                       for key in eager_outputs.keys():
                              self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]     )     )







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Tuple     )    ->				Any:
         for tf_tokenizer in self.tf_tokenizers:
                A						:     Tuple         =ModelToSave(tokenizer=SCREAMING_SNAKE_CASE__     )
                A						:     List[str]         =tf.convert_to_tensor(self.test_sentences     )
                A						:     Union[str, Any]         =model(SCREAMING_SNAKE_CASE__     )  # Build model with some sample inputs
                with TemporaryDirectory() as tempdir:
                       A						:     int         =Path(SCREAMING_SNAKE_CASE__     ) / 'saved.model'
                       model.save(SCREAMING_SNAKE_CASE__     )
                       A						:     Optional[Any]         =tf.keras.models.load_model(SCREAMING_SNAKE_CASE__     )
                A						:     List[Any]         =loaded_model(SCREAMING_SNAKE_CASE__     )
                # We may see small differences because the loaded model is compiled, so we need an epsilon for the test
                self.assertLessEqual(tf.reduce_max(tf.abs(out - loaded_output     )     )      ,	1e-5     )







import collections
import importlib.util
import os
import re
from pathlib import Path


_lowercase							:      List[str]      							='''src/transformers'''


# Matches is_xxx_available()
_lowercase							:      Dict      							=re.compile(R'''is\_([a-z_]*)_available()''')
# Catches a one-line _import_struct = {xxx}
_lowercase							:      List[Any]      							=re.compile(R'''^_import_structure\s+=\s+\{([^\}]+)\}''')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
_lowercase							:      Tuple      							=re.compile(R'''\s+"\S*":\s+\[([^\]]*)\]''')
# Catches a line if not is_foo_available
_lowercase							:      Dict      							=re.compile(R'''^\s*if\s+not\s+is\_[a-z_]*\_available\(\)''')
# Catches a line _import_struct["bla"].append("foo")
_lowercase							:      List[Any]      							=re.compile(R'''^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)''')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
_lowercase							:      str      							=re.compile(R'''^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]''')
# Catches a line with an object between quotes and a comma:     "MyModel",
_lowercase							:      Optional[int]      							=re.compile('''^\s+"([^"]+)",''')
# Catches a line with objects between brackets only:    ["foo", "bar"],
_lowercase							:      Any      							=re.compile('''^\s+\[([^\]]+)\]''')
# Catches a line with from foo import bar, bla, boo
_lowercase							:      List[Any]      							=re.compile(R'''\s+from\s+\S*\s+import\s+([^\(\s].*)\n''')
# Catches a line with try:
_lowercase							:      Optional[Any]      							=re.compile(R'''^\s*try:''')
# Catches a line with else:
_lowercase							:      List[Any]      							=re.compile(R'''^\s*else:''')



def A__  (			lowercase:     Dict					)				->		int:
       if _re_test_backend.search(lowercase					) is None:
              return None
       A						:     Any         =[b[0] for b in _re_backend.findall(lowercase					)]
       backends.sort()
       return "_and_".join(lowercase					)



def A__  (			lowercase:     Any					)				->		List[Any]:
       with open(lowercase,		'r',		encoding='utf-8',		newline='\n'					) as f:
              A						:     Optional[Any]         =f.readlines()

       A						:     Dict         =0
       while line_index < len(lowercase					) and not lines[line_index].startswith('_import_structure = {'					):
              line_index += 1

       # If this is a traditional init, just return.
       if line_index >= len(lowercase					):
              return None

       # First grab the objects without a specific backend in _import_structure
       A						:     Optional[int]         =[]
       while not lines[line_index].startswith('if TYPE_CHECKING'					) and find_backend(lines[line_index]					) is None:
              A						:     int         =lines[line_index]
              # If we have everything on a single line, let's deal with it.
              if _re_one_line_import_struct.search(lowercase					):
                     A						:     int         =_re_one_line_import_struct.search(lowercase					).groups()[0]
                     A						:     int         =re.findall('\[([^\]]+)\]',		lowercase					)
                     for imp in imports:
                            objects.extend([obj[1:-1] for obj in imp.split(', '					)]					)
                     line_index += 1
                     continue
              A						:     Optional[int]         =_re_import_struct_key_value.search(lowercase					)
              if single_line_import_search is not None:
                     A						:     Dict         =[obj[1:-1] for obj in single_line_import_search.groups()[0].split(', '					) if len(lowercase					) > 0]
                     objects.extend(lowercase					)
              elif line.startswith(' ' * 8 + '"'					):
                     objects.append(line[9:-3]					)
              line_index += 1

       A						:     str         ={'none': objects}
       # Let's continue with backend-specific objects in _import_structure
       while not lines[line_index].startswith('if TYPE_CHECKING'					):
              # If the line is an if not is_backend_available, we grab all objects associated.
              A						:     Optional[int]         =find_backend(lines[line_index]					)
              # Check if the backend declaration is inside a try block:
              if _re_try.search(lines[line_index - 1]					) is None:
                     A						:     str         =None

              if backend is not None:
                     line_index += 1

                     # Scroll until we hit the else block of try-except-else
                     while _re_else.search(lines[line_index]					) is None:
                            line_index += 1

                     line_index += 1

                     A						:     List[str]         =[]
                     # Until we unindent, add backend objects to the list
                     while len(lines[line_index]					) <= 1 or lines[line_index].startswith(' ' * 4					):
                            A						:     Optional[Any]         =lines[line_index]
                            if _re_import_struct_add_one.search(lowercase					) is not None:
                                   objects.append(_re_import_struct_add_one.search(lowercase					).groups()[0]					)
                            elif _re_import_struct_add_many.search(lowercase					) is not None:
                                   A						:     Optional[Any]         =_re_import_struct_add_many.search(lowercase					).groups()[0].split(', '					)
                                   A						:     int         =[obj[1:-1] for obj in imports if len(lowercase					) > 0]
                                   objects.extend(lowercase					)
                            elif _re_between_brackets.search(lowercase					) is not None:
                                   A						:     Optional[int]         =_re_between_brackets.search(lowercase					).groups()[0].split(', '					)
                                   A						:     Optional[int]         =[obj[1:-1] for obj in imports if len(lowercase					) > 0]
                                   objects.extend(lowercase					)
                            elif _re_quote_object.search(lowercase					) is not None:
                                   objects.append(_re_quote_object.search(lowercase					).groups()[0]					)
                            elif line.startswith(' ' * 8 + '"'					):
                                   objects.append(line[9:-3]					)
                            elif line.startswith(' ' * 12 + '"'					):
                                   objects.append(line[13:-3]					)
                            line_index += 1

                     A						:     Optional[Any]         =objects
              else:
                     line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
       A						:     Optional[Any]         =[]
       while (
           line_index < len(lowercase					)
           and find_backend(lines[line_index]					) is None
           and not lines[line_index].startswith('else'					)
       ):
              A						:     Any         =lines[line_index]
              A						:     Optional[int]         =_re_import.search(lowercase					)
              if single_line_import_search is not None:
                     objects.extend(single_line_import_search.groups()[0].split(', '					)					)
              elif line.startswith(' ' * 8					):
                     objects.append(line[8:-2]					)
              line_index += 1

       A						:     Optional[Any]         ={'none': objects}
       # Let's continue with backend-specific objects
       while line_index < len(lowercase					):
              # If the line is an if is_backend_available, we grab all objects associated.
              A						:     str         =find_backend(lines[line_index]					)
              # Check if the backend declaration is inside a try block:
              if _re_try.search(lines[line_index - 1]					) is None:
                     A						:     Optional[Any]         =None

              if backend is not None:
                     line_index += 1

                     # Scroll until we hit the else block of try-except-else
                     while _re_else.search(lines[line_index]					) is None:
                            line_index += 1

                     line_index += 1

                     A						:     List[str]         =[]
                     # Until we unindent, add backend objects to the list
                     while len(lines[line_index]					) <= 1 or lines[line_index].startswith(' ' * 8					):
                            A						:     Any         =lines[line_index]
                            A						:     Any         =_re_import.search(lowercase					)
                            if single_line_import_search is not None:
                                   objects.extend(single_line_import_search.groups()[0].split(', '					)					)
                            elif line.startswith(' ' * 12					):
                                   objects.append(line[12:-2]					)
                            line_index += 1

                     A						:     Dict         =objects
              else:
                     line_index += 1

       return import_dict_objects, type_hint_objects



def A__  (			lowercase:     Any,		lowercase:     int					)				->		Dict:

       def find_duplicates(lowercase:     List[str]					):
              return [k for k, v in collections.Counter(lowercase					).items() if v > 1]

       if list(import_dict_objects.keys()					) != list(type_hint_objects.keys()					):
              return ["Both sides of the init do not have the same backends!"]

       A						:     List[Any]         =[]
       for key in import_dict_objects.keys():
              A						:     List[Any]         =find_duplicates(import_dict_objects[key]					)
              if duplicate_imports:
                     errors.append(F'Duplicate _import_structure definitions for: {duplicate_imports}'					)
              A						:     Tuple         =find_duplicates(type_hint_objects[key]					)
              if duplicate_type_hints:
                     errors.append(F'Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'					)

              if sorted(set(import_dict_objects[key]					)					) != sorted(set(type_hint_objects[key]					)					):
                     A						:     Tuple         ='base imports' if key == 'none' else F'{key} backend'
                     errors.append(F'Differences for {name}:'					)
                     for a in type_hint_objects[key]:
                            if a not in import_dict_objects[key]:
                                   errors.append(F'  {a} in TYPE_HINT but not in _import_structure.'					)
                     for a in import_dict_objects[key]:
                            if a not in type_hint_objects[key]:
                                   errors.append(F'  {a} in _import_structure but not in TYPE_HINT.'					)
       return errors



def A__  (			)				->		List[str]:
       A						:     Dict         =[]
       for root, _, files in os.walk(lowercase					):
              if "__init__.py" in files:
                     A						:     Any         =os.path.join(lowercase,		'__init__.py'					)
                     A						:     Union[str, Any]         =parse_init(lowercase					)
                     if objects is not None:
                            A						:     str         =analyze_results(*lowercase					)
                            if len(lowercase					) > 0:
                                   A						:     Any         =F'Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'
                                   failures.append('\n'.join(lowercase					)					)
       if len(lowercase					) > 0:
              raise ValueError('\n\n'.join(lowercase					)					)



def A__  (			)				->		int:
       A						:     List[str]         =[]
       for path, directories, files in os.walk(lowercase					):
              for folder in directories:
                     # Ignore private modules
                     if folder.startswith('_'					):
                            directories.remove(lowercase					)
                            continue
                     # Ignore leftovers from branches (empty folders apart from pycache)
                     if len(list((Path(lowercase					) / folder).glob('*.py'					)					)					) == 0:
                            continue
                     A						:     Any         =str((Path(lowercase					) / folder).relative_to(lowercase					)					)
                     A						:     List[str]         =short_path.replace(os.path.sep,		'.'					)
                     submodules.append(lowercase					)
              for fname in files:
                     if fname == "__init__.py":
                            continue
                     A						:     Optional[Any]         =str((Path(lowercase					) / fname).relative_to(lowercase					)					)
                     A						:     Dict         =short_path.replace('.py',		''					).replace(os.path.sep,		'.'					)
                     if len(submodule.split('.'					)					) == 1:
                            submodules.append(lowercase					)
       return submodules


_lowercase							:      Tuple      							=[
    '''convert_pytorch_checkpoint_to_tf2''',
    '''modeling_flax_pytorch_utils''',
]



def A__  (			)				->		Tuple:
       # This is to make sure the transformers module imported is the one in the repo.
       A						:     str         =importlib.util.spec_from_file_location(
           'transformers',		os.path.join(lowercase,		'__init__.py'					),		submodule_search_locations=[PATH_TO_TRANSFORMERS],		)
       A						:     Any         =spec.loader.load_module()

       A						:     Any         =[
           module
           for module in get_transformers_submodules()
           if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
       ]
       if len(lowercase					) > 0:
              A						:     Dict         ='\n'.join(F'- {module}' for module in module_not_registered					)
              raise ValueError(
                  'The following submodules are not properly registered in the main init of Transformers:\n'
                  F'{list_of_modules}\n'
                  'Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'					)


if __name__ == "__main__":
    check_all_inits()
    check_submodules()






