

import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
lowerCAmelCase__	   = """src/transformers"""
lowerCAmelCase__	   = """docs/source/en"""
lowerCAmelCase__	   = """."""



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Union[str, Any]	,       SCREAMING_SNAKE_CASE_:							Dict	,       SCREAMING_SNAKE_CASE_:							Dict      )			->      Union[str, Any]:




				'''simple docstring'''





				with open(SCREAMING_SNAKE_CASE_	,       "r"	,       encoding="utf-8"	,       newline="\n"      ) as f:
								A__        =			f.readlines()
				# Find the start prompt.
				A__        =			0
				while not lines[start_index].startswith(SCREAMING_SNAKE_CASE_      ):
								start_index += 1
				start_index += 1

				A__        =			start_index
				while not lines[end_index].startswith(SCREAMING_SNAKE_CASE_      ):
								end_index += 1
				end_index -= 1

				while len(lines[start_index]      ) <= 1:
								start_index += 1
				while len(lines[end_index]      ) <= 1:
								end_index -= 1
				end_index += 1
				return "".join(lines[start_index:end_index]      ), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
lowerCAmelCase__	   = """Model|Encoder|Decoder|ForConditionalGeneration"""
# Regexes that match TF/Flax/PT model names.
lowerCAmelCase__	   = re.compile(R"""TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
lowerCAmelCase__	   = re.compile(R"""Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowerCAmelCase__	   = re.compile(R"""(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")


# This is to make sure the transformers module imported is the one in the repo.
lowerCAmelCase__	   = direct_transformers_import(TRANSFORMERS_PATH)



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							List[Any]      )			->      Optional[int]:




				'''simple docstring'''





				A__        =			re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"	,       SCREAMING_SNAKE_CASE_      )
				return [m.group(0      ) for m in matches]



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							List[Any]	,       SCREAMING_SNAKE_CASE_:							List[Any]      )			->      Union[str, Any]:




				'''simple docstring'''





				A__        =			2 if text == "✅" or text == "❌" else len(SCREAMING_SNAKE_CASE_      )
				A__        =			(width - text_length) // 2
				A__        =			width - text_length - left_indent
				return " " * left_indent + text + " " * right_indent



def 					lowerCAmelCase__							(	)			->      Dict:




				'''simple docstring'''





				A__        =			transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
				A__        =			{
				    name: config_maping_names[code]
				    for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
				    if code in config_maping_names
				}
				A__        =			{name: config.replace("Config"	,       ""      ) for name, config in model_name_to_config.items()}

				# Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
				A__        =			collections.defaultdict(SCREAMING_SNAKE_CASE_      )
				A__        =			collections.defaultdict(SCREAMING_SNAKE_CASE_      )
				A__        =			collections.defaultdict(SCREAMING_SNAKE_CASE_      )
				A__        =			collections.defaultdict(SCREAMING_SNAKE_CASE_      )
				A__        =			collections.defaultdict(SCREAMING_SNAKE_CASE_      )

				# Let's lookup through all transformers object (once).
				for attr_name in dir(SCREAMING_SNAKE_CASE_      ):
								A__        =			None
								if attr_name.endswith("Tokenizer"      ):
												A__        =			slow_tokenizers
												A__        =			attr_name[:-9]
								elif attr_name.endswith("TokenizerFast"      ):
												A__        =			fast_tokenizers
												A__        =			attr_name[:-1_3]
								elif _re_tf_models.match(SCREAMING_SNAKE_CASE_      ) is not None:
												A__        =			tf_models
												A__        =			_re_tf_models.match(SCREAMING_SNAKE_CASE_      ).groups()[0]
								elif _re_flax_models.match(SCREAMING_SNAKE_CASE_      ) is not None:
												A__        =			flax_models
												A__        =			_re_flax_models.match(SCREAMING_SNAKE_CASE_      ).groups()[0]
								elif _re_pt_models.match(SCREAMING_SNAKE_CASE_      ) is not None:
												A__        =			pt_models
												A__        =			_re_pt_models.match(SCREAMING_SNAKE_CASE_      ).groups()[0]

								if lookup_dict is not None:
												while len(SCREAMING_SNAKE_CASE_      ) > 0:
																if attr_name in model_name_to_prefix.values():
																				A__        =			True
																				break
																# Try again after removing the last word in the name
																A__        =			"".join(camel_case_split(SCREAMING_SNAKE_CASE_      )[:-1]      )

    # Let's build that table!
				A__        =			list(model_name_to_config.keys()      )
				model_names.sort(key=str.lower      )
				A__        =			["Model", "Tokenizer slow", "Tokenizer fast", "PyTorch support", "TensorFlow support", "Flax Support"]
				# We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
				A__        =			[len(SCREAMING_SNAKE_CASE_      ) + 2 for c in columns]
				A__        =			max([len(SCREAMING_SNAKE_CASE_      ) for name in model_names]      ) + 2

				# Build the table per se
				A__        =			"|" + "|".join([_center_text(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ) for c, w in zip(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      )]      ) + "|\n"
				# Use ":-----:" format to center-aligned table cell texts
				table += "|" + "|".join([":" + "-" * (w - 2) + ":" for w in widths]      ) + "|\n"

				A__        =			{True: "✅", False: "❌"}
				for name in model_names:
								A__        =			model_name_to_prefix[name]
								A__        =			[
								    name,
								    check[slow_tokenizers[prefix]],
								    check[fast_tokenizers[prefix]],
								    check[pt_models[prefix]],
								    check[tf_models[prefix]],
								    check[flax_models[prefix]],
								]
								table += "|" + "|".join([_center_text(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ) for l, w in zip(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      )]      ) + "|\n"
				return table



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							Union[str, Any]=False      )			->      List[str]:




				'''simple docstring'''





				A__    ,				A__    ,				A__    ,				A__        =			_find_text_in_file(
				    filename=os.path.join(SCREAMING_SNAKE_CASE_	,       "index.md"      )	,       start_prompt="<!--This table is updated automatically from the auto modules"	,       end_prompt="<!-- End table-->"	,       )
				A__        =			get_model_table_from_auto_modules()

				if current_table != new_table:
								if overwrite:
												with open(os.path.join(SCREAMING_SNAKE_CASE_	,       "index.md"      )	,       "w"	,       encoding="utf-8"	,       newline="\n"      ) as f:
																f.writelines(lines[:start_index] + [new_table] + lines[end_index:]      )
								else:
												raise ValueError(
												    "The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this."      )


if __name__ == "__main__":
	lowerCAmelCase__	   = argparse.ArgumentParser()
	parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
	lowerCAmelCase__	   = parser.parse_args()

	check_model_table(args.fix_and_overwrite)




lowerCAmelCase__	   = """ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"""



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							bytes      )			->      bytes:




				'''simple docstring'''





				if not isinstance(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
								A__        =			F'a bytes-like object is required, not \'{data.__class__.__name__}\''
								raise TypeError(SCREAMING_SNAKE_CASE_      )

				A__        =			"".join(bin(SCREAMING_SNAKE_CASE_      )[2:].zfill(8      ) for byte in data      )

				A__        =			len(SCREAMING_SNAKE_CASE_      ) % 6 != 0

				if padding_needed:
								# The padding that will be added later
								A__        =			b"=" * ((6 - len(SCREAMING_SNAKE_CASE_      ) % 6) // 2)

								# Append binary_stream with arbitrary binary digits (0's by default) to make its
								# length a multiple of 6.
								binary_stream += "0" * (6 - len(SCREAMING_SNAKE_CASE_      ) % 6)
				else:
								A__        =			b""

				# Encode every 6 binary digits to their corresponding Base64 character
				return (
				    "".join(
				        B64_CHARSET[int(binary_stream[index : index + 6]	,       2      )]
				        for index in range(0	,       len(SCREAMING_SNAKE_CASE_      )	,       6      )      ).encode()
				    + padding
				)



def 					lowerCAmelCase__							(	SCREAMING_SNAKE_CASE_:							str      )			->      bytes:




				'''simple docstring'''





				if not isinstance(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ) and not isinstance(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
								A__        =			(
								    "argument should be a bytes-like object or ASCII string, "
								    F'not \'{encoded_data.__class__.__name__}\''
								)
								raise TypeError(SCREAMING_SNAKE_CASE_      )

				# In case encoded_data is a bytes-like object, make sure it contains only
				# ASCII characters so we convert it to a string object
				if isinstance(SCREAMING_SNAKE_CASE_	,       SCREAMING_SNAKE_CASE_      ):
								try:
												A__        =			encoded_data.decode("utf-8"      )
								except UnicodeDecodeError:
												raise ValueError("base64 encoded data should only contain ASCII characters"      )

				A__        =			encoded_data.count("="      )

				# Check if the encoded string contains non base64 characters
				if padding:
								assert all(
								    char in B64_CHARSET for char in encoded_data[:-padding]      ), "Invalid base64 character(s) found."
				else:
								assert all(
								    char in B64_CHARSET for char in encoded_data      ), "Invalid base64 character(s) found."

				# Check the padding
				assert len(SCREAMING_SNAKE_CASE_      ) % 4 == 0 and padding < 3, "Incorrect padding"

				if padding:
								# Remove padding if there is one
								A__        =			encoded_data[:-padding]

								A__        =			"".join(
								    bin(B64_CHARSET.index(SCREAMING_SNAKE_CASE_      )      )[2:].zfill(6      ) for char in encoded_data      )[: -padding * 2]
				else:
								A__        =			"".join(
								    bin(B64_CHARSET.index(SCREAMING_SNAKE_CASE_      )      )[2:].zfill(6      ) for char in encoded_data      )

				A__        =			[
				    int(binary_stream[index : index + 8]	,       2      )
				    for index in range(0	,       len(SCREAMING_SNAKE_CASE_      )	,       8      )
				]

				return bytes(SCREAMING_SNAKE_CASE_      )


if __name__ == "__main__":
	import doctest

	doctest.testmod()

