

import collections
import os
import re
from pathlib import Path


lowerCAmelCase__								=		"""src/transformers"""


# Matches is_xxx_available()
lowerCAmelCase__								=		re.compile(R"""is\_([a-z_]*)_available()""")
# Catches a one-line _import_struct = {xxx}
lowerCAmelCase__								=		re.compile(R"""^_import_structure\s+=\s+\{([^\}]+)\}""")
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
lowerCAmelCase__								=		re.compile(R"""\s+\"\S*\":\s+\[([^\]]*)\]""")
# Catches a line if not is_foo_available
lowerCAmelCase__								=		re.compile(R"""^\s*if\s+not\s+is\_[a-z_]*\_available\(\)""")
# Catches a line _import_struct["bla"].append("foo")
lowerCAmelCase__								=		re.compile(R"""^\s*_import_structure\[\"\S*\"\]\.append\(\"(\S*)\"\)""")
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
lowerCAmelCase__								=		re.compile(R"""^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]""")
# Catches a line with an object between quotes and a comma:     "MyModel",
lowerCAmelCase__								=		re.compile(R"""^\s+\"([^\"]+)\",""")
# Catches a line with objects between brackets only:    ["foo", "bar"],
lowerCAmelCase__								=		re.compile(R"""^\s+\[([^\]]+)\]""")
# Catches a line with from foo import bar, bla, boo
lowerCAmelCase__								=		re.compile(R"""\s+from\s+\S*\s+import\s+([^\(\s].*)\n""")
# Catches a line with try:
lowerCAmelCase__								=		re.compile(R"""^\s*try:""")
# Catches a line with else:
lowerCAmelCase__								=		re.compile(R"""^\s*else:""")






def 					lowerCamelCase_     (							UpperCAmelCase_							:     Dict	)						->				str:




	'''simple docstring'''




	if _re_test_backend.search(UpperCAmelCase_	) is None:
		return None
	_UpperCamelCase   :		List[Any]			=						[b[0] for b in _re_backend.findall(UpperCAmelCase_	)]
	backends.sort()
	return "_and_".join(UpperCAmelCase_	)






def 					lowerCamelCase_     (							UpperCAmelCase_							:     Union[str, Any]	)						->				int:




	'''simple docstring'''




	with open(UpperCAmelCase_							,       'r'							,       encoding='utf-8'							,       newline='\n'	) as f:
		_UpperCamelCase   :		Union[str, Any]			=						f.readlines()

	_UpperCamelCase   :		List[Any]			=						0
	while line_index < len(UpperCAmelCase_	) and not lines[line_index].startswith('_import_structure = {'	):
		line_index += 1

	# If this is a traditional init, just return.
	if line_index >= len(UpperCAmelCase_	):
		return None

	# First grab the objects without a specific backend in _import_structure
	_UpperCamelCase   :		List[Any]			=						[]
	while not lines[line_index].startswith('if TYPE_CHECKING'	) and find_backend(lines[line_index]	) is None:
		_UpperCamelCase   :		Tuple			=						lines[line_index]
		# If we have everything on a single line, let's deal with it.
		if _re_one_line_import_struct.search(UpperCAmelCase_	):
			_UpperCamelCase   :		List[Any]			=						_re_one_line_import_struct.search(UpperCAmelCase_	).groups()[0]
			_UpperCamelCase   :		Optional[Any]			=						re.findall(R'\[([^\]]+)\]'							,       UpperCAmelCase_	)
			for imp in imports:
				objects.extend([obj[1:-1] for obj in imp.split(', '	)]	)
			line_index += 1
			continue
		_UpperCamelCase   :		Any			=						_re_import_struct_key_value.search(UpperCAmelCase_	)
		if single_line_import_search is not None:
			_UpperCamelCase   :		Optional[Any]			=						[obj[1:-1] for obj in single_line_import_search.groups()[0].split(', '	) if len(UpperCAmelCase_	) > 0]
			objects.extend(UpperCAmelCase_	)
		elif line.startswith(' ' * 8 + '"'	):
			objects.append(line[9:-3]	)
		line_index += 1

	_UpperCamelCase   :		Optional[Any]			=						{'none': objects}
	# Let's continue with backend-specific objects in _import_structure
	while not lines[line_index].startswith('if TYPE_CHECKING'	):
		# If the line is an if not is_backend_available, we grab all objects associated.
		_UpperCamelCase   :		Dict			=						find_backend(lines[line_index]	)
		# Check if the backend declaration is inside a try block:
		if _re_try.search(lines[line_index - 1]	) is None:
			_UpperCamelCase   :		Optional[int]			=						None

		if backend is not None:
			line_index += 1

			# Scroll until we hit the else block of try-except-else
			while _re_else.search(lines[line_index]	) is None:
				line_index += 1

			line_index += 1

			_UpperCamelCase   :		Optional[int]			=						[]
			# Until we unindent, add backend objects to the list
			while len(lines[line_index]	) <= 1 or lines[line_index].startswith(' ' * 4	):
				_UpperCamelCase   :		Optional[int]			=						lines[line_index]
				if _re_import_struct_add_one.search(UpperCAmelCase_	) is not None:
					objects.append(_re_import_struct_add_one.search(UpperCAmelCase_	).groups()[0]	)
				elif _re_import_struct_add_many.search(UpperCAmelCase_	) is not None:
					_UpperCamelCase   :		Union[str, Any]			=						_re_import_struct_add_many.search(UpperCAmelCase_	).groups()[0].split(', '	)
					_UpperCamelCase   :		Optional[int]			=						[obj[1:-1] for obj in imports if len(UpperCAmelCase_	) > 0]
					objects.extend(UpperCAmelCase_	)
				elif _re_between_brackets.search(UpperCAmelCase_	) is not None:
					_UpperCamelCase   :		Optional[Any]			=						_re_between_brackets.search(UpperCAmelCase_	).groups()[0].split(', '	)
					_UpperCamelCase   :		Optional[Any]			=						[obj[1:-1] for obj in imports if len(UpperCAmelCase_	) > 0]
					objects.extend(UpperCAmelCase_	)
				elif _re_quote_object.search(UpperCAmelCase_	) is not None:
					objects.append(_re_quote_object.search(UpperCAmelCase_	).groups()[0]	)
				elif line.startswith(' ' * 8 + '"'	):
					objects.append(line[9:-3]	)
				elif line.startswith(' ' * 1_2 + '"'	):
					objects.append(line[1_3:-3]	)
				line_index += 1

			_UpperCamelCase   :		Optional[Any]			=						objects
		else:
			line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
	_UpperCamelCase   :		Union[str, Any]			=						[]
	while (
	    line_index < len(UpperCAmelCase_	)
	    and find_backend(lines[line_index]	) is None
	    and not lines[line_index].startswith('else'	)
	):
		_UpperCamelCase   :		Optional[int]			=						lines[line_index]
		_UpperCamelCase   :		Tuple			=						_re_import.search(UpperCAmelCase_	)
		if single_line_import_search is not None:
			objects.extend(single_line_import_search.groups()[0].split(', '	)	)
		elif line.startswith(' ' * 8	):
			objects.append(line[8:-2]	)
		line_index += 1

	_UpperCamelCase   :		int			=						{'none': objects}
	# Let's continue with backend-specific objects
	while line_index < len(UpperCAmelCase_	):
		# If the line is an if is_backend_available, we grab all objects associated.
		_UpperCamelCase   :		List[str]			=						find_backend(lines[line_index]	)
		# Check if the backend declaration is inside a try block:
		if _re_try.search(lines[line_index - 1]	) is None:
			_UpperCamelCase   :		Optional[Any]			=						None

		if backend is not None:
			line_index += 1

			# Scroll until we hit the else block of try-except-else
			while _re_else.search(lines[line_index]	) is None:
				line_index += 1

			line_index += 1

			_UpperCamelCase   :		Tuple			=						[]
			# Until we unindent, add backend objects to the list
			while len(lines[line_index]	) <= 1 or lines[line_index].startswith(' ' * 8	):
				_UpperCamelCase   :		Tuple			=						lines[line_index]
				_UpperCamelCase   :		List[Any]			=						_re_import.search(UpperCAmelCase_	)
				if single_line_import_search is not None:
					objects.extend(single_line_import_search.groups()[0].split(', '	)	)
				elif line.startswith(' ' * 1_2	):
					objects.append(line[1_2:-2]	)
				line_index += 1

			_UpperCamelCase   :		List[Any]			=						objects
		else:
			line_index += 1

	return import_dict_objects, type_hint_objects






def 					lowerCamelCase_     (							UpperCAmelCase_							:     Any							,       UpperCAmelCase_							:     Optional[Any]	)						->				Optional[Any]:




	'''simple docstring'''




	def find_duplicates(UpperCAmelCase_							:     Tuple	):
		return [k for k, v in collections.Counter(UpperCAmelCase_	).items() if v > 1]

	if list(import_dict_objects.keys()	) != list(type_hint_objects.keys()	):
		return ["Both sides of the init do not have the same backends!"]

	_UpperCamelCase   :		str			=						[]
	for key in import_dict_objects.keys():
		_UpperCamelCase   :		List[Any]			=						find_duplicates(import_dict_objects[key]	)
		if duplicate_imports:
			errors.append(F'''Duplicate _import_structure definitions for: {duplicate_imports}'''	)
		_UpperCamelCase   :		Optional[int]			=						find_duplicates(type_hint_objects[key]	)
		if duplicate_type_hints:
			errors.append(F'''Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'''	)

		if sorted(set(import_dict_objects[key]	)	) != sorted(set(type_hint_objects[key]	)	):
			_UpperCamelCase   :		Dict			=						'base imports' if key == 'none' else F'''{key} backend'''
			errors.append(F'''Differences for {name}:'''	)
			for a in type_hint_objects[key]:
				if a not in import_dict_objects[key]:
					errors.append(F'''  {a} in TYPE_HINT but not in _import_structure.'''	)
			for a in import_dict_objects[key]:
				if a not in type_hint_objects[key]:
					errors.append(F'''  {a} in _import_structure but not in TYPE_HINT.'''	)
	return errors






def 					lowerCamelCase_     (							)						->				int:




	'''simple docstring'''




	_UpperCamelCase   :		List[Any]			=						[]
	for root, _, files in os.walk(UpperCAmelCase_	):
		if "__init__.py" in files:
			_UpperCamelCase   :		List[Any]			=						os.path.join(UpperCAmelCase_							,       '__init__.py'	)
			_UpperCamelCase   :		List[str]			=						parse_init(UpperCAmelCase_	)
			if objects is not None:
				_UpperCamelCase   :		Optional[Any]			=						analyze_results(*UpperCAmelCase_	)
				if len(UpperCAmelCase_	) > 0:
					_UpperCamelCase   :		List[Any]			=						F'''Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'''
					failures.append('\n'.join(UpperCAmelCase_	)	)
	if len(UpperCAmelCase_	) > 0:
		raise ValueError('\n\n'.join(UpperCAmelCase_	)	)






def 					lowerCamelCase_     (							)						->				Any:




	'''simple docstring'''




	_UpperCamelCase   :		int			=						[]
	for path, directories, files in os.walk(UpperCAmelCase_	):
		for folder in directories:
			# Ignore private modules
			if folder.startswith('_'	):
				directories.remove(UpperCAmelCase_	)
				continue
			# Ignore leftovers from branches (empty folders apart from pycache)
			if len(list((Path(UpperCAmelCase_	) / folder).glob('*.py'	)	)	) == 0:
				continue
			_UpperCamelCase   :		str			=						str((Path(UpperCAmelCase_	) / folder).relative_to(UpperCAmelCase_	)	)
			_UpperCamelCase   :		List[str]			=						short_path.replace(os.path.sep							,       '.'	)
			submodules.append(UpperCAmelCase_	)
		for fname in files:
			if fname == "__init__.py":
				continue
			_UpperCamelCase   :		Any			=						str((Path(UpperCAmelCase_	) / fname).relative_to(UpperCAmelCase_	)	)
			_UpperCamelCase   :		Optional[int]			=						short_path.replace('.py'							,       ''	).replace(os.path.sep							,       '.'	)
			if len(submodule.split('.'	)	) == 1:
				submodules.append(UpperCAmelCase_	)
	return submodules


lowerCAmelCase__								=		[
    """convert_pytorch_checkpoint_to_tf2""",
    """modeling_flax_pytorch_utils""",
    """models.esm.openfold_utils""",
]






def 					lowerCamelCase_     (							)						->				Optional[Any]:




	'''simple docstring'''




	from transformers.utils import direct_transformers_import

	_UpperCamelCase   :		Dict			=						direct_transformers_import(UpperCAmelCase_	)

	_UpperCamelCase   :		List[str]			=						set(transformers._import_structure.keys()	)
	# This contains all the base keys of the _import_structure object defined in the init, but if the user is missing
	# some optional dependencies, they may not have all of them. Thus we read the init to read all additions and
	# (potentiall re-) add them.
	with open(os.path.join(UpperCAmelCase_							,       '__init__.py'	)							,       'r'	) as f:
		_UpperCamelCase   :		str			=						f.read()
	import_structure_keys.update(set(re.findall(R'import_structure\[\"([^\"]*)\"\]'							,       UpperCAmelCase_	)	)	)

	_UpperCamelCase   :		Any			=						[
	    module
	    for module in get_transformers_submodules()
	    if module not in IGNORE_SUBMODULES and module not in import_structure_keys
	]

	if len(UpperCAmelCase_	) > 0:
		_UpperCamelCase   :		Any			=						'\n'.join(F'''- {module}''' for module in module_not_registered	)
		raise ValueError(
		    'The following submodules are not properly registed in the main init of Transformers:\n'
		    F'''{list_of_modules}\n'''
		    'Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'	)


if __name__ == "__main__":
							check_all_inits()
							check_submodules()







import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase__								=		get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
							import sentencepiece as sp


lowerCAmelCase__								=		5
lowerCAmelCase__								=		1_0







@require_sentencepiece
@require_tokenizers
class 	lowercase    (				_lowercase    ,							unittest.TestCase							):


		"""simple docstring"""




		a__					       =  SpeechaTextTokenizer
		a__					       =  False
		a__					       =  True


		def 		A__					(				self):
			super().setUp()

			_UpperCamelCase   :		Any			=						sp.SentencePieceProcessor()
			spm_model.Load(__snake_case)
			_UpperCamelCase   :		List[str]			=						['<s>', '<pad>', '</s>', '<unk>']

			vocab += [spm_model.IdToPiece(id_) for id_ in range(len(__snake_case))]
			_UpperCamelCase   :		Dict			=						dict(zip(__snake_case	,       range(len(__snake_case))))

			_UpperCamelCase   :		Tuple			=						Path(self.tmpdirname)
			save_json(__snake_case	,       save_dir / VOCAB_FILES_NAMES['vocab_file'])
			if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
				copyfile(__snake_case	,       save_dir / VOCAB_FILES_NAMES['spm_file'])

			_UpperCamelCase   :		int			=						SpeechaTextTokenizer.from_pretrained(self.tmpdirname)
			tokenizer.save_pretrained(self.tmpdirname)


		def 		A__					(				self):
			_UpperCamelCase   :		str			=						'<pad>'
			_UpperCamelCase   :		Dict			=						1

			self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case)	,       __snake_case)
			self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case)	,       __snake_case)


		def 		A__					(				self):
			_UpperCamelCase   :		Optional[Any]			=						list(self.get_tokenizer().get_vocab().keys())

			self.assertEqual(vocab_keys[0]	,       '<s>')
			self.assertEqual(vocab_keys[1]	,       '<pad>')
			self.assertEqual(vocab_keys[-1]	,       'j')
			self.assertEqual(len(__snake_case)	,       10_01)


		def 		A__					(				self):
			self.assertEqual(self.get_tokenizer().vocab_size	,       10_01)


		def 		A__					(				self):
			_UpperCamelCase   :		Any			=						SpeechaTextTokenizer.from_pretrained(self.tmpdirname)

			_UpperCamelCase   :		List[str]			=						tokenizer.tokenize('This is a test')
			self.assertListEqual(__snake_case	,       ['▁This', '▁is', '▁a', '▁t', 'est'])

			self.assertListEqual(
			    tokenizer.convert_tokens_to_ids(__snake_case)	,       [2_89, 50, 14, 1_74, 3_86]	,       )

			_UpperCamelCase   :		int			=						tokenizer.tokenize('I was born in 92000, and this is falsé.')
			self.assertListEqual(
			    __snake_case	,       [SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '9', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', 'é', '.']	,       )
			_UpperCamelCase   :		int			=						tokenizer.convert_tokens_to_ids(__snake_case)
			self.assertListEqual(__snake_case	,       [12, 25, 88, 59, 28, 23, 11, 4, 6_06, 3_51, 3_51, 3_51, 7, 16, 70, 50, 76, 84, 10, 4, 8])

			_UpperCamelCase   :		Tuple			=						tokenizer.convert_ids_to_tokens(__snake_case)
			self.assertListEqual(
			    __snake_case	,       [SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '<unk>', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', '<unk>', '.']	,       )






		@slow
		def 		A__					(				self):
			# fmt: off
			_UpperCamelCase   :		Optional[int]			=						{'input_ids': [[37_91, 7_97, 31, 11, 64, 7_97, 31, 24_29, 4_33, 12, 11_76, 12, 20, 7_86, 9_15, 1_42, 24_13, 2_40, 37, 32_38, 7_97, 31, 11, 35, 93, 9_15, 1_42, 24_13, 2_40, 37, 55_40, 5_67, 12_76, 93, 37, 6_10, 40, 62, 4_55, 6_57, 10_42, 1_23, 7_80, 1_77, 37, 3_09, 2_41, 12_98, 5_14, 20, 2_92, 27_37, 1_14, 24_69, 2_41, 85, 64, 3_02, 5_48, 5_28, 4_23, 4, 5_09, 4_06, 4_23, 37, 6_01, 4, 7_77, 3_02, 5_48, 5_28, 4_23, 2_84, 4, 33_88, 5_11, 4_59, 4, 35_55, 40, 3_21, 3_02, 7_05, 4, 33_88, 5_11, 5_83, 3_26, 5, 5, 5, 62, 33_10, 5_60, 1_77, 26_80, 2_17, 15_08, 32, 31, 8_53, 4_18, 64, 5_83, 5_11, 16_05, 62, 35, 93, 5_60, 1_77, 26_80, 2_17, 15_08, 15_21, 64, 5_83, 5_11, 5_19, 62, 20, 15_15, 7_64, 20, 1_49, 2_61, 56_25, 79_72, 20, 55_40, 5_67, 12_76, 93, 39_25, 16_75, 11, 15, 8_02, 79_72, 5_76, 2_17, 15_08, 11, 35, 93, 12_53, 24_41, 15, 2_89, 6_52, 31, 4_16, 3_21, 38_42, 1_15, 40, 9_11, 8, 4_76, 6_19, 4, 3_80, 1_42, 4_23, 3_35, 2_40, 35, 93, 2_64, 8, 11, 3_35, 5_69, 4_20, 1_63, 5, 2], [2_60, 5_48, 5_28, 4_23, 20, 4_51, 20, 26_81, 11_53, 34_34, 20, 55_40, 37, 5_67, 1_26, 12_53, 24_41, 33_76, 4_49, 2_10, 4_31, 15_63, 1_77, 7_67, 55_40, 11, 12_03, 4_72, 11, 29_53, 6_85, 2_85, 3_64, 7_06, 11_53, 20, 67_99, 20, 28_69, 20, 44_64, 1_26, 40, 24_29, 20, 10_40, 8_66, 26_64, 4_18, 20, 3_18, 20, 17_26, 1_86, 20, 2_65, 5_22, 35, 93, 21_91, 46_34, 20, 10_40, 12, 67_99, 15, 2_28, 23_56, 1_42, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_75, 26_66, 6_84, 15_82, 11_76, 12, 6_27, 1_49, 6_19, 20, 49_02, 5_63, 11, 20, 1_49, 2_61, 34_20, 23_56, 1_74, 1_42, 47_14, 1_31, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
			# fmt: on

			self.tokenizer_integration_test_util(
			    expected_encoding=__snake_case	,       model_name='facebook/s2t-small-mustc-en-de-st'	,       revision='a14f04cf0776c02f62a8cb800cf7909e15ea23ad'	,       )







@require_sentencepiece
class 	lowercase    (				unittest.TestCase							):


		"""simple docstring"""




		a__					       =  "valhalla/s2t_mustc_multilinguial_medium"

		a__					       =  "C'est trop cool"
		a__					       =  "Esto es genial"


		@classmethod
		def 		A__					(				cls):
			_UpperCamelCase   :		SpeechaTextTokenizer			=						SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name)
			return cls


		def 		A__					(				self):
			self.assertEqual(self.tokenizer.lang_code_to_id['pt']	,       4)
			self.assertEqual(self.tokenizer.lang_code_to_id['ru']	,       6)
			self.assertEqual(self.tokenizer.lang_code_to_id['it']	,       9)
			self.assertEqual(self.tokenizer.lang_code_to_id['de']	,       11)


		def 		A__					(				self):
			self.assertEqual(self.tokenizer.vocab_size	,       1_00_00)


		def 		A__					(				self):
			self.assertIn(__snake_case	,       self.tokenizer.all_special_ids)
			_UpperCamelCase   :		Optional[int]			=						[ES_CODE, 4, 16_01, 47, 76_47, 2]
			_UpperCamelCase   :		Tuple			=						self.tokenizer.decode(__snake_case	,       skip_special_tokens=__snake_case)
			_UpperCamelCase   :		Dict			=						self.tokenizer.decode(generated_ids[1:]	,       skip_special_tokens=__snake_case)
			self.assertEqual(__snake_case	,       __snake_case)
			self.assertNotIn(self.tokenizer.eos_token	,       __snake_case)


		def 		A__					(				self):
			_UpperCamelCase   :		Any			=						'fr'
			_UpperCamelCase   :		List[Any]			=						self.tokenizer(self.french_text).input_ids
			self.assertEqual(encoded[0]	,       __snake_case)
			self.assertEqual(encoded[-1]	,       self.tokenizer.eos_token_id)






		def 		A__					(				self):
			_UpperCamelCase   :		Union[str, Any]			=						'fr'
			self.assertListEqual(self.tokenizer.prefix_tokens	,       [FR_CODE])

			_UpperCamelCase   :		List[str]			=						'es'
			self.assertListEqual(self.tokenizer.prefix_tokens	,       [ES_CODE])




