



import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__     = get_tests_dir("fixtures/test_sentencepiece.model")

if is_sentencepiece_available():
			import sentencepiece as sp


UpperCAmelCase__     = 5
UpperCAmelCase__     = 10







@require_sentencepiece
@require_tokenizers
class     __lowerCAmelCase					(    A       ,		unittest.TestCase		):
			UpperCamelCase           =  SpeechaTextTokenizer
			UpperCamelCase           =  False
			UpperCamelCase           =  True



			def 					_lowerCamelCase (		self     :						Tuple)     ->					List[str]:



						"""simple docstring"""






						super().setUp()

						_UpperCAmelCase	 =     sp.SentencePieceProcessor()
						spm_model.Load(A)
						_UpperCAmelCase	 =     ['<s>', '<pad>', '</s>', '<unk>']

						vocab += [spm_model.IdToPiece(id_) for id_ in range(len(A))]
						_UpperCAmelCase	 =     dict(zip(A				,	range(len(A))))

						_UpperCAmelCase	 =     Path(self.tmpdirname)
						save_json(A				,	save_dir / VOCAB_FILES_NAMES['vocab_file'])
						if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
									copyfile(A				,	save_dir / VOCAB_FILES_NAMES['spm_file'])

						_UpperCAmelCase	 =     SpeechaTextTokenizer.from_pretrained(self.tmpdirname)
						tokenizer.save_pretrained(self.tmpdirname)



			def 					_lowerCamelCase (		self     :						Tuple)     ->					List[str]:



						"""simple docstring"""






						_UpperCAmelCase	 =     '<pad>'
						_UpperCAmelCase	 =     1

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(A)				,	A)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(A)				,	A)



			def 					_lowerCamelCase (		self     :						Union[str, Any])     ->					str:



						"""simple docstring"""






						_UpperCAmelCase	 =     list(self.get_tokenizer().get_vocab().keys())

						self.assertEqual(vocab_keys[0]				,	'<s>')
						self.assertEqual(vocab_keys[1]				,	'<pad>')
						self.assertEqual(vocab_keys[-1]				,	'j')
						self.assertEqual(len(A)				,	10_01)



			def 					_lowerCamelCase (		self     :						Optional[int])     ->					Tuple:



						"""simple docstring"""






						self.assertEqual(self.get_tokenizer().vocab_size				,	10_01)



			def 					_lowerCamelCase (		self     :						str)     ->					Tuple:



						"""simple docstring"""






						_UpperCAmelCase	 =     SpeechaTextTokenizer.from_pretrained(self.tmpdirname)

						_UpperCAmelCase	 =     tokenizer.tokenize('This is a test')
						self.assertListEqual(A				,	['▁This', '▁is', '▁a', '▁t', 'est'])

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(A)				,	[2_89, 50, 14, 1_74, 3_86]				,	)

						_UpperCAmelCase	 =     tokenizer.tokenize('I was born in 92000, and this is falsé.')
						self.assertListEqual(
						    A				,	[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '9', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', 'é', '.']				,	)
						_UpperCAmelCase	 =     tokenizer.convert_tokens_to_ids(A)
						self.assertListEqual(A				,	[12, 25, 88, 59, 28, 23, 11, 4, 6_06, 3_51, 3_51, 3_51, 7, 16, 70, 50, 76, 84, 10, 4, 8])

						_UpperCAmelCase	 =     tokenizer.convert_ids_to_tokens(A)
						self.assertListEqual(
						    A				,	[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '<unk>', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', '<unk>', '.']				,	)



			@slow
			def 					_lowerCamelCase (		self     :						Optional[Any])     ->					Dict:



						"""simple docstring"""






						_UpperCAmelCase	 =     {'input_ids': [[37_91, 7_97, 31, 11, 64, 7_97, 31, 24_29, 4_33, 12, 11_76, 12, 20, 7_86, 9_15, 1_42, 24_13, 2_40, 37, 32_38, 7_97, 31, 11, 35, 93, 9_15, 1_42, 24_13, 2_40, 37, 55_40, 5_67, 12_76, 93, 37, 6_10, 40, 62, 4_55, 6_57, 10_42, 1_23, 7_80, 1_77, 37, 3_09, 2_41, 12_98, 5_14, 20, 2_92, 27_37, 1_14, 24_69, 2_41, 85, 64, 3_02, 5_48, 5_28, 4_23, 4, 5_09, 4_06, 4_23, 37, 6_01, 4, 7_77, 3_02, 5_48, 5_28, 4_23, 2_84, 4, 33_88, 5_11, 4_59, 4, 35_55, 40, 3_21, 3_02, 7_05, 4, 33_88, 5_11, 5_83, 3_26, 5, 5, 5, 62, 33_10, 5_60, 1_77, 26_80, 2_17, 15_08, 32, 31, 8_53, 4_18, 64, 5_83, 5_11, 16_05, 62, 35, 93, 5_60, 1_77, 26_80, 2_17, 15_08, 15_21, 64, 5_83, 5_11, 5_19, 62, 20, 15_15, 7_64, 20, 1_49, 2_61, 56_25, 79_72, 20, 55_40, 5_67, 12_76, 93, 39_25, 16_75, 11, 15, 8_02, 79_72, 5_76, 2_17, 15_08, 11, 35, 93, 12_53, 24_41, 15, 2_89, 6_52, 31, 4_16, 3_21, 38_42, 1_15, 40, 9_11, 8, 4_76, 6_19, 4, 3_80, 1_42, 4_23, 3_35, 2_40, 35, 93, 2_64, 8, 11, 3_35, 5_69, 4_20, 1_63, 5, 2], [2_60, 5_48, 5_28, 4_23, 20, 4_51, 20, 26_81, 11_53, 34_34, 20, 55_40, 37, 5_67, 1_26, 12_53, 24_41, 33_76, 4_49, 2_10, 4_31, 15_63, 1_77, 7_67, 55_40, 11, 12_03, 4_72, 11, 29_53, 6_85, 2_85, 3_64, 7_06, 11_53, 20, 67_99, 20, 28_69, 20, 44_64, 1_26, 40, 24_29, 20, 10_40, 8_66, 26_64, 4_18, 20, 3_18, 20, 17_26, 1_86, 20, 2_65, 5_22, 35, 93, 21_91, 46_34, 20, 10_40, 12, 67_99, 15, 2_28, 23_56, 1_42, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_75, 26_66, 6_84, 15_82, 11_76, 12, 6_27, 1_49, 6_19, 20, 49_02, 5_63, 11, 20, 1_49, 2_61, 34_20, 23_56, 1_74, 1_42, 47_14, 1_31, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=A				,	model_name='facebook/s2t-small-mustc-en-de-st'				,	revision='a14f04cf0776c02f62a8cb800cf7909e15ea23ad'				,	)







@require_sentencepiece
class     __lowerCAmelCase					(    unittest.TestCase		):
			UpperCamelCase           =  '''valhalla/s2t_mustc_multilinguial_medium'''

			UpperCamelCase           =  '''C\'est trop cool'''
			UpperCamelCase           =  '''Esto es genial'''



			@classmethod
			def 					_lowerCamelCase (		cls     :						List[str])     ->					Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase	 =     SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name)
						return cls



			def 					_lowerCamelCase (		self     :						Tuple)     ->					Dict:



						"""simple docstring"""






						self.assertEqual(self.tokenizer.lang_code_to_id['pt']				,	4)
						self.assertEqual(self.tokenizer.lang_code_to_id['ru']				,	6)
						self.assertEqual(self.tokenizer.lang_code_to_id['it']				,	9)
						self.assertEqual(self.tokenizer.lang_code_to_id['de']				,	11)



			def 					_lowerCamelCase (		self     :						Dict)     ->					Optional[Any]:



						"""simple docstring"""






						self.assertEqual(self.tokenizer.vocab_size				,	1_00_00)



			def 					_lowerCamelCase (		self     :						str)     ->					Tuple:



						"""simple docstring"""






						self.assertIn(A				,	self.tokenizer.all_special_ids)
						_UpperCAmelCase	 =     [ES_CODE, 4, 16_01, 47, 76_47, 2]
						_UpperCAmelCase	 =     self.tokenizer.decode(A				,	skip_special_tokens=A)
						_UpperCAmelCase	 =     self.tokenizer.decode(generated_ids[1:]				,	skip_special_tokens=A)
						self.assertEqual(A				,	A)
						self.assertNotIn(self.tokenizer.eos_token				,	A)



			def 					_lowerCamelCase (		self     :						str)     ->					List[Any]:



						"""simple docstring"""






						_UpperCAmelCase	 =     'fr'
						_UpperCAmelCase	 =     self.tokenizer(self.french_text).input_ids
						self.assertEqual(encoded[0]				,	A)
						self.assertEqual(encoded[-1]				,	self.tokenizer.eos_token_id)



			def 					_lowerCamelCase (		self     :						Optional[Any])     ->					Dict:



						"""simple docstring"""






						_UpperCAmelCase	 =     'fr'
						self.assertListEqual(self.tokenizer.prefix_tokens				,	[FR_CODE])

						_UpperCAmelCase	 =     'es'
						self.assertListEqual(self.tokenizer.prefix_tokens				,	[ES_CODE])





import logging
from pathlib import Path

import numpy as np
import pytorch_lightning as pl
import torch
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
from pytorch_lightning.utilities import rank_zero_only
from utils_rag import save_json





def 							A			(  _UpperCAmelCase       :     List[Any]					)       ->				int:







			'''simple docstring'''
			_UpperCAmelCase	 =     filter(lambda _UpperCAmelCase					: p.requires_grad							,     model.parameters()					)
			_UpperCAmelCase	 =     sum([np.prod(p.size()					) for p in model_parameters]					)
			return params


UpperCAmelCase__     = logging.getLogger(__name__)





def 							A			(  _UpperCAmelCase       :     List[Any]							,     _UpperCAmelCase       :     Union[str, Any]					)       ->				Any:







			'''simple docstring'''
			if metric == "rouge2":
						_UpperCAmelCase	 =     '{val_avg_rouge2:.4f}-{step_count}'
			elif metric == "bleu":
						_UpperCAmelCase	 =     '{val_avg_bleu:.4f}-{step_count}'
			elif metric == "em":
						_UpperCAmelCase	 =     '{val_avg_em:.4f}-{step_count}'
			elif metric == "loss":
						_UpperCAmelCase	 =     '{val_avg_loss:.4f}-{step_count}'
			else:
						raise NotImplementedError(
						    F"seq2seq callbacks only support rouge2 and bleu, got {metric}, You can make your own by adding to this"
						    ' function.'					)

			_UpperCAmelCase	 =     ModelCheckpoint(
			    dirpath=_UpperCAmelCase							,     filename=_UpperCAmelCase							,     monitor=F"val_{metric}"							,     mode='max'							,     save_top_k=1							,     every_n_epochs=1							,     )

			return checkpoint_callback





def 							A			(  _UpperCAmelCase       :     Any							,     _UpperCAmelCase       :     int					)       ->				str:







			'''simple docstring'''
			return EarlyStopping(
			    monitor=F"val_{metric}"							,     mode='min' if 'loss' in metric else 'max'							,     patience=_UpperCAmelCase							,     verbose=_UpperCAmelCase							,     )







class     __lowerCAmelCase					(    pl.Callback		):



			def 					_lowerCamelCase (		self     :						Optional[int]				,	A     :						List[Any]				,	A     :						int)     ->					Dict:



						"""simple docstring"""






						_UpperCAmelCase	 =     {F"lr_group_{i}": param['lr'] for i, param in enumerate(pl_module.trainer.optimizers[0].param_groups)}
						pl_module.logger.log_metrics(A)



			@rank_zero_only
			def 					_lowerCamelCase (		self     :						Optional[Any]				,	A     :						pl.Trainer				,	A     :						pl.LightningModule				,	A     :						str				,	A     :						int=True)     ->					None:



						"""simple docstring"""






						logger.info(F"***** {type_path} results at step {trainer.global_step:05d} *****")
						_UpperCAmelCase	 =     trainer.callback_metrics
						trainer.logger.log_metrics({k: v for k, v in metrics.items() if k not in ['log', 'progress_bar', 'preds']})
						# Log results
						_UpperCAmelCase	 =     Path(pl_module.hparams.output_dir)
						if type_path == "test":
									_UpperCAmelCase	 =     od / 'test_results.txt'
									_UpperCAmelCase	 =     od / 'test_generations.txt'
						else:
									# this never gets hit. I prefer not to save intermediate generations, and results are in metrics.json
									# If people want this it will be easy enough to add back.
									_UpperCAmelCase	 =     od / F"{type_path}_results/{trainer.global_step:05d}.txt"
									_UpperCAmelCase	 =     od / F"{type_path}_generations/{trainer.global_step:05d}.txt"
									results_file.parent.mkdir(exist_ok=A)
									generations_file.parent.mkdir(exist_ok=A)
						with open(A				,	'a+') as writer:
									for key in sorted(A):
												if key in ["log", "progress_bar", "preds"]:
															continue
												_UpperCAmelCase	 =     metrics[key]
												if isinstance(A				,	torch.Tensor):
															_UpperCAmelCase	 =     val.item()
												_UpperCAmelCase	 =     F"{key}: {val:.6f}\n"
												writer.write(A)

						if not save_generations:
									return

						if "preds" in metrics:
									_UpperCAmelCase	 =     '\n'.join(metrics['preds'])
									generations_file.open('w+').write(A)



			@rank_zero_only
			def 					_lowerCamelCase (		self     :						str				,	A     :						Optional[int]				,	A     :						List[str])     ->					Optional[Any]:



						"""simple docstring"""






						try:
									_UpperCAmelCase	 =     pl_module.model.model.num_parameters()
						except AttributeError:
									_UpperCAmelCase	 =     pl_module.model.num_parameters()

						_UpperCAmelCase	 =     count_trainable_parameters(A)
						# mp stands for million parameters
						trainer.logger.log_metrics({'n_params': npars, 'mp': npars / 1E6, 'grad_mp': n_trainable_pars / 1E6})



			@rank_zero_only
			def 					_lowerCamelCase (		self     :						Dict				,	A     :						pl.Trainer				,	A     :						pl.LightningModule)     ->					int:



						"""simple docstring"""






						save_json(pl_module.metrics				,	pl_module.metrics_save_path)
						return self._write_logs(A				,	A				,	'test')



			@rank_zero_only
			def 					_lowerCamelCase (		self     :						Tuple				,	A     :						pl.Trainer				,	A     :						str)     ->					Dict:



						"""simple docstring"""






						save_json(pl_module.metrics				,	pl_module.metrics_save_path)
						# Uncommenting this will save val generations
						# return self._write_logs(trainer, pl_module, "valid")
