import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


_lowercase							:      int      							=get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_torch_available():
    from transformers.models.mbart.modeling_mbart import shift_tokens_right

_lowercase							:      Union[str, Any]      							=2_5_0_0_0_4
_lowercase							:      Any      							=2_5_0_0_2_0



@require_sentencepiece
@require_tokenizers
class  SCREAMING_SNAKE_CASE_ (					lowerCAmelCase_   ,						unittest.TestCase			):
  '''simple docstring'''


  lowercase		:     Any            =				MBartaaTokenizer
  lowercase		:     Union[str, Any]            =				MBartaaTokenizerFast
  lowercase		:     Dict            =				True
  lowercase		:     str            =				True







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Union[str, Any]     )    ->				str:
         super().setUp()

         # We have a SentencePiece fixture for testing
         A						:     str         =MBartaaTokenizer(SCREAMING_SNAKE_CASE__      ,	src_lang='en_XX'      ,	tgt_lang='ro_RO'      ,	keep_accents=SCREAMING_SNAKE_CASE__     )
         tokenizer.save_pretrained(self.tmpdirname     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				int:
         A						:     Optional[int]         ='<s>'
         A						:     str         =0

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE__     )      ,	SCREAMING_SNAKE_CASE__     )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE__     )      ,	SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]     )    ->				int:
         A						:     Optional[Any]         =list(self.get_tokenizer().get_vocab().keys()     )

         self.assertEqual(vocab_keys[0]      ,	'<s>'     )
         self.assertEqual(vocab_keys[1]      ,	'<pad>'     )
         self.assertEqual(vocab_keys[-1]      ,	'<mask>'     )
         self.assertEqual(len(SCREAMING_SNAKE_CASE__     )      ,	10_54     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Union[str, Any]     )    ->				Tuple:
         self.assertEqual(self.get_tokenizer().vocab_size      ,	10_54     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[int]     )    ->				Tuple:
         A						:     List[Any]         =MBartaaTokenizer(SCREAMING_SNAKE_CASE__      ,	src_lang='en_XX'      ,	tgt_lang='ro_RO'      ,	keep_accents=SCREAMING_SNAKE_CASE__     )

         A						:     str         =tokenizer.tokenize('This is a test'     )
         self.assertListEqual(SCREAMING_SNAKE_CASE__      ,	['▁This', '▁is', '▁a', '▁t', 'est']     )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__     )      ,	[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]      ,	)

         A						:     List[str]         =tokenizer.tokenize('I was born in 92000, and this is falsé.'     )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE__      ,	[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '9', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', 'é', '.']      ,	)
         A						:     Dict         =tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE__      ,	[
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
             ]      ,	)

         A						:     Optional[int]         =tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__     )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE__      ,	[SPIECE_UNDERLINE + 'I', SPIECE_UNDERLINE + 'was', SPIECE_UNDERLINE + 'b', 'or', 'n', SPIECE_UNDERLINE + 'in', SPIECE_UNDERLINE + '', '<unk>', '2', '0', '0', '0', ',', SPIECE_UNDERLINE + 'and', SPIECE_UNDERLINE + 'this', SPIECE_UNDERLINE + 'is', SPIECE_UNDERLINE + 'f', 'al', 's', '<unk>', '.']      ,	)







  @slow
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Any     )    ->				str:
         # fmt: off
         A						:     Union[str, Any]         ={'input_ids': [[25_00_04, 1_10_62, 8_27_72, 7, 15, 8_27_72, 5_38, 5_15_29, 2_37, 1_71_98, 12_90, 2_06, 9, 21_51_75, 13_14, 1_36, 1_71_98, 12_90, 2_06, 9, 5_63_59, 42, 12_20_09, 9, 1_64_66, 16, 8_73_44, 45_37, 9, 47_17, 7_83_81, 6, 15_99_58, 7, 15, 2_44_80, 6_18, 4, 5_27, 2_26_93, 54_28, 4, 27_77, 2_44_80, 98_74, 4, 4_35_23, 5_94, 4, 8_03, 1_83_92, 3_31_89, 18, 4, 4_35_23, 2_44_47, 1_23_99, 1_00, 2_49_55, 8_36_58, 96_26, 14_40_57, 15, 8_39, 2_23_35, 16, 1_36, 2_49_55, 8_36_58, 8_34_79, 15, 3_91_02, 7_24, 16, 6_78, 6_45, 27_89, 13_28, 45_89, 42, 12_20_09, 11_57_74, 23, 8_05, 13_28, 4_68_76, 7, 1_36, 5_38_94, 19_40, 4_22_27, 4_11_59, 1_77_21, 8_23, 4_25, 4, 2_75_12, 9_87_22, 2_06, 1_36, 55_31, 49_70, 9_19, 1_73_36, 5, 2], [25_00_04, 2_00_80, 6_18, 83, 8_27_75, 47, 4_79, 9, 15_17, 73, 5_38_94, 3_33, 8_05_81, 11_01_17, 1_88_11, 52_56, 12_95, 51, 15_25_26, 2_97, 79_86, 3_90, 12_44_16, 5_38, 3_54_31, 2_14, 98, 1_50_44, 2_57_37, 1_36, 71_08, 4_37_01, 23, 7_56, 13_53_55, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_00_04, 5_81, 6_37_73, 11_94_55, 6, 14_77_97, 8_82_03, 7, 6_45, 70, 21, 32_85, 1_02_69, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=SCREAMING_SNAKE_CASE__      ,	model_name='facebook/mbart-large-50'      ,	revision='d3913889c59cd5c9e456b269c376325eabad57e2'      ,	)







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[int]     )    ->				Dict:
         if not self.test_slow_tokenizer:
                # as we don't have a slow version, we can't compare the outputs between slow and fast versions
                return

         A						:     Any         =(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-mbart50', {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'     ):
                       A						:     Dict         =self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__      ,	**SCREAMING_SNAKE_CASE__     )
                       A						:     Tuple         =self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__      ,	**SCREAMING_SNAKE_CASE__     )

                       A						:     str         =tempfile.mkdtemp()

                       A						:     Optional[Any]         =tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__     )
                       A						:     Optional[int]         =tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Checks it save with the same files + the tokenizer.json file for the fast one
                       self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files     )     )
                       A						:     Dict         =tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f     )
                       self.assertSequenceEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

                       # Checks everything loads correctly in the same way
                       A						:     Tuple         =tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__     )
                       A						:     Optional[Any]         =tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )     )
                              # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
                              # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

                       shutil.rmtree(SCREAMING_SNAKE_CASE__     )

                       # Save tokenizer rust, legacy_format=True
                       A						:     Union[str, Any]         =tempfile.mkdtemp()

                       A						:     Any         =tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__      ,	legacy_format=SCREAMING_SNAKE_CASE__     )
                       A						:     Dict         =tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Checks it save with the same files
                       self.assertSequenceEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

                       # Checks everything loads correctly in the same way
                       A						:     List[Any]         =tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__     )
                       A						:     Tuple         =tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )     )

                       shutil.rmtree(SCREAMING_SNAKE_CASE__     )

                       # Save tokenizer rust, legacy_format=False
                       A						:     Optional[Any]         =tempfile.mkdtemp()

                       A						:     List[Any]         =tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE__      ,	legacy_format=SCREAMING_SNAKE_CASE__     )
                       A						:     List[Any]         =tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Checks it saved the tokenizer.json file
                       self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files     )     )

                       # Checks everything loads correctly in the same way
                       A						:     List[str]         =tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE__     )
                       A						:     Optional[Any]         =tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE__     )

                       # Check special tokens are set accordingly on Rust and Python
                       for key in tokenizer_pp.special_tokens_map:
                              self.assertTrue(hasattr(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )     )

                       shutil.rmtree(SCREAMING_SNAKE_CASE__     )







@require_torch
@require_sentencepiece
@require_tokenizers
class  SCREAMING_SNAKE_CASE_ (					unittest.TestCase			):
  '''simple docstring'''


  lowercase		:     List[str]            =				"facebook/mbart-large-50-one-to-many-mmt"
  lowercase		:     int            =				[
      " UN Chief Says There Is No Military Solution in Syria",
      " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
  ]
  lowercase		:     str            =				[
      "Şeful ONU declară că nu există o soluţie militară în Siria",
      "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
      " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
      " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
  ]
  lowercase		:     str            =				[EN_CODE, 8274, 127873, 25916, 7, 8622, 2071, 438, 67485, 53, 187895, 23, 51712, 2]







  @classmethod
  def 		SCREAMING_SNAKE_CASE_		(				cls    :					int     )    ->				Dict:
         A						:     MBartaaTokenizer         =MBartaaTokenizer.from_pretrained(
             cls.checkpoint_name      ,	src_lang='en_XX'      ,	tgt_lang='ro_RO'     )
         A						:     Any         =1
         return cls







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]     )    ->				Dict:
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ar_AR']      ,	25_00_01     )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['en_EN']      ,	25_00_04     )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ro_RO']      ,	25_00_20     )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['mr_IN']      ,	25_00_38     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					str     )    ->				str:
         A						:     Optional[Any]         =self.tokenizer.batch_encode_plus(self.src_text     ).input_ids[0]
         self.assertListEqual(self.expected_src_tokens      ,	SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]     )    ->				Any:
         self.assertIn(SCREAMING_SNAKE_CASE__      ,	self.tokenizer.all_special_ids     )
         A						:     List[str]         =[RO_CODE, 8_84, 90_19, 96, 9, 9_16, 8_67_92, 36, 1_87_43, 1_55_96, 5, 2]
         A						:     Any         =self.tokenizer.decode(SCREAMING_SNAKE_CASE__      ,	skip_special_tokens=SCREAMING_SNAKE_CASE__     )
         A						:     str         =self.tokenizer.decode(generated_ids[1:]      ,	skip_special_tokens=SCREAMING_SNAKE_CASE__     )
         self.assertEqual(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )
         self.assertNotIn(self.tokenizer.eos_token      ,	SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Tuple     )    ->				Any:
         A						:     Optional[int]         =['this is gunna be a long sentence ' * 20]
         assert isinstance(src_text[0]      ,	SCREAMING_SNAKE_CASE__     )
         A						:     Tuple         =10
         A						:     List[Any]         =self.tokenizer(SCREAMING_SNAKE_CASE__      ,	max_length=SCREAMING_SNAKE_CASE__      ,	truncation=SCREAMING_SNAKE_CASE__     ).input_ids[0]
         self.assertEqual(ids[0]      ,	SCREAMING_SNAKE_CASE__     )
         self.assertEqual(ids[-1]      ,	2     )
         self.assertEqual(len(SCREAMING_SNAKE_CASE__     )      ,	SCREAMING_SNAKE_CASE__     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[int]     )    ->				Optional[int]:
         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']     )      ,	[25_00_53, 25_00_01]     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					int     )    ->				Any:
         A						:     Dict         =tempfile.mkdtemp()
         A						:     str         =self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__     )
         A						:     Union[str, Any]         =MBartaaTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__     )
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids      ,	SCREAMING_SNAKE_CASE__     )







  @require_torch
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]     )    ->				Dict:
         A						:     Optional[int]         =self.tokenizer(self.src_text      ,	text_target=self.tgt_text      ,	padding=SCREAMING_SNAKE_CASE__      ,	return_tensors='pt'     )
         A						:     str         =shift_tokens_right(batch['labels']      ,	self.tokenizer.pad_token_id     )

         # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
         assert batch.input_ids[1][0] == EN_CODE
         assert batch.input_ids[1][-1] == 2
         assert batch.labels[1][0] == RO_CODE
         assert batch.labels[1][-1] == 2
         assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]







  @require_torch
  def 		SCREAMING_SNAKE_CASE_		(				self    :					str     )    ->				Any:
         A						:     Union[str, Any]         =self.tokenizer(
             self.src_text      ,	text_target=self.tgt_text      ,	padding=SCREAMING_SNAKE_CASE__      ,	truncation=SCREAMING_SNAKE_CASE__      ,	max_length=len(self.expected_src_tokens     )      ,	return_tensors='pt'      ,	)
         A						:     List[str]         =shift_tokens_right(batch['labels']      ,	self.tokenizer.pad_token_id     )

         self.assertIsInstance(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )

         self.assertEqual((2, 14)      ,	batch.input_ids.shape     )
         self.assertEqual((2, 14)      ,	batch.attention_mask.shape     )
         A						:     Dict         =batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens      ,	SCREAMING_SNAKE_CASE__     )
         self.assertEqual(2      ,	batch.decoder_input_ids[0, 0]     )  # decoder_start_token_id
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens      ,	[EN_CODE]     )
         self.assertEqual(self.tokenizer.suffix_tokens      ,	[self.tokenizer.eos_token_id]     )







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				str:
         A						:     int         =self.tokenizer(self.src_text      ,	padding=SCREAMING_SNAKE_CASE__      ,	truncation=SCREAMING_SNAKE_CASE__      ,	max_length=3      ,	return_tensors='pt'     )
         A						:     Tuple         =self.tokenizer(
             text_target=self.tgt_text      ,	padding=SCREAMING_SNAKE_CASE__      ,	truncation=SCREAMING_SNAKE_CASE__      ,	max_length=10      ,	return_tensors='pt'     )
         A						:     Tuple         =targets['input_ids']
         A						:     Any         =shift_tokens_right(SCREAMING_SNAKE_CASE__      ,	self.tokenizer.pad_token_id     )

         self.assertEqual(batch.input_ids.shape[1]      ,	3     )
         self.assertEqual(batch.decoder_input_ids.shape[1]      ,	10     )







  @require_torch
  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]     )    ->				Any:
         A						:     Dict         =self.tokenizer._build_translation_inputs(
             'A test'      ,	return_tensors='pt'      ,	src_lang='en_XX'      ,	tgt_lang='ar_AR'     )

         self.assertEqual(
             nested_simplify(SCREAMING_SNAKE_CASE__     )      ,	{
                 # en_XX, A, test, EOS
                 'input_ids': [[25_00_04, 62, 30_34, 2]],
                 'attention_mask': [[1, 1, 1, 1]],
                 # ar_AR
                 'forced_bos_token_id': 25_00_01,
             }      ,	)







import collections
import json
import math
import os
import re
import time
from fnmatch import fnmatch
from typing import Dict

import requests
from slack_sdk import WebClient


_lowercase							:      Optional[Any]      							=WebClient(token=os.environ['''CI_SLACK_BOT_TOKEN'''])



def A__  (			lowercase:     Optional[int]					)				->		Optional[int]:
       A						:     str         =test_results.split(' '					)

       A						:     List[str]         =0
       A						:     Tuple         =0

       # When the output is short enough, the output is surrounded by = signs: "== OUTPUT =="
       # When it is too long, those signs are not present.
       A						:     List[str]         =expressions[-2] if '=' in expressions[-1] else expressions[-1]

       for i, expression in enumerate(lowercase					):
              if "failed" in expression:
                     failed += int(expressions[i - 1]					)
              if "passed" in expression:
                     success += int(expressions[i - 1]					)

       return failed, success, time_spent



def A__  (			lowercase:     List[Any]					)				->		str:
       A						:     Union[str, Any]         ={}
       A						:     Optional[Any]         =None
       A						:     Union[str, Any]         =False
       for line in failures_short_lines.split('\n'					):
              if re.search(r'_ \[doctest\]',		lowercase					):
                     A						:     List[Any]         =True
                     A						:     Any         =line.split(' '					)[2]
              elif in_error and not line.split(' '					)[0].isdigit():
                     A						:     Dict         =line
                     A						:     List[str]         =False

       return failures



class  SCREAMING_SNAKE_CASE_ :
  '''simple docstring'''


  def __init__(				self    :					str      ,	SCREAMING_SNAKE_CASE__    :					str      ,	SCREAMING_SNAKE_CASE__    :					Dict     )    ->				List[str]:
         A						:     Tuple         =title

         A						:     Dict         =doc_test_results['time_spent'].split(','     )[0]
         A						:     Union[str, Any]         =doc_test_results['success']
         A						:     Any         =doc_test_results['failures']
         A						:     Optional[Any]         =self.n_success + self.n_failures

         # Failures and success of the modeling tests
         A						:     Union[str, Any]         =doc_test_results







  @property
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Tuple     )    ->				str:
         A						:     Any         =[self._time_spent]
         A						:     List[str]         =0

         for time in time_spent:
                A						:     List[Any]         =time.split(':'     )

                # Time can be formatted as xx:xx:xx, as .xx, or as x.xx if the time spent was less than a minute.
                if len(SCREAMING_SNAKE_CASE__     ) == 1:
                       A						:     List[str]         =[0, 0, time_parts[0]]

                A  ,  A  ,  A						:     Tuple         =int(time_parts[0]     ), int(time_parts[1]     ), float(time_parts[2]     )
                total_secs += hours * 36_00 + minutes * 60 + seconds

         A  ,  A  ,  A						:     str         =total_secs // 36_00, (total_secs % 36_00) // 60, total_secs % 60
         return f'{int(SCREAMING_SNAKE_CASE__     )}h{int(SCREAMING_SNAKE_CASE__     )}m{int(SCREAMING_SNAKE_CASE__     )}s'







  @property
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Tuple     )    ->				Dict:
         return {"type": "header", "text": {"type": "plain_text", "text": self.title}}







  @property
  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[str]     )    ->				Dict:
         return {
             "type": "section",
             "text": {
                 "type": "plain_text",
                 "text": f'🌞 There were no failures: all {self.n_tests} tests passed. The suite ran in {self.time}.',
                 "emoji": True,
             },
             "accessory": {
                 "type": "button",
                 "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
                 "url": f'https://github.com/huggingface/transformers/actions/runs/{os.environ["GITHUB_RUN_ID"]}',
             },
         }







  @property
  def 		SCREAMING_SNAKE_CASE_		(				self    :					List[Any]     )    ->				Dict:
         return {
             "type": "section",
             "text": {
                 "type": "plain_text",
                 "text": (
                     f'There were {self.n_failures} failures, out of {self.n_tests} tests.\nThe suite ran in'
                     f' {self.time}.'
                 ),
                 "emoji": True,
             },
             "accessory": {
                 "type": "button",
                 "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
                 "url": f'https://github.com/huggingface/transformers/actions/runs/{os.environ["GITHUB_RUN_ID"]}',
             },
         }







  @property
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[int]     )    ->				Dict:
         A						:     Tuple         =40
         A						:     Optional[Any]         ={k: v['failed'] for k, v in doc_test_results.items() if isinstance(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__     )}

         A						:     Any         =''
         for category, failures in category_failures.items():
                if len(SCREAMING_SNAKE_CASE__     ) == 0:
                       continue

                if report != "":
                       report += "\n\n"

                report += f'*{category} failures*:'.ljust(line_length // 2     ).rjust(line_length // 2     ) + "\n"
                report += "`"
                report += "`\n`".join(SCREAMING_SNAKE_CASE__     )
                report += "`"

         return {
             "type": "section",
             "text": {
                 "type": "mrkdwn",
                 "text": f'The following examples had failures:\n\n\n{report}\n',
             },
         }







  @property
  def 		SCREAMING_SNAKE_CASE_		(				self    :					Optional[Any]     )    ->				str:
         A						:     Optional[int]         =[self.header]

         if self.n_failures > 0:
                blocks.append(self.failures     )

         if self.n_failures > 0:
                blocks.extend([self.category_failures]     )

         if self.n_failures == 0:
                blocks.append(self.no_failures     )

         return json.dumps(SCREAMING_SNAKE_CASE__     )







  @staticmethod
  def 		SCREAMING_SNAKE_CASE_		(				)    ->				Optional[Any]:
         A						:     Tuple         =[
             {
                 'type': 'section',
                 'text': {
                     'type': 'plain_text',
                     'text': 'There was an issue running the tests.',
                 },
                 'accessory': {
                     'type': 'button',
                     'text': {'type': 'plain_text', 'text': 'Check Action results', 'emoji': True},
                     'url': f'https://github.com/huggingface/transformers/actions/runs/{os.environ["GITHUB_RUN_ID"]}',
                 },
             }
         ]

         print('Sending the following payload'     )
         print(json.dumps({'blocks': json.loads(SCREAMING_SNAKE_CASE__     )}     )     )

         client.chat_postMessage(
             channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']      ,	text='There was an issue running the tests.'      ,	blocks=SCREAMING_SNAKE_CASE__      ,	)







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict     )    ->				Optional[int]:
         print('Sending the following payload'     )
         print(json.dumps({'blocks': json.loads(self.payload     )}     )     )

         A						:     Any         =f'{self.n_failures} failures out of {self.n_tests} tests,' if self.n_failures else 'All tests passed.'

         A						:     Dict         =client.chat_postMessage(
             channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']      ,	blocks=self.payload      ,	text=SCREAMING_SNAKE_CASE__      ,	)







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Dict      ,	SCREAMING_SNAKE_CASE__    :					List[str]      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]      ,	SCREAMING_SNAKE_CASE__    :					int      ,	SCREAMING_SNAKE_CASE__    :					Optional[Any]     )    ->				Optional[int]:
         A						:     List[str]         =''
         for key, value in failures.items():
                A						:     Any         =value[:2_00] + ' [Truncated]' if len(SCREAMING_SNAKE_CASE__     ) > 2_50 else value
                failures_text += f'*{key}*\n_{value}_\n\n'

         A						:     Union[str, Any]         =job_name
         A						:     Any         ={'type': 'section', 'text': {'type': 'mrkdwn', 'text': text}}

         if job_link is not None:
                A						:     int         ={
                    'type': 'button',
                    'text': {'type': 'plain_text', 'text': 'GitHub Action job', 'emoji': True},
                    'url': job_link,
                }

         return [
             {"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}},
             content,
             {"type": "section", "text": {"type": "mrkdwn", "text": failures_text}},
         ]







  def 		SCREAMING_SNAKE_CASE_		(				self    :					Tuple     )    ->				List[Any]:
         if self.thread_ts is None:
                raise ValueError('Can only post reply if a post has been made.'     )

         A						:     Union[str, Any]         =self.doc_test_results.pop('job_link'     )
         self.doc_test_results.pop('failures'     )
         self.doc_test_results.pop('success'     )
         self.doc_test_results.pop('time_spent'     )

         A						:     Union[str, Any]         =sorted(self.doc_test_results.items()      ,	key=lambda SCREAMING_SNAKE_CASE__     : t[0]     )
         for job, job_result in sorted_dict:
                if len(job_result['failures']     ):
                       A						:     Any         =f'*Num failures* :{len(job_result["failed"]     )} \n'
                       A						:     List[Any]         =job_result['failures']
                       A						:     Any         =self.get_reply_blocks(SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__      ,	text=SCREAMING_SNAKE_CASE__     )

                       print('Sending the following reply'     )
                       print(json.dumps({'blocks': blocks}     )     )

                       client.chat_postMessage(
                           channel=os.environ['CI_SLACK_CHANNEL_ID_DAILY']      ,	text=f'Results for {job}'      ,	blocks=SCREAMING_SNAKE_CASE__      ,	thread_ts=self.thread_ts['ts']      ,	)

                       time.sleep(1     )







def A__  (			)				->		Union[str, Any]:
       A						:     Any         =os.environ['GITHUB_RUN_ID']
       A						:     List[Any]         =F'https://api.github.com/repos/huggingface/transformers/actions/runs/{run_id}/jobs?per_page=100'
       A						:     Union[str, Any]         =requests.get(lowercase					).json()
       A						:     List[Any]         ={}

       try:
              jobs.update({job['name']: job['html_url'] for job in result['jobs']}					)
              A						:     List[str]         =math.ceil((result['total_count'] - 100) / 100					)

              for i in range(lowercase					):
                     A						:     List[str]         =requests.get(url + F'&page={i + 2}'					).json()
                     jobs.update({job['name']: job['html_url'] for job in result['jobs']}					)

              return jobs
       except Exception as e:
              print('Unknown error, could not fetch links.',		lowercase					)

       return {}



def A__  (			lowercase:     str					)				->		Optional[Any]:
       A						:     Any         ={}

       if os.path.exists(lowercase					):
              A						:     List[Any]         =os.listdir(lowercase					)
              for file in files:
                     try:
                            with open(os.path.join(lowercase,		lowercase					),		encoding='utf-8'					) as f:
                                   A						:     Optional[int]         =f.read()
                     except UnicodeDecodeError as e:
                            raise ValueError(F'Could not open {os.path.join(lowercase,		lowercase					)}.'					) from e

       return _artifact



def A__  (			)				->		int:



       class  SCREAMING_SNAKE_CASE_ :
         '''simple docstring'''


         def __init__(				self    :					Optional[int]      ,	SCREAMING_SNAKE_CASE__    :					str     )    ->				List[str]:
                A						:     Dict         =name
                A						:     Dict         =[]







         def __str__(				self    :					Optional[Any]     )    ->				List[str]:
                return self.name







         def 		SCREAMING_SNAKE_CASE_		(				self    :					int      ,	SCREAMING_SNAKE_CASE__    :					str     )    ->				List[Any]:
                self.paths.append({'name': self.name, 'path': path}     )







       A						:     Dict[str, Artifact]         ={}

       A						:     str         =filter(os.path.isdir,		os.listdir()					)
       for directory in directories:
              A						:     Tuple         =directory
              if artifact_name not in _available_artifacts:
                     A						:     int         =Artifact(lowercase					)

                     _available_artifacts[artifact_name].add_path(lowercase					)

       return _available_artifacts


if __name__ == "__main__":
    _lowercase							:      Optional[int]      							=get_job_links()
    _lowercase							:      str      							=retrieve_available_artifacts()

    _lowercase							:      List[Any]      							=collections.OrderedDict(
        [
            ('''*.py''', '''API Examples'''),
            ('''*.md''', '''MD Examples'''),
        ]
    )

    # This dict will contain all the information relative to each doc test category:
    # - failed: list of failed tests
    # - failures: dict in the format 'test': 'error_message'
    _lowercase							:      Optional[Any]      							={
        v: {
            '''failed''': [],
            '''failures''': {},
        }
        for v in docs.values()
    }

    # Link to the GitHub Action job
    _lowercase							:      List[Any]      							=github_actions_job_links.get('''run_doctests''')

    _lowercase							:      int      							=available_artifacts['''doc_tests_gpu_test_reports'''].paths[0]
    _lowercase							:      Dict      							=retrieve_artifact(artifact_path['''name'''])
    if "stats" in artifact:
        _lowercase							,	_lowercase							,	_lowercase							:      List[Any]      							=handle_test_results(artifact['''stats'''])
        _lowercase							:      Any      							=failed
        _lowercase							:      Union[str, Any]      							=success
        _lowercase							:      str      							=time_spent[1:-1] + ''', '''

        _lowercase							:      Any      							=extract_first_line_failure(artifact['''failures_short'''])
        for line in artifact["summary_short"].split('''\n'''):
            if re.search('''FAILED''', line):
                _lowercase							:      Tuple      							=line.replace('''FAILED ''', '''''')
                _lowercase							:      int      							=line.split()[0].replace('''\n''', '''''')

                if "::" in line:
                    _lowercase							,	_lowercase							:      str      							=line.split('''::''')
                else:
                    _lowercase							,	_lowercase							:      Union[str, Any]      							=line, line

                for file_regex in docs.keys():
                    if fnmatch(file_path, file_regex):
                        _lowercase							:      Any      							=docs[file_regex]
                        doc_test_results[category]["failed"].append(test)

                        _lowercase							:      Any      							=all_failures[test] if test in all_failures else '''N/A'''
                        _lowercase							:      Tuple      							=failure
                        break

    _lowercase							:      Optional[int]      							=Message('''🤗 Results of the doc tests.''', doc_test_results)
    message.post()
    message.post_reply()






