






'''simple docstring'''
from typing import Dict, Optional

import numpy as np

import datasets


a_					= '\nIoU is the area of overlap between the predicted segmentation and the ground truth divided by the area of union\nbetween the predicted segmentation and the ground truth. For binary (two classes) or multi-class segmentation,\nthe mean IoU of the image is calculated by taking the IoU of each class and averaging them.\n'

a_					= '\nArgs:\n    predictions (`List[ndarray]`):\n        List of predicted segmentation maps, each of shape (height, width). Each segmentation map can be of a different size.\n    references (`List[ndarray]`):\n        List of ground truth segmentation maps, each of shape (height, width). Each segmentation map can be of a different size.\n    num_labels (`int`):\n        Number of classes (categories).\n    ignore_index (`int`):\n        Index that will be ignored during evaluation.\n    nan_to_num (`int`, *optional*):\n        If specified, NaN values will be replaced by the number defined by the user.\n    label_map (`dict`, *optional*):\n        If specified, dictionary mapping old label indices to new label indices.\n    reduce_labels (`bool`, *optional*, defaults to `False`):\n        Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background,\n        and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255.\n\nReturns:\n    `Dict[str, float | ndarray]` comprising various elements:\n    - *mean_iou* (`float`):\n        Mean Intersection-over-Union (IoU averaged over all categories).\n    - *mean_accuracy* (`float`):\n        Mean accuracy (averaged over all categories).\n    - *overall_accuracy* (`float`):\n        Overall accuracy on all images.\n    - *per_category_accuracy* (`ndarray` of shape `(num_labels,)`):\n        Per category accuracy.\n    - *per_category_iou* (`ndarray` of shape `(num_labels,)`):\n        Per category IoU.\n\nExamples:\n\n    >>> import numpy as np\n\n    >>> mean_iou = datasets.load_metric("mean_iou")\n\n    >>> # suppose one has 3 different segmentation maps predicted\n    >>> predicted_1 = np.array([[1, 2], [3, 4], [5, 255]])\n    >>> actual_1 = np.array([[0, 3], [5, 4], [6, 255]])\n\n    >>> predicted_2 = np.array([[2, 7], [9, 2], [3, 6]])\n    >>> actual_2 = np.array([[1, 7], [9, 2], [3, 6]])\n\n    >>> predicted_3 = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]])\n    >>> actual_3 = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]])\n\n    >>> predicted = [predicted_1, predicted_2, predicted_3]\n    >>> ground_truth = [actual_1, actual_2, actual_3]\n\n    >>> results = mean_iou.compute(predictions=predicted, references=ground_truth, num_labels=10, ignore_index=255, reduce_labels=False)\n    >>> print(results) # doctest: +NORMALIZE_WHITESPACE\n    {\'mean_iou\': 0.47750000000000004, \'mean_accuracy\': 0.5916666666666666, \'overall_accuracy\': 0.5263157894736842, \'per_category_iou\': array([0.   , 0.   , 0.375, 0.4  , 0.5  , 0.   , 0.5  , 1.   , 1.   , 1.   ]), \'per_category_accuracy\': array([0.        , 0.        , 0.75      , 0.66666667, 1.        , 0.        , 0.5       , 1.        , 1.        , 1.        ])}\n'

a_					= '\\n@software{MMSegmentation_Contributors_OpenMMLab_Semantic_Segmentation_2020,\nauthor = {{MMSegmentation Contributors}},\nlicense = {Apache-2.0},\nmonth = {7},\ntitle = {{OpenMMLab Semantic Segmentation Toolbox and Benchmark}},\nurl = {https://github.com/open-mmlab/mmsegmentation},\nyear = {2020}\n}'





def  _a(  UpperCamelCase__				:						Tuple,	UpperCamelCase__				:						int,	UpperCamelCase__				:						Optional[int],	UpperCamelCase__				:						bool,	UpperCamelCase__				:						Optional[Dict[int, int]] = None,	UpperCamelCase__				:						bool = False,	):




      '''simple docstring'''






      if label_map is not None:
            for old_id, new_id in label_map.items():
                  SCREAMING_SNAKE_CASE__     :				Optional[int]									=new_id

    # turn into Numpy arrays
      SCREAMING_SNAKE_CASE__     :				int									=np.array(UpperCamelCase__							)
      SCREAMING_SNAKE_CASE__     :				int									=np.array(UpperCamelCase__							)

      if reduce_labels:
            SCREAMING_SNAKE_CASE__     :				str									=2_5_5
            SCREAMING_SNAKE_CASE__     :				str									=label - 1
            SCREAMING_SNAKE_CASE__     :				Tuple									=2_5_5

      SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=label != ignore_index
      SCREAMING_SNAKE_CASE__     :				str									=np.not_equal(UpperCamelCase__,	UpperCamelCase__							)
      SCREAMING_SNAKE_CASE__     :				Any									=pred_label[mask]
      SCREAMING_SNAKE_CASE__     :				Tuple									=np.array(UpperCamelCase__							)[mask]

      SCREAMING_SNAKE_CASE__     :				List[Any]									=pred_label[pred_label == label]

      SCREAMING_SNAKE_CASE__     :				List[Any]									=np.histogram(UpperCamelCase__,	bins=UpperCamelCase__,	range=(0, num_labels - 1)							)[0]
      SCREAMING_SNAKE_CASE__     :				Optional[int]									=np.histogram(UpperCamelCase__,	bins=UpperCamelCase__,	range=(0, num_labels - 1)							)[0]
      SCREAMING_SNAKE_CASE__     :				List[str]									=np.histogram(UpperCamelCase__,	bins=UpperCamelCase__,	range=(0, num_labels - 1)							)[0]

      SCREAMING_SNAKE_CASE__     :				str									=area_pred_label + area_label - area_intersect

      return area_intersect, area_union, area_pred_label, area_label





def  _a(  UpperCamelCase__				:						List[Any],	UpperCamelCase__				:						Any,	UpperCamelCase__				:						int,	UpperCamelCase__				:						bool,	UpperCamelCase__				:						Optional[Dict[int, int]] = None,	UpperCamelCase__				:						bool = False,	):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__     :				Optional[int]									=np.zeros((num_labels,),	dtype=np.floataa							)
      SCREAMING_SNAKE_CASE__     :				List[str]									=np.zeros((num_labels,),	dtype=np.floataa							)
      SCREAMING_SNAKE_CASE__     :				Dict									=np.zeros((num_labels,),	dtype=np.floataa							)
      SCREAMING_SNAKE_CASE__     :				List[str]									=np.zeros((num_labels,),	dtype=np.floataa							)
      for result, gt_seg_map in zip(UpperCamelCase__,	UpperCamelCase__							):
            SCREAMING_SNAKE_CASE__		,	SCREAMING_SNAKE_CASE__		,	SCREAMING_SNAKE_CASE__		,	SCREAMING_SNAKE_CASE__     :				Optional[Any]									=intersect_and_union(
                UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__							)
            total_area_intersect += area_intersect
            total_area_union += area_union
            total_area_pred_label += area_pred_label
            total_area_label += area_label
      return total_area_intersect, total_area_union, total_area_pred_label, total_area_label





def  _a(  UpperCamelCase__				:						Optional[int],	UpperCamelCase__				:						str,	UpperCamelCase__				:						List[Any],	UpperCamelCase__				:						bool,	UpperCamelCase__				:						Optional[int] = None,	UpperCamelCase__				:						Optional[Dict[int, int]] = None,	UpperCamelCase__				:						bool = False,	):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__		,	SCREAMING_SNAKE_CASE__		,	SCREAMING_SNAKE_CASE__		,	SCREAMING_SNAKE_CASE__     :				List[Any]									=total_intersect_and_union(
          UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__,	UpperCamelCase__							)

      # compute metrics
      SCREAMING_SNAKE_CASE__     :				Tuple									={}

      SCREAMING_SNAKE_CASE__     :				List[Any]									=total_area_intersect.sum() / total_area_label.sum()
      SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=total_area_intersect / total_area_union
      SCREAMING_SNAKE_CASE__     :				int									=total_area_intersect / total_area_label

      SCREAMING_SNAKE_CASE__     :				Tuple									=np.nanmean(UpperCamelCase__							)
      SCREAMING_SNAKE_CASE__     :				List[Any]									=np.nanmean(UpperCamelCase__							)
      SCREAMING_SNAKE_CASE__     :				int									=all_acc
      SCREAMING_SNAKE_CASE__     :				int									=iou
      SCREAMING_SNAKE_CASE__     :				Optional[Any]									=acc

      if nan_to_num is not None:
            SCREAMING_SNAKE_CASE__     :				Tuple									={metric: np.nan_to_num(UpperCamelCase__,	nan=UpperCamelCase__							) for metric, metric_value in metrics.items()}

      return metrics







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,				_KWARGS_DESCRIPTION				)
class      __SCREAMING_SNAKE_CASE      (			datasets.Metric				):




 def      __magic_name__	(      self	:							Dict      )	->						Any:
       return datasets.MetricInfo(
           description=_DESCRIPTION					,      citation=_CITATION					,      inputs_description=_KWARGS_DESCRIPTION					,      features=datasets.Features(
               # 1st Seq - height dim, 2nd - width dim
               {
                   '''predictions''': datasets.Sequence(datasets.Sequence(datasets.Value('''uint16'''      )      )      ),
                   '''references''': datasets.Sequence(datasets.Sequence(datasets.Value('''uint16'''      )      )      ),
               }      )					,      reference_urls=[
               '''https://github.com/open-mmlab/mmsegmentation/blob/71c201b1813267d78764f306a297ca717827c4bf/mmseg/core/evaluation/metrics.py'''
           ]					,      )




 def      __magic_name__	(      self	:							int					,      __lowercase	:							List[str]					,      __lowercase	:							str					,      __lowercase	:							int					,      __lowercase	:							bool					,      __lowercase	:							Optional[int] = None					,      __lowercase	:							Optional[Dict[int, int]] = None					,      __lowercase	:							bool = False					,      )	->						int:
       SCREAMING_SNAKE_CASE__     :				Dict									=mean_iou(
           results=__lowercase					,      gt_seg_maps=__lowercase					,      num_labels=__lowercase					,      ignore_index=__lowercase					,      nan_to_num=__lowercase					,      label_map=__lowercase					,      reduce_labels=__lowercase					,      )
       return iou_result







'''simple docstring'''
import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


a_					= get_tests_dir('fixtures/test_sentencepiece_bpe_char.model')






@require_sentencepiece
@require_tokenizers
class      __SCREAMING_SNAKE_CASE      (			lowerCamelCase	,				unittest.TestCase				):
 snake_case_  							=							SpeechTaTokenizer
 snake_case_  							=							False
 snake_case_  							=							True




 def      __magic_name__	(      self	:							int      )	->						Any:
       super().setUp()

       # We have a SentencePiece fixture for testing
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=SpeechTaTokenizer(__lowercase      )

       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=AddedToken('''<mask>'''					,      lstrip=__lowercase					,      rstrip=__lowercase      )
       SCREAMING_SNAKE_CASE__     :				List[Any]									=mask_token
       tokenizer.add_special_tokens({'''mask_token''': mask_token}      )
       tokenizer.add_tokens(['''<ctc_blank>''']      )

       tokenizer.save_pretrained(self.tmpdirname      )




 def      __magic_name__	(      self	:							Dict					,      __lowercase	:							int      )	->						Optional[int]:
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									='''this is a test'''
       SCREAMING_SNAKE_CASE__     :				int									='''this is a test'''
       return input_text, output_text




 def      __magic_name__	(      self	:							List[Any]					,      __lowercase	:							int					,      __lowercase	:							Optional[Any]=False					,      __lowercase	:							Union[str, Any]=20					,      __lowercase	:							Any=5      )	->						Any:
       SCREAMING_SNAKE_CASE__		,	SCREAMING_SNAKE_CASE__     :				int									=self.get_input_output_texts(__lowercase      )
       SCREAMING_SNAKE_CASE__     :				str									=tokenizer.encode(__lowercase					,      add_special_tokens=__lowercase      )
       SCREAMING_SNAKE_CASE__     :				str									=tokenizer.decode(__lowercase					,      clean_up_tokenization_spaces=__lowercase      )
       return text, ids




 def      __magic_name__	(      self	:							Dict      )	->						str:
       SCREAMING_SNAKE_CASE__     :				Optional[int]									='''<pad>'''
       SCREAMING_SNAKE_CASE__     :				Optional[int]									=1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowercase      )					,      __lowercase      )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowercase      )					,      __lowercase      )




 def      __magic_name__	(      self	:							Tuple      )	->						List[str]:
       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=list(self.get_tokenizer().get_vocab().keys()      )

       self.assertEqual(vocab_keys[0]					,      '''<s>'''      )
       self.assertEqual(vocab_keys[1]					,      '''<pad>'''      )
       self.assertEqual(vocab_keys[-4]					,      '''œ'''      )
       self.assertEqual(vocab_keys[-2]					,      '''<mask>'''      )
       self.assertEqual(vocab_keys[-1]					,      '''<ctc_blank>'''      )
       self.assertEqual(len(__lowercase      )					,      81      )




 def      __magic_name__	(      self	:							Dict      )	->						List[str]:
       self.assertEqual(self.get_tokenizer().vocab_size					,      79      )




 def      __magic_name__	(      self	:							Optional[Any]      )	->						str:
       SCREAMING_SNAKE_CASE__     :				str									=self.get_tokenizers(do_lower_case=__lowercase      )
       for tokenizer in tokenizers:
             with self.subTest(F"{tokenizer.__class__.__name__}"      ):
                   SCREAMING_SNAKE_CASE__     :				Tuple									=tokenizer.vocab_size
                   SCREAMING_SNAKE_CASE__     :				Any									=len(__lowercase      )

                   self.assertNotEqual(__lowercase					,      0      )

                   # We usually have added tokens from the start in tests because our vocab fixtures are
                   # smaller than the original vocabs - let's not assert this
                   # self.assertEqual(vocab_size, all_size)

                   SCREAMING_SNAKE_CASE__     :				int									=['''aaaaa bbbbbb''', '''cccccccccdddddddd''']
                   SCREAMING_SNAKE_CASE__     :				List[str]									=tokenizer.add_tokens(__lowercase      )
                   SCREAMING_SNAKE_CASE__     :				List[str]									=tokenizer.vocab_size
                   SCREAMING_SNAKE_CASE__     :				Optional[int]									=len(__lowercase      )

                   self.assertNotEqual(__lowercase					,      0      )
                   self.assertEqual(__lowercase					,      __lowercase      )
                   self.assertEqual(__lowercase					,      len(__lowercase      )      )
                   self.assertEqual(__lowercase					,      all_size + len(__lowercase      )      )

                   SCREAMING_SNAKE_CASE__     :				Tuple									=tokenizer.encode('''aaaaa bbbbbb low cccccccccdddddddd l'''					,      add_special_tokens=__lowercase      )

                   self.assertGreaterEqual(len(__lowercase      )					,      4      )
                   self.assertGreater(tokens[0]					,      tokenizer.vocab_size - 1      )
                   self.assertGreater(tokens[-3]					,      tokenizer.vocab_size - 1      )

                   SCREAMING_SNAKE_CASE__     :				str									={'''eos_token''': '''>>>>|||<||<<|<<''', '''pad_token''': '''<<<<<|||>|>>>>|>'''}
                   SCREAMING_SNAKE_CASE__     :				int									=tokenizer.add_special_tokens(__lowercase      )
                   SCREAMING_SNAKE_CASE__     :				List[str]									=tokenizer.vocab_size
                   SCREAMING_SNAKE_CASE__     :				int									=len(__lowercase      )

                   self.assertNotEqual(__lowercase					,      0      )
                   self.assertEqual(__lowercase					,      __lowercase      )
                   self.assertEqual(__lowercase					,      len(__lowercase      )      )
                   self.assertEqual(__lowercase					,      all_size_a + len(__lowercase      )      )

                   SCREAMING_SNAKE_CASE__     :				List[Any]									=tokenizer.encode(
                       '''>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'''					,      add_special_tokens=__lowercase      )

                   self.assertGreaterEqual(len(__lowercase      )					,      6      )
                   self.assertGreater(tokens[0]					,      tokenizer.vocab_size - 1      )
                   self.assertGreater(tokens[0]					,      tokens[1]      )
                   self.assertGreater(tokens[-3]					,      tokenizer.vocab_size - 1      )
                   self.assertGreater(tokens[-3]					,      tokens[-4]      )
                   self.assertEqual(tokens[0]					,      tokenizer.eos_token_id      )
                   self.assertEqual(tokens[-3]					,      tokenizer.pad_token_id      )




 def      __magic_name__	(      self	:							Optional[Any]      )	->						Any:
       pass




 def      __magic_name__	(      self	:							List[str]      )	->						List[Any]:
       pass




 def      __magic_name__	(      self	:							Dict      )	->						List[Any]:
       SCREAMING_SNAKE_CASE__     :				Dict									=self.get_tokenizer()

       SCREAMING_SNAKE_CASE__     :				Optional[int]									=tokenizer.tokenize('''This is a test'''      )
       # fmt: off
       self.assertListEqual(__lowercase					,      [SPIECE_UNDERLINE, '''T''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''a''', SPIECE_UNDERLINE, '''t''', '''e''', '''s''', '''t''']      )
       # fmt: on

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(__lowercase      )					,      [4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]					,      )

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
       self.assertListEqual(
           __lowercase					,      [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''92000''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']      )

       SCREAMING_SNAKE_CASE__     :				Union[str, Any]									=tokenizer.convert_tokens_to_ids(__lowercase      )
       # fmt: off
       self.assertListEqual(__lowercase					,      [4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26]      )
       # fmt: on

       SCREAMING_SNAKE_CASE__     :				Optional[Any]									=tokenizer.convert_ids_to_tokens(__lowercase      )
       self.assertListEqual(
           __lowercase					,      [SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''<unk>''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.''']      )




 @slow
 def      __magic_name__	(      self	:							List[str]      )	->						List[str]:
       # Use custom sequence because this tokenizer does not handle numbers.
       SCREAMING_SNAKE_CASE__     :				List[Any]									=[
           '''Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '''
           '''general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '''
           '''Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '''
           '''models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.''',
           '''BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '''
           '''conditioning on both left and right context in all layers.''',
           '''The quick brown fox jumps over the lazy dog.''',
       ]

       # fmt: off
       SCREAMING_SNAKE_CASE__     :				str									={
           '''input_ids''': [
               [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
               [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
               [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
           ],
           '''attention_mask''': [
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
               [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
           ]
       }
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=__lowercase					,      model_name='''microsoft/speecht5_asr'''					,      revision='''c5ef64c71905caeccde0e4462ef3f9077224c524'''					,      sequences=__lowercase					,      )