import json
import os
import shutil
import tempfile
import unittest
from multiprocessing import get_context
from pathlib import Path

import datasets
import numpy as np
from datasets import load_dataset
from parameterized import parameterized

from transformers import AutoProcessor
from transformers.models.wavaveca import WavaVecaCTCTokenizer, WavaVecaFeatureExtractor
from transformers.models.wavaveca.tokenization_wavaveca import VOCAB_FILES_NAMES
from transformers.testing_utils import require_pyctcdecode, require_torch, require_torchaudio, slow
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_pyctcdecode_available, is_torch_available

from ..wavaveca.test_feature_extraction_wavaveca import floats_list


if is_pyctcdecode_available():
 from huggingface_hub import snapshot_download
 from pyctcdecode import BeamSearchDecoderCTC

 from transformers.models.wavaveca_with_lm import WavaVecaProcessorWithLM
 from transformers.models.wavaveca_with_lm.processing_wavaveca_with_lm import WavaVecaDecoderWithLMOutput

if is_torch_available():
 from transformers import WavaVecaForCTC






@require_pyctcdecode
class   __lowercase				(  unittest.TestCase  ):





      def        _lowercase				(		self						:					Union[str, Any]       )    ->   Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase							=     """| <pad> <unk> <s> </s> a b c d e f g h i j k""".split()
         UpperCAmelCase							=     dict(zip(_lowerCamelCase					,       range(len(_lowerCamelCase       )       )       )       )

         UpperCAmelCase							=     {
             """unk_token""": """<unk>""",
             """bos_token""": """<s>""",
             """eos_token""": """</s>""",
         }
         UpperCAmelCase							=     {
             """feature_size""": 1,
             """padding_value""": 0.0,
             """sampling_rate""": 1_6_0_0_0,
             """return_attention_mask""": False,
             """do_normalize""": True,
         }

         UpperCAmelCase							=     tempfile.mkdtemp()
         UpperCAmelCase							=     os.path.join(self.tmpdirname					,       VOCAB_FILES_NAMES["""vocab_file"""]       )
         UpperCAmelCase							=     os.path.join(self.tmpdirname					,       _lowerCamelCase       )
         with open(self.vocab_file					,       """w"""					,       encoding="""utf-8"""       ) as fp:
            fp.write(json.dumps(_lowerCamelCase       ) + """\n"""       )

         with open(self.feature_extraction_file					,       """w"""					,       encoding="""utf-8"""       ) as fp:
            fp.write(json.dumps(_lowerCamelCase       ) + """\n"""       )

         # load decoder from hub
         UpperCAmelCase							=     """hf-internal-testing/ngram-beam-search-decoder"""





      def        _lowercase				(		self						:					Tuple					,       **__lowerCamelCase						:					Optional[Any]       )    ->   List[str]:






         """simple docstring"""







         UpperCAmelCase							=     self.add_kwargs_tokens_map.copy()
         kwargs.update(_lowerCamelCase       )
         return WavaVecaCTCTokenizer.from_pretrained(self.tmpdirname					,       **_lowerCamelCase       )





      def        _lowercase				(		self						:					Optional[Any]					,       **__lowerCamelCase						:					Optional[int]       )    ->   List[Any]:






         """simple docstring"""







         return WavaVecaFeatureExtractor.from_pretrained(self.tmpdirname					,       **_lowerCamelCase       )





      def        _lowercase				(		self						:					Optional[int]					,       **__lowerCamelCase						:					List[str]       )    ->   Dict:






         """simple docstring"""







         return BeamSearchDecoderCTC.load_from_hf_hub(self.decoder_name					,       **_lowerCamelCase       )





      def        _lowercase				(		self						:					Any       )    ->   Optional[Any]:






         """simple docstring"""







         shutil.rmtree(self.tmpdirname       )





      def        _lowercase				(		self						:					int       )    ->   Optional[Any]:






         """simple docstring"""







         UpperCAmelCase							=     self.get_tokenizer()
         UpperCAmelCase							=     self.get_feature_extractor()
         UpperCAmelCase							=     self.get_decoder()

         UpperCAmelCase							=     WavaVecaProcessorWithLM(tokenizer=_lowerCamelCase					,       feature_extractor=_lowerCamelCase					,       decoder=_lowerCamelCase       )

         processor.save_pretrained(self.tmpdirname       )
         UpperCAmelCase							=     WavaVecaProcessorWithLM.from_pretrained(self.tmpdirname       )

         # tokenizer
         self.assertEqual(processor.tokenizer.get_vocab()					,       tokenizer.get_vocab()       )
         self.assertIsInstance(processor.tokenizer					,       _lowerCamelCase       )

         # feature extractor
         self.assertEqual(processor.feature_extractor.to_json_string()					,       feature_extractor.to_json_string()       )
         self.assertIsInstance(processor.feature_extractor					,       _lowerCamelCase       )

         # decoder
         self.assertEqual(processor.decoder._alphabet.labels					,       decoder._alphabet.labels       )
         self.assertEqual(
             processor.decoder.model_container[decoder._model_key]._unigram_set					,       decoder.model_container[decoder._model_key]._unigram_set					,       )
         self.assertIsInstance(processor.decoder					,       _lowerCamelCase       )





      def        _lowercase				(		self						:					Union[str, Any]       )    ->   Any:






         """simple docstring"""







         UpperCAmelCase							=     WavaVecaProcessorWithLM(
             tokenizer=self.get_tokenizer()					,       feature_extractor=self.get_feature_extractor()					,       decoder=self.get_decoder()       )
         processor.save_pretrained(self.tmpdirname       )

         # make sure that error is thrown when decoder alphabet doesn't match
         UpperCAmelCase							=     WavaVecaProcessorWithLM.from_pretrained(
             self.tmpdirname					,       alpha=5.0					,       beta=3.0					,       score_boundary=-7.0					,       unk_score_offset=3       )

         # decoder
         self.assertEqual(processor.language_model.alpha					,       5.0       )
         self.assertEqual(processor.language_model.beta					,       3.0       )
         self.assertEqual(processor.language_model.score_boundary					,       -7.0       )
         self.assertEqual(processor.language_model.unk_score_offset					,       3       )





      def        _lowercase				(		self						:					int       )    ->   Dict:






         """simple docstring"""







         UpperCAmelCase							=     self.get_tokenizer()
         # add token to trigger raise
         tokenizer.add_tokens(["""xx"""]       )
         with self.assertRaisesRegex(_lowerCamelCase					,       """include"""       ):
            WavaVecaProcessorWithLM(
                tokenizer=_lowerCamelCase					,       feature_extractor=self.get_feature_extractor()					,       decoder=self.get_decoder()       )





      def        _lowercase				(		self						:					Optional[Any]       )    ->   Optional[int]:






         """simple docstring"""







         UpperCAmelCase							=     self.get_feature_extractor()
         UpperCAmelCase							=     self.get_tokenizer()
         UpperCAmelCase							=     self.get_decoder()

         UpperCAmelCase							=     WavaVecaProcessorWithLM(tokenizer=_lowerCamelCase					,       feature_extractor=_lowerCamelCase					,       decoder=_lowerCamelCase       )

         UpperCAmelCase							=     floats_list((3, 1_0_0_0)       )

         UpperCAmelCase							=     feature_extractor(_lowerCamelCase					,       return_tensors="""np"""       )
         UpperCAmelCase							=     processor(_lowerCamelCase					,       return_tensors="""np"""       )

         for key in input_feat_extract.keys():
            self.assertAlmostEqual(input_feat_extract[key].sum()					,       input_processor[key].sum()					,       delta=1e-2       )





      def        _lowercase				(		self						:					int       )    ->   List[Any]:






         """simple docstring"""







         UpperCAmelCase							=     self.get_feature_extractor()
         UpperCAmelCase							=     self.get_tokenizer()
         UpperCAmelCase							=     self.get_decoder()

         UpperCAmelCase							=     WavaVecaProcessorWithLM(tokenizer=_lowerCamelCase					,       feature_extractor=_lowerCamelCase					,       decoder=_lowerCamelCase       )

         UpperCAmelCase							=     """This is a test string"""

         UpperCAmelCase							=     processor(text=_lowerCamelCase       )

         UpperCAmelCase							=     tokenizer(_lowerCamelCase       )

         for key in encoded_tok.keys():
            self.assertListEqual(encoded_tok[key]					,       encoded_processor[key]       )





      def        _lowercase				(		self						:					List[Any]					,       __lowerCamelCase						:					Dict=(2, 1_0, 1_6)					,       __lowerCamelCase						:					List[Any]=7_7       )    ->   Optional[int]:






         """simple docstring"""







         np.random.seed(_lowerCamelCase       )
         return np.random.rand(*_lowerCamelCase       )





      def        _lowercase				(		self						:					Dict       )    ->   str:






         """simple docstring"""







         UpperCAmelCase							=     self.get_feature_extractor()
         UpperCAmelCase							=     self.get_tokenizer()
         UpperCAmelCase							=     self.get_decoder()

         UpperCAmelCase							=     WavaVecaProcessorWithLM(tokenizer=_lowerCamelCase					,       feature_extractor=_lowerCamelCase					,       decoder=_lowerCamelCase       )

         UpperCAmelCase							=     self._get_dummy_logits(shape=(1_0, 1_6)					,       seed=1_3       )

         UpperCAmelCase							=     processor.decode(_lowerCamelCase       )

         UpperCAmelCase							=     decoder.decode_beams(_lowerCamelCase       )[0]

         self.assertEqual(decoded_decoder[0]					,       decoded_processor.text       )
         self.assertEqual("""</s> <s> </s>"""					,       decoded_processor.text       )
         self.assertEqual(decoded_decoder[-2]					,       decoded_processor.logit_score       )
         self.assertEqual(decoded_decoder[-1]					,       decoded_processor.lm_score       )





      @parameterized.expand([[None], ["""fork"""], ["""spawn"""]]       )
      def        _lowercase				(		self						:					Optional[Any]					,       __lowerCamelCase						:					Any       )    ->   Optional[Any]:






         """simple docstring"""







         UpperCAmelCase							=     self.get_feature_extractor()
         UpperCAmelCase							=     self.get_tokenizer()
         UpperCAmelCase							=     self.get_decoder()

         UpperCAmelCase							=     WavaVecaProcessorWithLM(tokenizer=_lowerCamelCase					,       feature_extractor=_lowerCamelCase					,       decoder=_lowerCamelCase       )

         UpperCAmelCase							=     self._get_dummy_logits()

         # note: pool should be instantiated *after* Wav2Vec2ProcessorWithLM.
         #       otherwise, the LM won't be available to the pool's sub-processes.
         # manual logic used to allow parameterized test for both pool=None and pool=Pool(...)
         if pool_context is None:
            UpperCAmelCase							=     processor.batch_decode(_lowerCamelCase       )
         else:
            with get_context(_lowerCamelCase       ).Pool() as pool:
               UpperCAmelCase							=     processor.batch_decode(_lowerCamelCase					,       _lowerCamelCase       )

         UpperCAmelCase							=     list(_lowerCamelCase       )

         with get_context("""fork"""       ).Pool() as p:
            UpperCAmelCase							=     decoder.decode_beams_batch(_lowerCamelCase					,       _lowerCamelCase       )

         UpperCAmelCase				,  UpperCAmelCase				,  UpperCAmelCase							=     [], [], []
         for beams in decoded_beams:
            texts_decoder.append(beams[0][0]       )
            logit_scores_decoder.append(beams[0][-2]       )
            lm_scores_decoder.append(beams[0][-1]       )

         self.assertListEqual(_lowerCamelCase					,       decoded_processor.text       )
         self.assertListEqual(["""<s> <s> </s>""", """<s> <s> <s>"""]					,       decoded_processor.text       )
         self.assertListEqual(_lowerCamelCase					,       decoded_processor.logit_score       )
         self.assertListEqual(_lowerCamelCase					,       decoded_processor.lm_score       )





      def        _lowercase				(		self						:					List[str]       )    ->   Any:






         """simple docstring"""







         UpperCAmelCase							=     self.get_feature_extractor()
         UpperCAmelCase							=     self.get_tokenizer()
         UpperCAmelCase							=     self.get_decoder()

         UpperCAmelCase							=     WavaVecaProcessorWithLM(tokenizer=_lowerCamelCase					,       feature_extractor=_lowerCamelCase					,       decoder=_lowerCamelCase       )

         UpperCAmelCase							=     self._get_dummy_logits()

         UpperCAmelCase							=     1_5
         UpperCAmelCase							=     -2_0.0
         UpperCAmelCase							=     -4.0

         UpperCAmelCase							=     processor.batch_decode(
             _lowerCamelCase					,       beam_width=_lowerCamelCase					,       beam_prune_logp=_lowerCamelCase					,       token_min_logp=_lowerCamelCase					,       )
         UpperCAmelCase							=     decoded_processor_out.text

         UpperCAmelCase							=     list(_lowerCamelCase       )

         with get_context("""fork"""       ).Pool() as pool:
            UpperCAmelCase							=     decoder.decode_beams_batch(
                _lowerCamelCase					,       _lowerCamelCase					,       beam_width=_lowerCamelCase					,       beam_prune_logp=_lowerCamelCase					,       token_min_logp=_lowerCamelCase					,       )

         UpperCAmelCase							=     [d[0][0] for d in decoded_decoder_out]
         UpperCAmelCase							=     [d[0][2] for d in decoded_decoder_out]
         UpperCAmelCase							=     [d[0][3] for d in decoded_decoder_out]

         self.assertListEqual(_lowerCamelCase					,       _lowerCamelCase       )
         self.assertListEqual(["""</s> <s> <s>""", """<s> <s> <s>"""]					,       _lowerCamelCase       )

         self.assertTrue(np.array_equal(_lowerCamelCase					,       decoded_processor_out.logit_score       )       )
         self.assertTrue(np.allclose([-2_0.0_5_4, -1_8.4_4_7]					,       _lowerCamelCase					,       atol=1e-3       )       )

         self.assertTrue(np.array_equal(_lowerCamelCase					,       decoded_processor_out.lm_score       )       )
         self.assertTrue(np.allclose([-1_5.5_5_4, -1_3.9_4_7_4]					,       _lowerCamelCase					,       atol=1e-3       )       )





      def        _lowercase				(		self						:					Union[str, Any]       )    ->   Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase							=     self.get_feature_extractor()
         UpperCAmelCase							=     self.get_tokenizer()
         UpperCAmelCase							=     self.get_decoder()

         UpperCAmelCase							=     WavaVecaProcessorWithLM(tokenizer=_lowerCamelCase					,       feature_extractor=_lowerCamelCase					,       decoder=_lowerCamelCase       )

         UpperCAmelCase							=     self._get_dummy_logits()

         UpperCAmelCase							=     2.0
         UpperCAmelCase							=     5.0
         UpperCAmelCase							=     -2_0.0
         UpperCAmelCase							=     True

         UpperCAmelCase							=     processor.batch_decode(
             _lowerCamelCase					,       alpha=_lowerCamelCase					,       beta=_lowerCamelCase					,       unk_score_offset=_lowerCamelCase					,       lm_score_boundary=_lowerCamelCase					,       )
         UpperCAmelCase							=     decoded_processor_out.text

         UpperCAmelCase							=     list(_lowerCamelCase       )
         decoder.reset_params(
             alpha=_lowerCamelCase					,       beta=_lowerCamelCase					,       unk_score_offset=_lowerCamelCase					,       lm_score_boundary=_lowerCamelCase					,       )

         with get_context("""fork"""       ).Pool() as pool:
            UpperCAmelCase							=     decoder.decode_beams_batch(
                _lowerCamelCase					,       _lowerCamelCase					,       )

         UpperCAmelCase							=     [d[0][0] for d in decoded_decoder_out]

         self.assertListEqual(_lowerCamelCase					,       _lowerCamelCase       )
         self.assertListEqual(["""<s> </s> <s> </s> </s>""", """</s> </s> <s> </s> </s>"""]					,       _lowerCamelCase       )
         UpperCAmelCase							=     processor.decoder.model_container[processor.decoder._model_key]
         self.assertEqual(lm_model.alpha					,       2.0       )
         self.assertEqual(lm_model.beta					,       5.0       )
         self.assertEqual(lm_model.unk_score_offset					,       -2_0.0       )
         self.assertEqual(lm_model.score_boundary					,       _lowerCamelCase       )





      def        _lowercase				(		self						:					int       )    ->   Optional[int]:






         """simple docstring"""







         UpperCAmelCase							=     WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm"""       )

         UpperCAmelCase							=     processor.decoder.model_container[processor.decoder._model_key]
         UpperCAmelCase							=     Path(language_model._kenlm_model.path.decode("""utf-8"""       )       ).parent.parent.absolute()

         UpperCAmelCase							=     os.listdir(_lowerCamelCase       )
         UpperCAmelCase							=     ["""alphabet.json""", """language_model"""]

         downloaded_decoder_files.sort()
         expected_decoder_files.sort()

         # test that only decoder relevant files from
         # https://huggingface.co/hf-internal-testing/processor_with_lm/tree/main
         # are downloaded and none of the rest (e.g. README.md, ...)
         self.assertListEqual(_lowerCamelCase					,       _lowerCamelCase       )





      def        _lowercase				(		self						:					str       )    ->   List[Any]:






         """simple docstring"""







         UpperCAmelCase							=     snapshot_download("""hf-internal-testing/processor_with_lm"""       )

         UpperCAmelCase							=     WavaVecaProcessorWithLM.from_pretrained(_lowerCamelCase       )

         UpperCAmelCase							=     processor.decoder.model_container[processor.decoder._model_key]
         UpperCAmelCase							=     Path(language_model._kenlm_model.path.decode("""utf-8"""       )       ).parent.parent.absolute()

         UpperCAmelCase							=     os.listdir(_lowerCamelCase       )
         UpperCAmelCase							=     os.listdir(_lowerCamelCase       )

         local_decoder_files.sort()
         expected_decoder_files.sort()

         # test that both decoder form hub and local files in cache are the same
         self.assertListEqual(_lowerCamelCase					,       _lowerCamelCase       )





      def        _lowercase				(		self						:					str       )    ->   Dict:






         """simple docstring"""







         UpperCAmelCase							=     WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm"""       )
         UpperCAmelCase							=     AutoProcessor.from_pretrained("""hf-internal-testing/processor_with_lm"""       )

         UpperCAmelCase							=     floats_list((3, 1_0_0_0)       )

         UpperCAmelCase							=     processor_wavaveca(_lowerCamelCase					,       return_tensors="""np"""       )
         UpperCAmelCase							=     processor_auto(_lowerCamelCase					,       return_tensors="""np"""       )

         for key in input_wavaveca.keys():
            self.assertAlmostEqual(input_wavaveca[key].sum()					,       input_auto[key].sum()					,       delta=1e-2       )

         UpperCAmelCase							=     self._get_dummy_logits()

         UpperCAmelCase							=     processor_wavaveca.batch_decode(_lowerCamelCase       )
         UpperCAmelCase							=     processor_auto.batch_decode(_lowerCamelCase       )

         self.assertListEqual(decoded_wavaveca.text					,       decoded_auto.text       )





      def        _lowercase				(		self						:					Dict       )    ->   Dict:






         """simple docstring"""







         UpperCAmelCase							=     self.get_feature_extractor()
         UpperCAmelCase							=     self.get_tokenizer()
         UpperCAmelCase							=     self.get_decoder()

         UpperCAmelCase							=     WavaVecaProcessorWithLM(tokenizer=_lowerCamelCase					,       feature_extractor=_lowerCamelCase					,       decoder=_lowerCamelCase       )

         self.assertListEqual(
             processor.model_input_names					,       feature_extractor.model_input_names					,       msg="""`processor` and `feature_extractor` model input names do not match"""					,       )





      @staticmethod
      def        _lowercase				(		__lowerCamelCase						:					int					,       __lowerCamelCase						:					int       )    ->   Optional[Any]:






         """simple docstring"""







         UpperCAmelCase							=     [d[key] for d in offsets]
         return retrieved_list





      def        _lowercase				(		self						:					Any       )    ->   Optional[int]:






         """simple docstring"""







         UpperCAmelCase							=     WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm"""       )
         UpperCAmelCase							=     self._get_dummy_logits()[0]

         UpperCAmelCase							=     processor.decode(_lowerCamelCase					,       output_word_offsets=_lowerCamelCase       )
         # check Wav2Vec2CTCTokenizerOutput keys for word
         self.assertEqual(len(outputs.keys()       )					,       4       )
         self.assertTrue("""text""" in outputs       )
         self.assertTrue("""word_offsets""" in outputs       )
         self.assertTrue(isinstance(_lowerCamelCase					,       _lowerCamelCase       )       )

         self.assertEqual(""" """.join(self.get_from_offsets(outputs["""word_offsets"""]					,       """word"""       )       )					,       outputs.text       )
         self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""]					,       """word"""       )					,       ["""<s>""", """<s>""", """</s>"""]       )
         self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""]					,       """start_offset"""       )					,       [0, 2, 4]       )
         self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""]					,       """end_offset"""       )					,       [1, 3, 5]       )





      def        _lowercase				(		self						:					Optional[Any]       )    ->   Optional[Any]:






         """simple docstring"""







         UpperCAmelCase							=     WavaVecaProcessorWithLM.from_pretrained("""hf-internal-testing/processor_with_lm"""       )
         UpperCAmelCase							=     self._get_dummy_logits()

         UpperCAmelCase							=     processor.batch_decode(_lowerCamelCase					,       output_word_offsets=_lowerCamelCase       )

         # check Wav2Vec2CTCTokenizerOutput keys for word
         self.assertEqual(len(outputs.keys()       )					,       4       )
         self.assertTrue("""text""" in outputs       )
         self.assertTrue("""word_offsets""" in outputs       )
         self.assertTrue(isinstance(_lowerCamelCase					,       _lowerCamelCase       )       )

         self.assertListEqual(
             [""" """.join(self.get_from_offsets(_lowerCamelCase					,       """word"""       )       ) for o in outputs["""word_offsets"""]]					,       outputs.text       )
         self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""][0]					,       """word"""       )					,       ["""<s>""", """<s>""", """</s>"""]       )
         self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""][0]					,       """start_offset"""       )					,       [0, 2, 4]       )
         self.assertListEqual(self.get_from_offsets(outputs["""word_offsets"""][0]					,       """end_offset"""       )					,       [1, 3, 5]       )





      @slow
      @require_torch
      @require_torchaudio
      def        _lowercase				(		self						:					Union[str, Any]       )    ->   List[Any]:






         """simple docstring"""







         import torch

         UpperCAmelCase							=     load_dataset("""common_voice"""					,       """en"""					,       split="""train"""					,       streaming=_lowerCamelCase       )
         UpperCAmelCase							=     ds.cast_column("""audio"""					,       datasets.Audio(sampling_rate=1_6_0_0_0       )       )
         UpperCAmelCase							=     iter(_lowerCamelCase       )
         UpperCAmelCase							=     next(_lowerCamelCase       )

         UpperCAmelCase							=     AutoProcessor.from_pretrained("""patrickvonplaten/wav2vec2-base-100h-with-lm"""       )
         UpperCAmelCase							=     WavaVecaForCTC.from_pretrained("""patrickvonplaten/wav2vec2-base-100h-with-lm"""       )

         # compare to filename `common_voice_en_100038.mp3` of dataset viewer on https://huggingface.co/datasets/common_voice/viewer/en/train
         UpperCAmelCase							=     processor(sample["""audio"""]["""array"""]					,       return_tensors="""pt"""       ).input_values

         with torch.no_grad():
            UpperCAmelCase							=     model(_lowerCamelCase       ).logits.cpu().numpy()

         UpperCAmelCase							=     processor.decode(logits[0]					,       output_word_offsets=_lowerCamelCase       )

         UpperCAmelCase							=     model.config.inputs_to_logits_ratio / processor.feature_extractor.sampling_rate
         UpperCAmelCase							=     [
             {
                 """start_time""": d["""start_offset"""] * time_offset,
                 """end_time""": d["""end_offset"""] * time_offset,
                 """word""": d["""word"""],
             }
             for d in output["""word_offsets"""]
         ]

         UpperCAmelCase							=     """WHY DOES MILISANDRA LOOK LIKE SHE WANTS TO CONSUME JOHN SNOW ON THE RIVER AT THE WALL"""

         # output words
         self.assertEqual(""" """.join(self.get_from_offsets(_lowerCamelCase					,       """word"""       )       )					,       _lowerCamelCase       )
         self.assertEqual(""" """.join(self.get_from_offsets(_lowerCamelCase					,       """word"""       )       )					,       output.text       )

         # output times
         UpperCAmelCase							=     torch.tensor(self.get_from_offsets(_lowerCamelCase					,       """start_time"""       )       )
         UpperCAmelCase							=     torch.tensor(self.get_from_offsets(_lowerCamelCase					,       """end_time"""       )       )

         # fmt: off
         UpperCAmelCase							=     torch.tensor([1.4_199, 1.6_599, 2.2_599, 3.0, 3.24, 3.5_999, 3.7_999, 4.0_999, 4.26, 4.94, 5.28, 5.6_599, 5.78, 5.94, 6.32, 6.5_399, 6.6_599]       )
         UpperCAmelCase							=     torch.tensor([1.5_399, 1.8_999, 2.9, 3.16, 3.5_399, 3.72, 4.0_199, 4.1_799, 4.76, 5.1_599, 5.5_599, 5.6_999, 5.86, 6.1_999, 6.38, 6.6_199, 6.94]       )
         # fmt: on

         self.assertTrue(torch.allclose(_lowerCamelCase					,       _lowerCamelCase					,       atol=0.01       )       )
         self.assertTrue(torch.allclose(_lowerCamelCase					,       _lowerCamelCase					,       atol=0.01       )       )







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available


__a     =			{"""tokenization_herbert""": ["""HerbertTokenizer"""]}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __a     =			["""HerbertTokenizerFast"""]


if TYPE_CHECKING:
 from .tokenization_herbert import HerbertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_herbert_fast import HerbertTokenizerFast

else:
 import sys

 __a     =			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)



