



"""simple docstring"""






import itertools
import random
import unittest

import numpy as np

from transformers import is_speech_available
from transformers.testing_utils import require_torch, require_torchaudio

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
	from transformers import SpeechaTextFeatureExtractor

__snake_case							:     Union[str, Any]   =					random.Random()


def     _lowercase       (						__snake_case      ,__snake_case=1.0      ,__snake_case=None      ,__snake_case=None			)		->     List[Any]:
			if rng is None:
						__lowerCAmelCase   :						Dict				= global_rng

			__lowerCAmelCase   :						Tuple				= []
			for batch_idx in range(shape[0]			):
						values.append([]			)
						for _ in range(shape[1]			):
									values[-1].append(rng.random() * scale			)

			return values



@require_torch
@require_torchaudio
class     A__     (						unittest.TestCase							):




	'''simple docstring'''





	def __init__(  self:	Optional[Any] ,    _SCREAMING_SNAKE_CASE:	Any ,    _SCREAMING_SNAKE_CASE:	List[Any]=7 ,    _SCREAMING_SNAKE_CASE:	int=400 ,    _SCREAMING_SNAKE_CASE:	List[str]=2000 ,    _SCREAMING_SNAKE_CASE:	Optional[Any]=24 ,    _SCREAMING_SNAKE_CASE:	Dict=24 ,    _SCREAMING_SNAKE_CASE:	Optional[int]=0.0 ,    _SCREAMING_SNAKE_CASE:	Any=1_6000 ,    _SCREAMING_SNAKE_CASE:	int=True ,    _SCREAMING_SNAKE_CASE:	Union[str, Any]=True ,    )       -> List[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						Any				= parent
				__lowerCAmelCase   :						str				= batch_size
				__lowerCAmelCase   :						List[str]				= min_seq_length
				__lowerCAmelCase   :						Any				= max_seq_length
				__lowerCAmelCase   :						Dict				= (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
				__lowerCAmelCase   :						Union[str, Any]				= feature_size
				__lowerCAmelCase   :						int				= num_mel_bins
				__lowerCAmelCase   :						Optional[Any]				= padding_value
				__lowerCAmelCase   :						List[Any]				= sampling_rate
				__lowerCAmelCase   :						Dict				= return_attention_mask
				__lowerCAmelCase   :						int				= do_normalize

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Optional[int]:

				"""simple docstring"""

				return {
				    "feature_size": self.feature_size,
				    "num_mel_bins": self.num_mel_bins,
				    "padding_value": self.padding_value,
				    "sampling_rate": self.sampling_rate,
				    "return_attention_mask": self.return_attention_mask,
				    "do_normalize": self.do_normalize,
				}







	def 							_SCREAMING_SNAKE_CASE     (  self:	Tuple ,    _SCREAMING_SNAKE_CASE:	Tuple=False ,    _SCREAMING_SNAKE_CASE:	int=False)       -> Optional[int]:

				"""simple docstring"""

				def _flatten(_SCREAMING_SNAKE_CASE:	Optional[Any]):
							return list(itertools.chain(*_SCREAMING_SNAKE_CASE))

				if equal_length:
							__lowerCAmelCase   :						Tuple				= [floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
				else:
							# make sure that inputs increase in size
							__lowerCAmelCase   :						List[Any]				= [
							    floats_list((x, self.feature_size))
							    for x in range(self.min_seq_length ,    self.max_seq_length ,    self.seq_length_diff)
							]
				if numpify:
							__lowerCAmelCase   :						Any				= [np.asarray(_SCREAMING_SNAKE_CASE) for x in speech_inputs]
				return speech_inputs






@require_torch
@require_torchaudio
class     A__     (						__SCREAMING_SNAKE_CASE						,	unittest.TestCase							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE					   =			SpeechaTextFeatureExtractor if is_speech_available() else None

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any])       -> Tuple:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= SpeechaTextFeatureExtractionTester(self)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[Any] ,    _SCREAMING_SNAKE_CASE:	Optional[Any])       -> str:

				"""simple docstring"""

				self.assertTrue(np.all(np.mean(_SCREAMING_SNAKE_CASE ,    axis=0) < 1e-3))
				self.assertTrue(np.all(np.abs(np.var(_SCREAMING_SNAKE_CASE ,    axis=0) - 1) < 1e-3))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any])       -> int:

				"""simple docstring"""

				__lowerCAmelCase   :						Dict				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				# create three inputs of length 800, 1000, and 1200
				__lowerCAmelCase   :						Tuple				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						Optional[int]				= [np.asarray(_SCREAMING_SNAKE_CASE) for speech_input in speech_inputs]

				# Test feature size
				__lowerCAmelCase   :						Optional[int]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    padding=_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				self.assertTrue(input_features.ndim == 3)
				self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size)

				# Test not batched input
				__lowerCAmelCase   :						Optional[Any]				= feature_extractor(speech_inputs[0] ,    return_tensors="np").input_features
				__lowerCAmelCase   :						Any				= feature_extractor(np_speech_inputs[0] ,    return_tensors="np").input_features
				self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    atol=1e-3))

				# Test batched
				__lowerCAmelCase   :						Tuple				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				__lowerCAmelCase   :						Tuple				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				for enc_seq_a, enc_seq_a in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    atol=1e-3))

				# Test 2-D numpy arrays are batched.
				__lowerCAmelCase   :						List[str]				= [floats_list((1, x))[0] for x in (800, 800, 800)]
				__lowerCAmelCase   :						Dict				= np.asarray(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						Optional[int]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				__lowerCAmelCase   :						Optional[Any]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="np").input_features
				for enc_seq_a, enc_seq_a in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							self.assertTrue(np.allclose(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    atol=1e-3))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Union[str, Any])       -> List[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						Tuple				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]

				__lowerCAmelCase   :						Optional[Any]				= ["longest", "max_length", "do_not_pad"]
				__lowerCAmelCase   :						str				= [None, 16, None]
				for max_length, padding in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							__lowerCAmelCase   :						str				= feature_extractor(
							    _SCREAMING_SNAKE_CASE ,    padding=_SCREAMING_SNAKE_CASE ,    max_length=_SCREAMING_SNAKE_CASE ,    return_attention_mask=_SCREAMING_SNAKE_CASE)
							__lowerCAmelCase   :						Optional[Any]				= inputs.input_features
							__lowerCAmelCase   :						Optional[Any]				= inputs.attention_mask
							__lowerCAmelCase   :						Tuple				= [np.sum(_SCREAMING_SNAKE_CASE) for x in attention_mask]

							self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]])
							self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]])
							self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						Any				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]

				__lowerCAmelCase   :						Dict				= ["longest", "max_length", "do_not_pad"]
				__lowerCAmelCase   :						List[Any]				= [None, 16, None]
				for max_length, padding in zip(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE):
							__lowerCAmelCase   :						List[str]				= feature_extractor(
							    _SCREAMING_SNAKE_CASE ,    max_length=_SCREAMING_SNAKE_CASE ,    padding=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE)
							__lowerCAmelCase   :						List[str]				= inputs.input_features
							__lowerCAmelCase   :						Dict				= inputs.attention_mask
							__lowerCAmelCase   :						Dict				= [np.sum(_SCREAMING_SNAKE_CASE) for x in attention_mask]

							self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]])
							self.assertTrue(input_features[0][fbank_feat_lengths[0] :].sum() < 1e-6)
							self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]])
							self.assertTrue(input_features[0][fbank_feat_lengths[1] :].sum() < 1e-6)
							self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]])

	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> Any:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						str				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						str				= feature_extractor(
				    _SCREAMING_SNAKE_CASE ,    padding="max_length" ,    max_length=4 ,    truncation=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE ,    )
				__lowerCAmelCase   :						List[str]				= inputs.input_features
				__lowerCAmelCase   :						Dict				= inputs.attention_mask
				__lowerCAmelCase   :						Any				= np.sum(attention_mask == 1 ,    axis=1)

				self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
				self._check_zero_mean_unit_variance(input_features[1])
				self._check_zero_mean_unit_variance(input_features[2])

	def 							_SCREAMING_SNAKE_CASE     (  self:	int)       -> List[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						Union[str, Any]				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						List[Any]				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						Any				= feature_extractor(
				    _SCREAMING_SNAKE_CASE ,    padding="longest" ,    max_length=4 ,    truncation=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE ,    )
				__lowerCAmelCase   :						List[str]				= inputs.input_features
				__lowerCAmelCase   :						Optional[Any]				= inputs.attention_mask
				__lowerCAmelCase   :						Optional[Any]				= np.sum(attention_mask == 1 ,    axis=1)

				self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
				self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]])
				self._check_zero_mean_unit_variance(input_features[2])

				# make sure that if max_length < longest -> then pad to max_length
				self.assertEqual(input_features.shape ,    (3, 4, 24))

				__lowerCAmelCase   :						List[str]				= [floats_list((1, x))[0] for x in range(800 ,    1400 ,    200)]
				__lowerCAmelCase   :						List[str]				= feature_extractor(
				    _SCREAMING_SNAKE_CASE ,    padding="longest" ,    max_length=16 ,    truncation=_SCREAMING_SNAKE_CASE ,    return_tensors="np" ,    return_attention_mask=_SCREAMING_SNAKE_CASE ,    )
				__lowerCAmelCase   :						Optional[Any]				= inputs.input_features
				__lowerCAmelCase   :						Optional[int]				= inputs.attention_mask
				__lowerCAmelCase   :						str				= np.sum(attention_mask == 1 ,    axis=1)

				self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]])
				self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]])
				self._check_zero_mean_unit_variance(input_features[2])

				# make sure that if max_length < longest -> then pad to max_length
				self.assertEqual(input_features.shape ,    (3, 6, 24))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Dict:

				"""simple docstring"""

				import torch

				__lowerCAmelCase   :						Dict				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						List[str]				= np.random.rand(100 ,    32).astype(np.floataa)
				__lowerCAmelCase   :						List[Any]				= np_speech_inputs.tolist()

				for inputs in [py_speech_inputs, np_speech_inputs]:
							__lowerCAmelCase   :						Any				= feature_extractor.pad([{"input_features": inputs}] ,    return_tensors="np")
							self.assertTrue(np_processed.input_features.dtype == np.floataa)
							__lowerCAmelCase   :						List[Any]				= feature_extractor.pad([{"input_features": inputs}] ,    return_tensors="pt")
							self.assertTrue(pt_processed.input_features.dtype == torch.floataa)

	def 							_SCREAMING_SNAKE_CASE     (  self:	str ,    _SCREAMING_SNAKE_CASE:	List[Any])       -> Optional[Any]:

				"""simple docstring"""

				from datasets import load_dataset

				__lowerCAmelCase   :						Any				= load_dataset("hf-internal-testing/librispeech_asr_dummy" ,    "clean" ,    split="validation")
				# automatic decoding with librispeech
				__lowerCAmelCase   :						List[Any]				= ds.sort("id").select(range(_SCREAMING_SNAKE_CASE))[:num_samples]["audio"]

				return [x["array"] for x in speech_samples]







	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> Any:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= np.array([
				    -1.5745, -1.7713, -1.7020, -1.6069, -1.2250, -1.1105, -0.9072, -0.8241,
				    -1.2310, -0.8098, -0.3320, -0.4101, -0.7985, -0.4996, -0.8213, -0.9128,
				    -1.0420, -1.1286, -1.0440, -0.7999, -0.8405, -1.2275, -1.5443, -1.4625,
				])
				# fmt: on

				__lowerCAmelCase   :						str				= self._load_datasamples(1)
				__lowerCAmelCase   :						Any				= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
				__lowerCAmelCase   :						List[str]				= feature_extractor(_SCREAMING_SNAKE_CASE ,    return_tensors="pt").input_features
				self.assertEquals(input_features.shape ,    (1, 584, 24))
				self.assertTrue(np.allclose(input_features[0, 0, :30] ,    _SCREAMING_SNAKE_CASE ,    atol=1e-4))





"""simple docstring"""






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class     A__     (						__SCREAMING_SNAKE_CASE						,	unittest.TestCase							):




	'''simple docstring'''





	SCREAMING_SNAKE_CASE					   =			RoCBertTokenizer
	SCREAMING_SNAKE_CASE					   =			None
	SCREAMING_SNAKE_CASE					   =			False
	SCREAMING_SNAKE_CASE					   =			True
	SCREAMING_SNAKE_CASE					   =			filter_non_english

	def 							_SCREAMING_SNAKE_CASE     (  self:	int)       -> Optional[Any]:

				"""simple docstring"""

				super().setUp()

				__lowerCAmelCase   :						Dict				= ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
				__lowerCAmelCase   :						str				= {}
				__lowerCAmelCase   :						List[Any]				= {}
				for i, value in enumerate(_SCREAMING_SNAKE_CASE):
							__lowerCAmelCase   :						int				= i
							__lowerCAmelCase   :						List[str]				= i
				__lowerCAmelCase   :						int				= os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES["vocab_file"])
				__lowerCAmelCase   :						Union[str, Any]				= os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES["word_shape_file"])
				__lowerCAmelCase   :						Any				= os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES["word_pronunciation_file"])
				with open(self.vocab_file ,    "w" ,    encoding="utf-8") as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))
				with open(self.word_shape_file ,    "w" ,    encoding="utf-8") as word_shape_writer:
							json.dump(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    ensure_ascii=_SCREAMING_SNAKE_CASE)
				with open(self.word_pronunciation_file ,    "w" ,    encoding="utf-8") as word_pronunciation_writer:
							json.dump(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    ensure_ascii=_SCREAMING_SNAKE_CASE)

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> str:

				"""simple docstring"""

				__lowerCAmelCase   :						Dict				= self.tokenizer_class(self.vocab_file ,    self.word_shape_file ,    self.word_pronunciation_file)

				__lowerCAmelCase   :						List[Any]				= tokenizer.tokenize("你好[SEP]你是谁")
				self.assertListEqual(_SCREAMING_SNAKE_CASE ,    ["你", "好", "[SEP]", "你", "是", "谁"])
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE) ,    [5, 6, 2, 5, 7, 8])
				self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_SCREAMING_SNAKE_CASE) ,    [5, 6, 2, 5, 7, 8])
				self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_SCREAMING_SNAKE_CASE) ,    [5, 6, 2, 5, 7, 8])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> int:

				"""simple docstring"""

				__lowerCAmelCase   :						List[str]				= RoCBertBasicTokenizer()

				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz") ,    ["ah", "\u535A", "\u63A8", "zz"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[Any])       -> Tuple:

				"""simple docstring"""

				__lowerCAmelCase   :						int				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ") ,    ["hello", "!", "how", "are", "you", "?"])
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo") ,    ["hello"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> List[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						Optional[int]				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE ,    strip_accents=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ") ,    ["hällo", "!", "how", "are", "you", "?"])
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo") ,    ["h\u00E9llo"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Tuple)       -> Dict:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE ,    strip_accents=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ") ,    ["hallo", "!", "how", "are", "you", "?"])
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo") ,    ["hello"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> Dict:

				"""simple docstring"""

				__lowerCAmelCase   :						Dict				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ") ,    ["hallo", "!", "how", "are", "you", "?"])
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo") ,    ["hello"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> List[str]:

				"""simple docstring"""

				__lowerCAmelCase   :						Any				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ") ,    ["HeLLo", "!", "how", "Are", "yoU", "?"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Optional[Any])       -> Tuple:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE ,    strip_accents=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ") ,    ["HäLLo", "!", "how", "Are", "yoU", "?"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	Any)       -> Dict:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE ,    strip_accents=_SCREAMING_SNAKE_CASE)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ") ,    ["HaLLo", "!", "how", "Are", "yoU", "?"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> str:

				"""simple docstring"""

				__lowerCAmelCase   :						Optional[Any]				= RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE ,    never_split=["[UNK]"])

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]") ,    ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase   :						str				= ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

				__lowerCAmelCase   :						Union[str, Any]				= {}
				for i, token in enumerate(_SCREAMING_SNAKE_CASE):
							__lowerCAmelCase   :						Optional[int]				= i
				__lowerCAmelCase   :						Any				= RoCBertWordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE ,    unk_token="[UNK]")

				self.assertListEqual(tokenizer.tokenize("") ,    [])

				self.assertListEqual(tokenizer.tokenize("unwanted running") ,    ["un", "##want", "##ed", "runn", "##ing"])

				self.assertListEqual(tokenizer.tokenize("unwantedX running") ,    ["[UNK]", "runn", "##ing"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> int:

				"""simple docstring"""

				self.assertTrue(_is_whitespace(" "))
				self.assertTrue(_is_whitespace("\t"))
				self.assertTrue(_is_whitespace("\r"))
				self.assertTrue(_is_whitespace("\n"))
				self.assertTrue(_is_whitespace("\u00A0"))

				self.assertFalse(_is_whitespace("A"))
				self.assertFalse(_is_whitespace("-"))

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any])       -> Dict:

				"""simple docstring"""

				self.assertTrue(_is_control("\u0005"))

				self.assertFalse(_is_control("A"))
				self.assertFalse(_is_control(" "))
				self.assertFalse(_is_control("\t"))
				self.assertFalse(_is_control("\r"))

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[Any])       -> str:

				"""simple docstring"""

				self.assertTrue(_is_punctuation("-"))
				self.assertTrue(_is_punctuation("$"))
				self.assertTrue(_is_punctuation("`"))
				self.assertTrue(_is_punctuation("."))

				self.assertFalse(_is_punctuation("A"))
				self.assertFalse(_is_punctuation(" "))

	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> Tuple:

				"""simple docstring"""

				__lowerCAmelCase   :						Dict				= self.get_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(_SCREAMING_SNAKE_CASE) for t in ["Test", "\xad", "test"]] ,    [["[UNK]"], [], ["[UNK]"]])

				if self.test_rust_tokenizer:
							__lowerCAmelCase   :						List[str]				= self.get_rust_tokenizer()
							self.assertListEqual(
							    [rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE) for t in ["Test", "\xad", "test"]] ,    [["[UNK]"], [], ["[UNK]"]])

	def 							_SCREAMING_SNAKE_CASE     (  self:	int)       -> List[str]:

				"""simple docstring"""

				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
										__lowerCAmelCase   :						List[Any]				= self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						Optional[Any]				= F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
										__lowerCAmelCase   :						List[Any]				= tokenizer_r.encode_plus(
										    _SCREAMING_SNAKE_CASE ,    return_attention_mask=_SCREAMING_SNAKE_CASE ,    return_token_type_ids=_SCREAMING_SNAKE_CASE ,    return_offsets_mapping=_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE ,    )

										__lowerCAmelCase   :						Optional[Any]				= tokenizer_r.do_lower_case if hasattr(_SCREAMING_SNAKE_CASE ,    "do_lower_case") else False
										__lowerCAmelCase   :						int				= (
										    [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "A"),
										        ((1, 2), ","),
										        ((3, 5), "na"),
										        ((5, 6), "##ï"),
										        ((6, 8), "##ve"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "Allen"),
										        ((21, 23), "##NL"),
										        ((23, 24), "##P"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										    if not do_lower_case
										    else [
										        ((0, 0), tokenizer_r.cls_token),
										        ((0, 1), "a"),
										        ((1, 2), ","),
										        ((3, 8), "naive"),
										        ((9, 15), tokenizer_r.mask_token),
										        ((16, 21), "allen"),
										        ((21, 23), "##nl"),
										        ((23, 24), "##p"),
										        ((25, 33), "sentence"),
										        ((33, 34), "."),
										        ((0, 0), tokenizer_r.sep_token),
										    ]
										)

										self.assertEqual(
										    [e[1] for e in expected_results] ,    tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
										self.assertEqual([e[0] for e in expected_results] ,    tokens["offset_mapping"])

	def 							_SCREAMING_SNAKE_CASE     (  self:	List[str])       -> Optional[Any]:

				"""simple docstring"""

				__lowerCAmelCase   :						List[str]				= ["的", "人", "有"]
				__lowerCAmelCase   :						Tuple				= "".join(_SCREAMING_SNAKE_CASE)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
										__lowerCAmelCase   :						Any				= True
										__lowerCAmelCase   :						Dict				= self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						int				= self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						Dict				= tokenizer_p.encode(_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						List[Any]				= tokenizer_r.encode(_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						Optional[Any]				= tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Optional[int]				= tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE)

										# it is expected that each Chinese character is not preceded by "##"
										self.assertListEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)
										self.assertListEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						int				= False
										__lowerCAmelCase   :						List[Any]				= self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						List[Any]				= self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,    **_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						Any				= tokenizer_r.encode(_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						List[Any]				= tokenizer_p.encode(_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						int				= tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Dict				= tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE)

										# it is expected that only the first Chinese character is not preceded by "##".
										__lowerCAmelCase   :						Union[str, Any]				= [
										    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(_SCREAMING_SNAKE_CASE)
										]
										self.assertListEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)
										self.assertListEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)

	@slow
	def 							_SCREAMING_SNAKE_CASE     (  self:	Dict)       -> str:

				"""simple docstring"""

				__lowerCAmelCase   :						Tuple				= self.tokenizer_class(self.vocab_file ,    self.word_shape_file ,    self.word_pronunciation_file)

				__lowerCAmelCase   :						List[Any]				= tokenizer.encode("你好" ,    add_special_tokens=_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						Union[str, Any]				= tokenizer.encode("你是谁" ,    add_special_tokens=_SCREAMING_SNAKE_CASE)

				__lowerCAmelCase   :						List[str]				= tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE)
				__lowerCAmelCase   :						str				= tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)

				assert encoded_sentence == [1] + text + [2]
				assert encoded_pair == [1] + text + [2] + text_a + [2]







	def 							_SCREAMING_SNAKE_CASE     (  self:	str)       -> Optional[int]:

				"""simple docstring"""

				__lowerCAmelCase   :						Union[str, Any]				= self.get_tokenizers(do_lower_case=_SCREAMING_SNAKE_CASE)
				for tokenizer in tokenizers:
							with self.subTest(F"""{tokenizer.__class__.__name__}"""):
										__lowerCAmelCase   :						Dict				= "你好，你是谁"
										__lowerCAmelCase   :						Tuple				= tokenizer.tokenize(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						Optional[Any]				= tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						List[str]				= tokenizer.convert_tokens_to_shape_ids(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						str				= tokenizer.convert_tokens_to_pronunciation_ids(_SCREAMING_SNAKE_CASE)
										__lowerCAmelCase   :						int				= tokenizer.prepare_for_model(
										    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)

										__lowerCAmelCase   :						Union[str, Any]				= tokenizer.encode_plus(_SCREAMING_SNAKE_CASE ,    add_special_tokens=_SCREAMING_SNAKE_CASE)

										self.assertEqual(_SCREAMING_SNAKE_CASE ,    _SCREAMING_SNAKE_CASE)