






'''simple docstring'''







import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text





def 		_UpperCAmelCase   (							__A							:   List[str]      ):
				a_		:  List[str]											=      []
				for line in lines:
								a_		:  Optional[int]											=      re.sub(R'''#.*'''						,		''''''						,		__A      )  # remove comments
								if line:
												filtered_lines.append(__A      )
				a_		:  Union[str, Any]											=      '''\n'''.join(__A      )

				# Make a hash from all this code
				a_		:  int											=      full_str.encode('''utf-8'''      )
				return shaaaa(__A      ).hexdigest()


# get importable module names and hash for caching
__lowerCAmelCase       		=      {
    """csv""": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    """json""": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    """pandas""": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    """parquet""": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    """arrow""": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    """text""": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    """imagefolder""": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    """audiofolder""": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
__lowerCAmelCase       		=      {
    """.csv""": ("""csv""", {}),
    """.tsv""": ("""csv""", {"""sep""": """\t"""}),
    """.json""": ("""json""", {}),
    """.jsonl""": ("""json""", {}),
    """.parquet""": ("""parquet""", {}),
    """.arrow""": ("""arrow""", {}),
    """.txt""": ("""text""", {}),
}
_EXTENSION_TO_MODULE.update({ext: ('''imagefolder''', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('''imagefolder''', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ('''audiofolder''', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('''audiofolder''', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
__lowerCAmelCase       		=      {"""imagefolder""", """audiofolder"""}

# Used to filter data files based on extensions given a module name
__lowerCAmelCase       		=      {}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
		_MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append('''.zip''')




_MODULE_TO_EXTENSIONS["audiofolder"].append('''.zip''')












'''simple docstring'''







import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 	SCREAMING_SNAKE_CASE					(					SCREAMING_SNAKE_CASE_	,   unittest.TestCase       ):
			snake_case__  =				LongformerTokenizer
			snake_case__  =				True
			snake_case__  =				LongformerTokenizerFast
			snake_case__  =				True






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[str]:
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							a_		:  Tuple											=      [
							    '''l''',
							    '''o''',
							    '''w''',
							    '''e''',
							    '''r''',
							    '''s''',
							    '''t''',
							    '''i''',
							    '''d''',
							    '''n''',
							    '''\u0120''',
							    '''\u0120l''',
							    '''\u0120n''',
							    '''\u0120lo''',
							    '''\u0120low''',
							    '''er''',
							    '''\u0120lowest''',
							    '''\u0120newer''',
							    '''\u0120wider''',
							    '''<unk>''',
							]
							a_		:  Optional[Any]											=      dict(zip(__SCREAMING_SNAKE_CASE  ,						range(len(__SCREAMING_SNAKE_CASE					)					)					)					)
							a_		:  Union[str, Any]											=      ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
							a_		:  Any											=      {'''unk_token''': '''<unk>'''}

							a_		:  Union[str, Any]											=      os.path.join(self.tmpdirname  ,						VOCAB_FILES_NAMES['''vocab_file''']					)
							a_		:  Optional[int]											=      os.path.join(self.tmpdirname  ,						VOCAB_FILES_NAMES['''merges_file''']					)
							with open(self.vocab_file  ,						'''w'''  ,						encoding='''utf-8'''					) as fp:
											fp.write(json.dumps(__SCREAMING_SNAKE_CASE					) + '''\n'''					)
							with open(self.merges_file  ,						'''w'''  ,						encoding='''utf-8'''					) as fp:
											fp.write('''\n'''.join(__SCREAMING_SNAKE_CASE					)					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Any  ,						**__SCREAMING_SNAKE_CASE      :			Any					)			->			int:
							kwargs.update(self.special_tokens_map					)
							return self.tokenizer_class.from_pretrained(self.tmpdirname  ,						**__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Union[str, Any]  ,						**__SCREAMING_SNAKE_CASE      :			List[Any]					)			->			List[str]:
							kwargs.update(self.special_tokens_map					)
							return self.rust_tokenizer_class.from_pretrained(self.tmpdirname  ,						**__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Dict  ,						__SCREAMING_SNAKE_CASE      :			List[Any]					)			->			Any:
							a_		:  Union[str, Any]											=      '''lower newer'''
							a_		:  List[Any]											=      '''lower newer'''
							return input_text, output_text






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[str]:
							a_		:  Optional[Any]											=      self.tokenizer_class(self.vocab_file  ,						self.merges_file  ,						**self.special_tokens_map					)
							a_		:  List[str]											=      '''lower newer'''
							a_		:  str											=      ['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
							a_		:  Optional[int]											=      tokenizer.tokenize(__SCREAMING_SNAKE_CASE					)  # , add_prefix_space=True)
							self.assertListEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							a_		:  Dict											=      tokens + [tokenizer.unk_token]
							a_		:  Any											=      [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)  ,						__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			List[str]					)			->			Optional[Any]:
							a_		:  Union[str, Any]											=      self.get_tokenizer()

							self.assertListEqual(tokenizer.encode('''Hello world!'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)  ,						[0, 3_1414, 232, 328, 2]					)
							self.assertListEqual(
							    tokenizer.encode('''Hello world! cécé herlolip 418'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)  ,						[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]  ,						)






			@slow
			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			Optional[Any]:
							a_		:  Dict											=      self.tokenizer_class.from_pretrained('''allenai/longformer-base-4096'''					)

							a_		:  Tuple											=      tokenizer.encode('''sequence builders'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
							a_		:  Union[str, Any]											=      tokenizer.encode('''multi-sequence build'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)

							a_		:  Any											=      tokenizer.encode(
							    '''sequence builders'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE					)
							a_		:  Any											=      tokenizer.encode(
							    '''sequence builders'''  ,						'''multi-sequence build'''  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE					)

							a_		:  List[str]											=      tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE					)
							a_		:  Union[str, Any]											=      tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							assert encoded_sentence == encoded_text_from_decode
							assert encoded_pair == encoded_pair_from_decode






			def 			SCREAMING_SNAKE_CASE      ( self      :			str					)			->			int:
							a_		:  str											=      self.get_tokenizer()

							a_		:  int											=      '''Encode this sequence.'''
							a_		:  List[str]											=      tokenizer.byte_encoder[''' '''.encode('''utf-8'''					)[0]]

							# Testing encoder arguments
							a_		:  Dict											=      tokenizer.encode(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[Any]											=      tokenizer.convert_ids_to_tokens(encoded[0]					)[0]
							self.assertNotEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							a_		:  Dict											=      tokenizer.encode(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE					)
							a_		:  Any											=      tokenizer.convert_ids_to_tokens(encoded[0]					)[0]
							self.assertEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							tokenizer.add_special_tokens({'''bos_token''': '''<s>'''}					)
							a_		:  Dict											=      tokenizer.encode(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
							a_		:  Dict											=      tokenizer.convert_ids_to_tokens(encoded[1]					)[0]
							self.assertNotEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							# Testing spaces after special tokens
							a_		:  Optional[Any]											=      '''<mask>'''
							tokenizer.add_special_tokens(
							    {'''mask_token''': AddedToken(__SCREAMING_SNAKE_CASE  ,						lstrip=__SCREAMING_SNAKE_CASE  ,						rstrip=__SCREAMING_SNAKE_CASE					)}					)  # mask token has a left space
							a_		:  Optional[int]											=      tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE					)

							a_		:  List[Any]											=      '''Encode <mask> sequence'''
							a_		:  List[str]											=      '''Encode <mask>sequence'''

							a_		:  int											=      tokenizer.encode(__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[int]											=      encoded.index(__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[int]											=      tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]					)[0]
							self.assertEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)

							a_		:  Union[str, Any]											=      tokenizer.encode(__SCREAMING_SNAKE_CASE					)
							a_		:  Optional[int]											=      encoded.index(__SCREAMING_SNAKE_CASE					)
							a_		:  str											=      tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]					)[0]
							self.assertNotEqual(__SCREAMING_SNAKE_CASE  ,						__SCREAMING_SNAKE_CASE					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[int]					)			->			Union[str, Any]:
							pass






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			Any:
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
															a_		:  Any											=      self.rust_tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)
															a_		:  Any											=      self.tokenizer_class.from_pretrained(__SCREAMING_SNAKE_CASE  ,						**__SCREAMING_SNAKE_CASE					)
															a_		:  str											=      '''A, <mask> AllenNLP sentence.'''
															a_		:  List[Any]											=      tokenizer_r.encode_plus(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						return_token_type_ids=__SCREAMING_SNAKE_CASE					)
															a_		:  Dict											=      tokenizer_p.encode_plus(__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE  ,						return_token_type_ids=__SCREAMING_SNAKE_CASE					)

															# token_type_ids should put 0 everywhere
															self.assertEqual(sum(tokens_r['''token_type_ids''']					)  ,						sum(tokens_p['''token_type_ids''']					)					)

															# attention_mask should put 1 everywhere, so sum over length should be 1
															self.assertEqual(
															    sum(tokens_r['''attention_mask''']					) / len(tokens_r['''attention_mask''']					)  ,						sum(tokens_p['''attention_mask''']					) / len(tokens_p['''attention_mask''']					)  ,						)

															a_		:  str											=      tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']					)
															a_		:  Tuple											=      tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']					)

															# Rust correctly handles the space before the mask while python doesnt
															self.assertSequenceEqual(tokens_p['''input_ids''']  ,						[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]					)
															self.assertSequenceEqual(tokens_r['''input_ids''']  ,						[0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]					)

															self.assertSequenceEqual(
															    __SCREAMING_SNAKE_CASE  ,						['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']					)
															self.assertSequenceEqual(
															    __SCREAMING_SNAKE_CASE  ,						['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']					)






			def 			SCREAMING_SNAKE_CASE      ( self      :			Optional[Any]					)			->			List[Any]:
							for trim_offsets, add_prefix_space in itertools.product([True, False]  ,						repeat=2					):
											a_		:  Any											=      self.rust_tokenizer_class.from_pretrained(
											    self.tmpdirname  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)

											a_		:  Union[str, Any]											=      json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()					)
											a_		:  str											=      json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()					)

											self.assertEqual(pre_tokenizer_state['''add_prefix_space''']  ,						__SCREAMING_SNAKE_CASE					)

											self.assertEqual(post_processor_state['''add_prefix_space''']  ,						__SCREAMING_SNAKE_CASE					)
											self.assertEqual(post_processor_state['''trim_offsets''']  ,						__SCREAMING_SNAKE_CASE					)







			def 			SCREAMING_SNAKE_CASE      ( self      :			int					)			->			Optional[int]:
							# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
							# `trim_offsets`
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
															a_		:  Dict											=      '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
															a_		:  Union[str, Any]											=      f'{text_of_1_token} {text_of_1_token}'

															a_		:  Any											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  Optional[int]											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(len(__SCREAMING_SNAKE_CASE					) + 1, len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  Any											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  str											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(len(__SCREAMING_SNAKE_CASE					) + 1, len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  int											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  Union[str, Any]											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(len(__SCREAMING_SNAKE_CASE					), len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  Tuple											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  Any											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(len(__SCREAMING_SNAKE_CASE					), len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  Union[str, Any]											=      f' {text}'

															# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
															#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
															# )
															# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
															# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
															# self.assertEqual(
															#     encoding.offset_mapping[1],
															#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
															# )

															a_		:  str											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  int											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(1, 1 + len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(1 + len(__SCREAMING_SNAKE_CASE					) + 1, 1 + len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  int											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  str											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, 1 + len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(1 + len(__SCREAMING_SNAKE_CASE					), 1 + len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)

															a_		:  Optional[int]											=      self.rust_tokenizer_class.from_pretrained(
															    __SCREAMING_SNAKE_CASE  ,						use_fast=__SCREAMING_SNAKE_CASE  ,						add_prefix_space=__SCREAMING_SNAKE_CASE  ,						trim_offsets=__SCREAMING_SNAKE_CASE					)
															a_		:  int											=      tokenizer_r(__SCREAMING_SNAKE_CASE  ,						return_offsets_mapping=__SCREAMING_SNAKE_CASE  ,						add_special_tokens=__SCREAMING_SNAKE_CASE					)
															self.assertEqual(encoding.offset_mapping[0]  ,						(0, 1 + len(__SCREAMING_SNAKE_CASE					))					)
															self.assertEqual(
															    encoding.offset_mapping[1]  ,						(1 + len(__SCREAMING_SNAKE_CASE					), 1 + len(__SCREAMING_SNAKE_CASE					) + 1 + len(__SCREAMING_SNAKE_CASE					))  ,						)




