


import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map


def      a	(							SCREAMING_SNAKE_CASE_      :		Union[dict, list, tuple, torch.Tensor]       ):






       """simple docstring"""





       UpperCamelCase    :	Tuple   	=		[]
       if isinstance(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):
              for v in tree.values():
                     shapes.extend(_fetch_dims(SCREAMING_SNAKE_CASE_       )       )
       elif isinstance(SCREAMING_SNAKE_CASE_							,			(list, tuple)       ):
              for t in tree:
                     shapes.extend(_fetch_dims(SCREAMING_SNAKE_CASE_       )       )
       elif isinstance(SCREAMING_SNAKE_CASE_							,			torch.Tensor       ):
              shapes.append(tree.shape       )
       else:
              raise ValueError('''Not supported'''       )

       return shapes


@torch.jit.ignore
def      a	(							SCREAMING_SNAKE_CASE_      :		int							,			SCREAMING_SNAKE_CASE_      :		Tuple[int, ...]       ):






       """simple docstring"""





       UpperCamelCase    :	Dict   	=		[]
       for d in reversed(SCREAMING_SNAKE_CASE_       ):
              idx.append(flat_idx % d       )
              UpperCamelCase    :	Optional[Any]   	=		flat_idx // d

       return tuple(reversed(SCREAMING_SNAKE_CASE_       )       )


@torch.jit.ignore
def      a	(							SCREAMING_SNAKE_CASE_      :		Sequence[int]							,			SCREAMING_SNAKE_CASE_      :		Sequence[int]							,			SCREAMING_SNAKE_CASE_      :		Sequence[int]							,			SCREAMING_SNAKE_CASE_      :		Optional[Sequence[bool]] = None							,			SCREAMING_SNAKE_CASE_      :		Optional[Sequence[bool]] = None							,			):






       """simple docstring"""





       def reduce_edge_list(SCREAMING_SNAKE_CASE_      :		List[bool]       ) -> None:
              UpperCamelCase    :	Optional[int]   	=		True
              for i in range(len(SCREAMING_SNAKE_CASE_       )       ):
                     UpperCamelCase    :	List[Any]   	=		-1 * (i + 1)
                     l[reversed_idx] &= tally
                     UpperCamelCase    :	Optional[int]   	=		l[reversed_idx]

       if start_edges is None:
              UpperCamelCase    :	Any   	=		[s == 0 for s in start]
              reduce_edge_list(SCREAMING_SNAKE_CASE_       )
       if end_edges is None:
              UpperCamelCase    :	Union[str, Any]   	=		[e == (d - 1) for e, d in zip(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       )]
              reduce_edge_list(SCREAMING_SNAKE_CASE_       )

       # Base cases. Either start/end are empty and we're done, or the final,
       # one-dimensional tensor can be simply sliced
       if len(SCREAMING_SNAKE_CASE_       ) == 0:
              return [()]
       elif len(SCREAMING_SNAKE_CASE_       ) == 1:
              return [(slice(start[0]							,			end[0] + 1       ),)]

       UpperCamelCase    :	List[Tuple[slice, ...]]   	=		[]
       UpperCamelCase    :	List[slice]   	=		[]

       # Dimensions common to start and end can be selected directly
       for s, e in zip(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):
              if s == e:
                     path_list.append(slice(SCREAMING_SNAKE_CASE_							,			s + 1       )       )
              else:
                     break

       UpperCamelCase    :	Tuple[slice, ...]   	=		tuple(SCREAMING_SNAKE_CASE_       )
       UpperCamelCase    :	Dict   	=		len(SCREAMING_SNAKE_CASE_       )

       # start == end, and we're done
       if divergence_idx == len(SCREAMING_SNAKE_CASE_       ):
              return [path]

       def upper() -> Tuple[Tuple[slice, ...], ...]:
              assert start_edges is not None
              assert end_edges is not None

              UpperCamelCase    :	str   	=		start[divergence_idx]
              return tuple(
                  path + (slice(SCREAMING_SNAKE_CASE_							,			sdi + 1       ),) + s
                  for s in _get_minimal_slice_set(
                      start[divergence_idx + 1 :]							,			[d - 1 for d in dims[divergence_idx + 1 :]]							,			dims[divergence_idx + 1 :]							,			start_edges=start_edges[divergence_idx + 1 :]							,			end_edges=[True for _ in end_edges[divergence_idx + 1 :]]							,			)       )

       def lower() -> Tuple[Tuple[slice, ...], ...]:
              assert start_edges is not None
              assert end_edges is not None

              UpperCamelCase    :	str   	=		end[divergence_idx]
              return tuple(
                  path + (slice(SCREAMING_SNAKE_CASE_							,			edi + 1       ),) + s
                  for s in _get_minimal_slice_set(
                      [0 for _ in start[divergence_idx + 1 :]]							,			end[divergence_idx + 1 :]							,			dims[divergence_idx + 1 :]							,			start_edges=[True for _ in start_edges[divergence_idx + 1 :]]							,			end_edges=end_edges[divergence_idx + 1 :]							,			)       )

       # If both start and end are at the edges of the subtree rooted at
       # divergence_idx, we can just select the whole subtree at once
       if start_edges[divergence_idx] and end_edges[divergence_idx]:
              slices.append(path + (slice(start[divergence_idx]							,			end[divergence_idx] + 1       ),)       )
       # If just start is at the edge, we can grab almost all of the subtree,
       # treating only the ragged bottom edge as an edge case
       elif start_edges[divergence_idx]:
              slices.append(path + (slice(start[divergence_idx]							,			end[divergence_idx]       ),)       )
              slices.extend(lower()       )
       # Analogous to the previous case, but the top is ragged this time
       elif end_edges[divergence_idx]:
              slices.extend(upper()       )
              slices.append(path + (slice(start[divergence_idx] + 1							,			end[divergence_idx] + 1       ),)       )
       # If both sides of the range are ragged, we need to handle both sides
       # separately. If there's contiguous meat in between them, we can index it
       # in one big chunk
       else:
              slices.extend(upper()       )
              UpperCamelCase    :	int   	=		end[divergence_idx] - start[divergence_idx]
              if middle_ground > 1:
                     slices.append(path + (slice(start[divergence_idx] + 1							,			end[divergence_idx]       ),)       )
              slices.extend(lower()       )

       return slices


@torch.jit.ignore
def      a	(							SCREAMING_SNAKE_CASE_      :		torch.Tensor							,			SCREAMING_SNAKE_CASE_      :		int							,			SCREAMING_SNAKE_CASE_      :		int							,			SCREAMING_SNAKE_CASE_      :		int       ):






       """simple docstring"""





       UpperCamelCase    :	int   	=		t.shape[:no_batch_dims]
       UpperCamelCase    :	Union[str, Any]   	=		list(_flat_idx_to_idx(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       )       )
       # _get_minimal_slice_set is inclusive
       UpperCamelCase    :	Any   	=		list(_flat_idx_to_idx(flat_end - 1							,			SCREAMING_SNAKE_CASE_       )       )

       # Get an ordered list of slices to perform
       UpperCamelCase    :	Optional[int]   	=		_get_minimal_slice_set(
           SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_							,			)

       UpperCamelCase    :	Tuple   	=		[t[s] for s in slices]

       return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]       ) for s in sliced_tensors]       )


def      a	(							SCREAMING_SNAKE_CASE_      :		Callable							,			SCREAMING_SNAKE_CASE_      :		Dict[str, Any]							,			SCREAMING_SNAKE_CASE_      :		int							,			SCREAMING_SNAKE_CASE_      :		int							,			SCREAMING_SNAKE_CASE_      :		bool = False							,			SCREAMING_SNAKE_CASE_      :		Any = None							,			SCREAMING_SNAKE_CASE_      :		bool = False							,			):






       """simple docstring"""





       if not (len(SCREAMING_SNAKE_CASE_       ) > 0):
              raise ValueError('''Must provide at least one input'''       )

       UpperCamelCase    :	str   	=		[shape[:no_batch_dims] for shape in _fetch_dims(SCREAMING_SNAKE_CASE_       )]
       UpperCamelCase    :	str   	=		tuple([max(SCREAMING_SNAKE_CASE_       ) for s in zip(*SCREAMING_SNAKE_CASE_       )]       )

       def _prep_inputs(SCREAMING_SNAKE_CASE_      :		torch.Tensor       ) -> torch.Tensor:
              if not low_mem:
                     if not sum(t.shape[:no_batch_dims]       ) == no_batch_dims:
                            UpperCamelCase    :	Optional[int]   	=		t.expand(orig_batch_dims + t.shape[no_batch_dims:]       )
                     UpperCamelCase    :	List[str]   	=		t.reshape(-1							,			*t.shape[no_batch_dims:]       )
              else:
                     UpperCamelCase    :	Dict   	=		t.expand(orig_batch_dims + t.shape[no_batch_dims:]       )
              return t

       UpperCamelCase    :	Dict[str, Any]   	=		tensor_tree_map(_prep_inputs							,			SCREAMING_SNAKE_CASE_       )
       UpperCamelCase    :	Union[str, Any]   	=		None
       if _out is not None:
              UpperCamelCase    :	str   	=		tensor_tree_map(lambda SCREAMING_SNAKE_CASE_       : t.view([-1] + list(t.shape[no_batch_dims:]       )       )							,			_out       )

       UpperCamelCase    :	str   	=		1
       for d in orig_batch_dims:
              flat_batch_dim *= d

       UpperCamelCase    :	Optional[Any]   	=		flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

       def _select_chunk(SCREAMING_SNAKE_CASE_      :		torch.Tensor       ) -> torch.Tensor:
              return t[i : i + chunk_size] if t.shape[0] != 1 else t

       UpperCamelCase    :	Optional[int]   	=		0
       UpperCamelCase    :	Tuple   	=		prepped_outputs
       for _ in range(SCREAMING_SNAKE_CASE_       ):
              # Chunk the input
              if not low_mem:
                     UpperCamelCase    :	Optional[Any]   	=		_select_chunk
              else:
                     UpperCamelCase    :	Optional[Any]   	=		partial(
                         _chunk_slice							,			flat_start=SCREAMING_SNAKE_CASE_							,			flat_end=min(SCREAMING_SNAKE_CASE_							,			i + chunk_size       )							,			no_batch_dims=len(SCREAMING_SNAKE_CASE_       )							,			)

              UpperCamelCase    :	Dict[str, Any]   	=		tensor_tree_map(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       )

              # Run the layer on the chunk
              UpperCamelCase    :	Optional[int]   	=		layer(**SCREAMING_SNAKE_CASE_       )

              # Allocate space for the output
              if out is None:
                     UpperCamelCase    :	Any   	=		tensor_tree_map(lambda SCREAMING_SNAKE_CASE_       : t.new_zeros((flat_batch_dim,) + t.shape[1:]       )							,			SCREAMING_SNAKE_CASE_       )

              # Put the chunk in its pre-allocated space
              if isinstance(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):

                     def assign(SCREAMING_SNAKE_CASE_      :		dict							,			SCREAMING_SNAKE_CASE_      :		dict       ) -> None:
                            for k, v in da.items():
                                   if isinstance(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):
                                          assign(SCREAMING_SNAKE_CASE_							,			da[k]       )
                                   else:
                                          if _add_into_out:
                                                 v[i : i + chunk_size] += da[k]
                                          else:
                                                 UpperCamelCase    :	Optional[int]   	=		da[k]

                     assign(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       )
              elif isinstance(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):
                     for xa, xa in zip(SCREAMING_SNAKE_CASE_							,			SCREAMING_SNAKE_CASE_       ):
                            if _add_into_out:
                                   xa[i : i + chunk_size] += xa
                            else:
                                   UpperCamelCase    :	Union[str, Any]   	=		xa
              elif isinstance(SCREAMING_SNAKE_CASE_							,			torch.Tensor       ):
                     if _add_into_out:
                            out[i : i + chunk_size] += output_chunk
                     else:
                            UpperCamelCase    :	Any   	=		output_chunk
              else:
                     raise ValueError('''Not supported'''       )

              i += chunk_size

       UpperCamelCase    :	Tuple   	=		tensor_tree_map(lambda SCREAMING_SNAKE_CASE_       : t.view(orig_batch_dims + t.shape[1:]       )							,			SCREAMING_SNAKE_CASE_       )

       return out



class    UpperCAmelCase_	:
       '''simple docstring'''






       def __init__(     self ,					__SCREAMING_SNAKE_CASE = 512 ,					):




              """simple docstring"""




              UpperCamelCase    :	Dict   	=		max_chunk_size
              UpperCamelCase    :	Optional[int]   	=		None
              UpperCamelCase    :	Optional[tuple]   	=		None






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              logging.info('''Tuning chunk size...'''						)

              if min_chunk_size >= self.max_chunk_size:
                     return min_chunk_size

              UpperCamelCase    :	List[int]   	=		[2**l for l in range(int(math.log(self.max_chunk_size ,					2						)						) + 1						)]
              UpperCamelCase    :	List[Any]   	=		[c for c in candidates if c > min_chunk_size]
              UpperCamelCase    :	str   	=		[min_chunk_size] + candidates
              candidates[-1] += 4

              def test_chunk_size(__SCREAMING_SNAKE_CASE						) -> bool:
                     try:
                            with torch.no_grad():
                                   fn(*__SCREAMING_SNAKE_CASE ,					chunk_size=__SCREAMING_SNAKE_CASE						)
                            return True
                     except RuntimeError:
                            return False

              UpperCamelCase    :	Optional[int]   	=		0
              UpperCamelCase    :	List[str]   	=		len(__SCREAMING_SNAKE_CASE						) - 1
              while i > min_viable_chunk_size_index:
                     UpperCamelCase    :	Optional[int]   	=		test_chunk_size(candidates[i]						)
                     if not viable:
                            UpperCamelCase    :	Any   	=		(min_viable_chunk_size_index + i) // 2
                     else:
                            UpperCamelCase    :	Tuple   	=		i
                            UpperCamelCase    :	Optional[int]   	=		(i + len(__SCREAMING_SNAKE_CASE						) - 1) // 2

              return candidates[min_viable_chunk_size_index]






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Optional[Any]   	=		True
              for aa, aa in zip(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):
                     assert type(__SCREAMING_SNAKE_CASE						) == type(__SCREAMING_SNAKE_CASE						)
                     if isinstance(__SCREAMING_SNAKE_CASE ,					(list, tuple)						):
                            consistent &= self._compare_arg_caches(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)
                     elif isinstance(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):
                            UpperCamelCase    :	Union[str, Any]   	=		[v for _, v in sorted(aa.items() ,					key=lambda __SCREAMING_SNAKE_CASE						: x[0]						)]
                            UpperCamelCase    :	Optional[int]   	=		[v for _, v in sorted(aa.items() ,					key=lambda __SCREAMING_SNAKE_CASE						: x[0]						)]
                            consistent &= self._compare_arg_caches(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)
                     else:
                            consistent &= aa == aa

              return consistent






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					):




              """simple docstring"""




              UpperCamelCase    :	List[Any]   	=		True
              UpperCamelCase    :	tuple   	=		tree_map(lambda __SCREAMING_SNAKE_CASE						: a.shape if isinstance(__SCREAMING_SNAKE_CASE ,					torch.Tensor						) else a ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)
              if self.cached_arg_data is not None:
                     # If args have changed shape/value, we need to re-tune
                     assert len(self.cached_arg_data						) == len(__SCREAMING_SNAKE_CASE						)
                     UpperCamelCase    :	Optional[Any]   	=		self._compare_arg_caches(self.cached_arg_data ,					__SCREAMING_SNAKE_CASE						)
              else:
                     # Otherwise, we can reuse the precomputed value
                     UpperCamelCase    :	Optional[Any]   	=		False

              if not consistent:
                     UpperCamelCase    :	Optional[int]   	=		self._determine_favorable_chunk_size(
                         __SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					)
                     UpperCamelCase    :	Any   	=		arg_data

              assert self.cached_chunk_size is not None

              return self.cached_chunk_size





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__UpperCAmelCase : Dict 			=				logging.get_logger(__name__)

__UpperCAmelCase : Optional[Any] 			=				{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase : Dict 			=				{
    "vocab_file": {
        "junnyu/roformer_chinese_small": "https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_base": "https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt",
        "junnyu/roformer_chinese_char_small": (
            "https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_chinese_char_base": (
            "https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_discriminator": (
            "https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt"
        ),
        "junnyu/roformer_small_generator": (
            "https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt"
        ),
    }
}

__UpperCAmelCase : Tuple 			=				{
    "junnyu/roformer_chinese_small": 1536,
    "junnyu/roformer_chinese_base": 1536,
    "junnyu/roformer_chinese_char_small": 512,
    "junnyu/roformer_chinese_char_base": 512,
    "junnyu/roformer_small_discriminator": 128,
    "junnyu/roformer_small_generator": 128,
}


__UpperCAmelCase : Any 			=				{
    "junnyu/roformer_chinese_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_base": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_small": {"do_lower_case": True},
    "junnyu/roformer_chinese_char_base": {"do_lower_case": True},
    "junnyu/roformer_small_discriminator": {"do_lower_case": True},
    "junnyu/roformer_small_generator": {"do_lower_case": True},
}



class    UpperCAmelCase_	(							_a):
       '''simple docstring'''
       __UpperCamelCase      :     str							=					VOCAB_FILES_NAMES
       __UpperCamelCase      :     Optional[int]							=					PRETRAINED_VOCAB_FILES_MAP
       __UpperCamelCase      :     Union[str, Any]							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __UpperCamelCase      :     int							=					PRETRAINED_INIT_CONFIGURATION
       __UpperCamelCase      :     Any							=					RoFormerTokenizer






       def __init__(     self ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE="[UNK]" ,					__SCREAMING_SNAKE_CASE="[SEP]" ,					__SCREAMING_SNAKE_CASE="[PAD]" ,					__SCREAMING_SNAKE_CASE="[CLS]" ,					__SCREAMING_SNAKE_CASE="[MASK]" ,					__SCREAMING_SNAKE_CASE=True ,					__SCREAMING_SNAKE_CASE=None ,					**__SCREAMING_SNAKE_CASE ,					):




              """simple docstring"""




              super().__init__(
                  __SCREAMING_SNAKE_CASE ,					tokenizer_file=__SCREAMING_SNAKE_CASE ,					do_lower_case=__SCREAMING_SNAKE_CASE ,					unk_token=__SCREAMING_SNAKE_CASE ,					sep_token=__SCREAMING_SNAKE_CASE ,					pad_token=__SCREAMING_SNAKE_CASE ,					cls_token=__SCREAMING_SNAKE_CASE ,					mask_token=__SCREAMING_SNAKE_CASE ,					tokenize_chinese_chars=__SCREAMING_SNAKE_CASE ,					strip_accents=__SCREAMING_SNAKE_CASE ,					**__SCREAMING_SNAKE_CASE ,					)

              UpperCamelCase    :	List[Any]   	=		json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
              if (
                  pre_tok_state.get('''lowercase''' ,					__SCREAMING_SNAKE_CASE						) != do_lower_case
                  or pre_tok_state.get('''strip_accents''' ,					__SCREAMING_SNAKE_CASE						) != strip_accents
              ):
                     UpperCamelCase    :	List[Any]   	=		getattr(__SCREAMING_SNAKE_CASE ,					pre_tok_state.pop('''type'''						)						)
                     UpperCamelCase    :	Optional[int]   	=		do_lower_case
                     UpperCamelCase    :	Optional[Any]   	=		strip_accents
                     UpperCamelCase    :	List[Any]   	=		pre_tok_class(**__SCREAMING_SNAKE_CASE						)

              UpperCamelCase    :	List[Any]   	=		do_lower_case






       def __getstate__(     self						):




              """simple docstring"""




              UpperCamelCase    :	Optional[int]   	=		self.__dict__.copy()
              UpperCamelCase    :	Any   	=		BertPreTokenizer()
              return state






       def __setstate__(     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	Any   	=		d
              UpperCamelCase    :	List[str]   	=		self.__dict__['''_tokenizer'''].get_vocab()
              UpperCamelCase    :	Any   	=		PreTokenizer.custom(JiebaPreTokenizer(__SCREAMING_SNAKE_CASE						)						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None						):




              """simple docstring"""




              UpperCamelCase    :	Any   	=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

              if token_ids_a:
                     output += token_ids_a + [self.sep_token_id]

              return output






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None						):




              """simple docstring"""




              UpperCamelCase    :	Dict   	=		[self.sep_token_id]
              UpperCamelCase    :	int   	=		[self.cls_token_id]
              if token_ids_a is None:
                     return len(cls + token_ids_a + sep						) * [0]
              return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE = None						):




              """simple docstring"""




              UpperCamelCase    :	List[Any]   	=		self._tokenizer.model.save(__SCREAMING_SNAKE_CASE ,					name=__SCREAMING_SNAKE_CASE						)
              return tuple(__SCREAMING_SNAKE_CASE						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=None ,					__SCREAMING_SNAKE_CASE=False ,					**__SCREAMING_SNAKE_CASE ,					):




              """simple docstring"""




              UpperCamelCase    :	Any   	=		BertPreTokenizer()
              return super().save_pretrained(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE ,					**__SCREAMING_SNAKE_CASE						)

