






"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


UpperCamelCase							       =    logging.get_logger(__name__)

UpperCamelCase							       =    {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase							       =    {
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/tokenizer.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/tokenizer.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json""",
        """roberta-base-openai-detector""": (
            """https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"""
        ),
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCamelCase							       =    {
    """roberta-base""": 512,
    """roberta-large""": 512,
    """roberta-large-mnli""": 512,
    """distilroberta-base""": 512,
    """roberta-base-openai-detector""": 512,
    """roberta-large-openai-detector""": 512,
}




class    lowercase_    (_UpperCAmelCase     ):
						A__	:		List[Any]       			=				VOCAB_FILES_NAMES
						A__	:		Dict       			=				PRETRAINED_VOCAB_FILES_MAP
						A__	:		Tuple       			=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						A__	:		List[Any]       			=				['''input_ids''', '''attention_mask''']
						A__	:		Tuple       			=				RobertaTokenizer






						def __init__(    self     ,     a_=None     ,     a_=None     ,     a_=None     ,     a_="replace"     ,     a_="<s>"     ,     a_="</s>"     ,     a_="</s>"     ,     a_="<s>"     ,     a_="<unk>"     ,     a_="<pad>"     ,     a_="<mask>"     ,     a_=False     ,     a_=True     ,     **a_     ,     )	->Optional[int]:







								'''simple docstring'''


								super().__init__(
								    a_     ,     a_     ,     tokenizer_file=a_     ,     errors=a_     ,     bos_token=a_     ,     eos_token=a_     ,     sep_token=a_     ,     cls_token=a_     ,     unk_token=a_     ,     pad_token=a_     ,     mask_token=a_     ,     add_prefix_space=a_     ,     trim_offsets=a_     ,     **a_     ,     )

								_a							=	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
								if pre_tok_state.get("add_prefix_space"     ,     a_	) != add_prefix_space:
										_a							=	getattr(a_     ,     pre_tok_state.pop("type"	)	)
										_a							=	add_prefix_space
										_a							=	pre_tok_class(**a_	)

								_a							=	add_prefix_space

								_a							=	"post_processor"
								_a							=	getattr(self.backend_tokenizer     ,     a_     ,     a_	)
								if tokenizer_component_instance:
										_a							=	json.loads(tokenizer_component_instance.__getstate__()	)

										# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
										if "sep" in state:
												_a							=	tuple(state["sep"]	)
										if "cls" in state:
												_a							=	tuple(state["cls"]	)

										_a							=	False

										if state.get("add_prefix_space"     ,     a_	) != add_prefix_space:
												_a							=	add_prefix_space
												_a							=	True

										if state.get("trim_offsets"     ,     a_	) != trim_offsets:
												_a							=	trim_offsets
												_a							=	True

										if changes_to_apply:
												_a							=	getattr(a_     ,     state.pop("type"	)	)
												_a							=	component_class(**a_	)
												setattr(self.backend_tokenizer     ,     a_     ,     a_	)






						@property
						def 			lowerCamelCase__    (    self	)	->str:







								'''simple docstring'''


								if self._mask_token is None:
										if self.verbose:
												logger.error("Using mask_token, but it is not set yet."	)
										return None
								return str(self._mask_token	)






						@mask_token.setter
						def 			lowerCamelCase__    (    self     ,     a_	)	->List[str]:







								'''simple docstring'''


								_a							=	AddedToken(a_     ,     lstrip=a_     ,     rstrip=a_	) if isinstance(a_     ,     a_	) else value
								_a							=	value






						def 			lowerCamelCase__    (    self     ,     *a_     ,     **a_	)	->BatchEncoding:







								'''simple docstring'''


								_a							=	kwargs.get("is_split_into_words"     ,     a_	)
								assert self.add_prefix_space or not is_split_into_words, (
								    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
								    "to use it with pretokenized inputs."
								)

								return super()._batch_encode_plus(*a_     ,     **a_	)






						def 			lowerCamelCase__    (    self     ,     *a_     ,     **a_	)	->BatchEncoding:







								'''simple docstring'''


								_a							=	kwargs.get("is_split_into_words"     ,     a_	)

								assert self.add_prefix_space or not is_split_into_words, (
								    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
								    "to use it with pretokenized inputs."
								)

								return super()._encode_plus(*a_     ,     **a_	)






						def 			lowerCamelCase__    (    self     ,     a_     ,     a_ = None	)	->Tuple[str]:







								'''simple docstring'''


								_a							=	self._tokenizer.model.save(a_     ,     name=a_	)
								return tuple(a_	)






						def 			lowerCamelCase__    (    self     ,     a_     ,     a_=None	)	->Dict:







								'''simple docstring'''


								_a							=	[self.bos_token_id] + token_ids_a + [self.eos_token_id]
								if token_ids_a is None:
										return output

								return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






						def 			lowerCamelCase__    (    self     ,     a_     ,     a_ = None	)	->List[int]:







								'''simple docstring'''


								_a							=	[self.sep_token_id]
								_a							=	[self.cls_token_id]

								if token_ids_a is None:
										return len(cls + token_ids_a + sep	) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]










"""simple docstring"""




import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






def        lowerCAmelCase (      UpperCamelCase_:							Features					)       ->							Optional[int]:







  '''simple docstring'''







  _a							=	np.inf

  def set_batch_size(UpperCamelCase_:							FeatureType					) -> None:
    nonlocal batch_size
    if isinstance(UpperCamelCase_     , UpperCamelCase_					):
      _a							=	min(UpperCamelCase_     , config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS					)
    elif isinstance(UpperCamelCase_     , UpperCamelCase_					):
      _a							=	min(UpperCamelCase_     , config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS					)
    elif isinstance(UpperCamelCase_     , UpperCamelCase_					) and feature.dtype == "binary":
      _a							=	min(UpperCamelCase_     , config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS					)

  _visit(UpperCamelCase_     , UpperCamelCase_					)

  return None if batch_size is np.inf else batch_size




class    lowercase_    (_UpperCAmelCase     ):






      def __init__(    self     ,     a_     ,     a_ = None     ,     a_ = None     ,     a_ = None     ,     a_ = False     ,     a_ = False     ,     a_ = None     ,     **a_     ,     )	->Optional[int]:







        '''simple docstring'''


        super().__init__(
            a_     ,     split=a_     ,     features=a_     ,     cache_dir=a_     ,     keep_in_memory=a_     ,     streaming=a_     ,     num_proc=a_     ,     **a_     ,     )
        _a							=	path_or_paths if isinstance(a_     ,     a_	) else {self.split: path_or_paths}
        _a							=	_PACKAGED_DATASETS_MODULES["parquet"][1]
        _a							=	Parquet(
            cache_dir=a_     ,     data_files=a_     ,     features=a_     ,     hash=a_     ,     **a_     ,     )






      def 			lowerCamelCase__    (    self	)	->str:







        '''simple docstring'''


        if self.streaming:
          _a							=	self.builder.as_streaming_dataset(split=self.split	)
        # Build regular (map-style) dataset
        else:
          _a							=	None
          _a							=	None
          _a							=	None
          _a							=	None

          self.builder.download_and_prepare(
              download_config=a_     ,     download_mode=a_     ,     verification_mode=a_     ,     base_path=a_     ,     num_proc=self.num_proc     ,     )
          _a							=	self.builder.as_dataset(
              split=self.split     ,     verification_mode=a_     ,     in_memory=self.keep_in_memory	)
        return dataset







class    lowercase_    :






      def __init__(    self     ,     a_     ,     a_     ,     a_ = None     ,     **a_     ,     )	->int:







        '''simple docstring'''


        _a							=	dataset
        _a							=	path_or_buf
        _a							=	batch_size or get_writer_batch_size(dataset.features	)
        _a							=	parquet_writer_kwargs






      def 			lowerCamelCase__    (    self	)	->int:







        '''simple docstring'''


        _a							=	self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

        if isinstance(self.path_or_buf     ,     (str, bytes, os.PathLike)	):
          with open(self.path_or_buf     ,     "wb+"	) as buffer:
            _a							=	self._write(file_obj=a_     ,     batch_size=a_     ,     **self.parquet_writer_kwargs	)
        else:
          _a							=	self._write(file_obj=self.path_or_buf     ,     batch_size=a_     ,     **self.parquet_writer_kwargs	)
        return written






      def 			lowerCamelCase__    (    self     ,     a_     ,     a_     ,     **a_	)	->int:







        '''simple docstring'''


        _a							=	0
        _a							=	parquet_writer_kwargs.pop("path_or_buf"     ,     a_	)
        _a							=	self.dataset.features.arrow_schema

        _a							=	pq.ParquetWriter(a_     ,     schema=a_     ,     **a_	)

        for offset in logging.tqdm(
            range(0     ,     len(self.dataset	)     ,     a_	)     ,     unit="ba"     ,     disable=not logging.is_progress_bar_enabled()     ,     desc="Creating parquet from Arrow format"     ,     ):
          _a							=	query_table(
              table=self.dataset._data     ,     key=slice(a_     ,     offset + batch_size	)     ,     indices=self.dataset._indices if self.dataset._indices is not None else None     ,     )
          writer.write_table(a_	)
          written += batch.nbytes
        writer.close()
        return written


