






'''simple docstring'''
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class        _UpperCAmelCase						(		_UpperCAmelCase				,					unittest.TestCase			):
    __lowerCamelCase:   str		       =    BertTokenizer
    __lowerCamelCase:   Optional[int]		       =    BertTokenizerFast
    __lowerCamelCase:   Any		       =    True
    __lowerCamelCase:   int		       =    True
    __lowerCamelCase:   Dict		       =    filter_non_english




    def 				lowerCAmelCase__				(			self  :       Optional[Any]			):






       '''simple docstring'''






       super().setUp()

       lowercase_	:    str           =     [
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       lowercase_	:    int           =     os.path.join(self.tmpdirname  ,     VOCAB_FILES_NAMES["vocab_file"]			)
       with open(self.vocab_file  ,     "w"  ,     encoding="utf-8"			) as vocab_writer:
          vocab_writer.write("".join([x + "\n" for x in vocab_tokens]			)			)




    def 				lowerCAmelCase__				(			self  :       Optional[Any]  ,     a  :       List[Any]			):






       '''simple docstring'''






       lowercase_	:    Tuple           =     "UNwant\u00E9d,running"
       lowercase_	:    str           =     "unwanted, running"
       return input_text, output_text




    def 				lowerCAmelCase__				(			self  :       List[Any]			):






       '''simple docstring'''






       lowercase_	:    str           =     self.tokenizer_class(self.vocab_file			)

       lowercase_	:    List[str]           =     tokenizer.tokenize("UNwant\u00E9d,running"			)
       self.assertListEqual(lowercase__  ,     ["un", "##want", "##ed", ",", "runn", "##ing"]			)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase__			)  ,     [9, 6, 7, 1_2, 1_0, 1_1]			)




    def 				lowerCAmelCase__				(			self  :       Any			):






       '''simple docstring'''






       if not self.test_rust_tokenizer:
          return

       lowercase_	:    Any           =     self.get_tokenizer()
       lowercase_	:    Union[str, Any]           =     self.get_rust_tokenizer()

       lowercase_	:    List[str]           =     "UNwant\u00E9d,running"

       lowercase_	:    Tuple           =     tokenizer.tokenize(lowercase__			)
       lowercase_	:    List[Any]           =     rust_tokenizer.tokenize(lowercase__			)
       self.assertListEqual(lowercase__  ,     lowercase__			)

       lowercase_	:    Optional[Any]           =     tokenizer.encode(lowercase__  ,     add_special_tokens=lowercase__			)
       lowercase_	:    Union[str, Any]           =     rust_tokenizer.encode(lowercase__  ,     add_special_tokens=lowercase__			)
       self.assertListEqual(lowercase__  ,     lowercase__			)

       lowercase_	:    Optional[int]           =     self.get_rust_tokenizer()
       lowercase_	:    Dict           =     tokenizer.encode(lowercase__			)
       lowercase_	:    Union[str, Any]           =     rust_tokenizer.encode(lowercase__			)
       self.assertListEqual(lowercase__  ,     lowercase__			)

       # With lower casing
       lowercase_	:    Union[str, Any]           =     self.get_tokenizer(do_lower_case=lowercase__			)
       lowercase_	:    List[Any]           =     self.get_rust_tokenizer(do_lower_case=lowercase__			)

       lowercase_	:    Optional[Any]           =     "UNwant\u00E9d,running"

       lowercase_	:    Optional[Any]           =     tokenizer.tokenize(lowercase__			)
       lowercase_	:    Any           =     rust_tokenizer.tokenize(lowercase__			)
       self.assertListEqual(lowercase__  ,     lowercase__			)

       lowercase_	:    Union[str, Any]           =     tokenizer.encode(lowercase__  ,     add_special_tokens=lowercase__			)
       lowercase_	:    List[Any]           =     rust_tokenizer.encode(lowercase__  ,     add_special_tokens=lowercase__			)
       self.assertListEqual(lowercase__  ,     lowercase__			)

       lowercase_	:    int           =     self.get_rust_tokenizer()
       lowercase_	:    List[Any]           =     tokenizer.encode(lowercase__			)
       lowercase_	:    List[Any]           =     rust_tokenizer.encode(lowercase__			)
       self.assertListEqual(lowercase__  ,     lowercase__			)




    def 				lowerCAmelCase__				(			self  :       List[Any]			):






       '''simple docstring'''






       lowercase_	:    Union[str, Any]           =     BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"			)  ,     ["ah", "\u535A", "\u63A8", "zz"]			)




    def 				lowerCAmelCase__				(			self  :       Any			):






       '''simple docstring'''






       lowercase_	:    Tuple           =     BasicTokenizer(do_lower_case=lowercase__			)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)  ,     ["hello", "!", "how", "are", "you", "?"]			)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)  ,     ["hello"]			)




    def 				lowerCAmelCase__				(			self  :       Tuple			):






       '''simple docstring'''






       lowercase_	:    Optional[Any]           =     BasicTokenizer(do_lower_case=lowercase__  ,     strip_accents=lowercase__			)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)  ,     ["hällo", "!", "how", "are", "you", "?"]			)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)  ,     ["h\u00E9llo"]			)




    def 				lowerCAmelCase__				(			self  :       Optional[Any]			):






       '''simple docstring'''






       lowercase_	:    Dict           =     BasicTokenizer(do_lower_case=lowercase__  ,     strip_accents=lowercase__			)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)  ,     ["hallo", "!", "how", "are", "you", "?"]			)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)  ,     ["hello"]			)




    def 				lowerCAmelCase__				(			self  :       Dict			):






       '''simple docstring'''






       lowercase_	:    Optional[int]           =     BasicTokenizer(do_lower_case=lowercase__			)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)  ,     ["hallo", "!", "how", "are", "you", "?"]			)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"			)  ,     ["hello"]			)




    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       lowercase_	:    Optional[Any]           =     BasicTokenizer(do_lower_case=lowercase__			)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "			)  ,     ["HeLLo", "!", "how", "Are", "yoU", "?"]			)




    def 				lowerCAmelCase__				(			self  :       Tuple			):






       '''simple docstring'''






       lowercase_	:    Optional[int]           =     BasicTokenizer(do_lower_case=lowercase__  ,     strip_accents=lowercase__			)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)  ,     ["HäLLo", "!", "how", "Are", "yoU", "?"]			)




    def 				lowerCAmelCase__				(			self  :       Dict			):






       '''simple docstring'''






       lowercase_	:    Union[str, Any]           =     BasicTokenizer(do_lower_case=lowercase__  ,     strip_accents=lowercase__			)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "			)  ,     ["HaLLo", "!", "how", "Are", "yoU", "?"]			)




    def 				lowerCAmelCase__				(			self  :       List[Any]			):






       '''simple docstring'''






       lowercase_	:    Optional[Any]           =     BasicTokenizer(do_lower_case=lowercase__  ,     never_split=["[UNK]"]			)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"			)  ,     ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]			)




    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       lowercase_	:    Optional[int]           =     BasicTokenizer()
       lowercase_	:    Optional[Any]           =     "a\n'll !!to?'d of, can't."
       lowercase_	:    List[Any]           =     ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
       self.assertListEqual(tokenizer.tokenize(lowercase__			)  ,     lowercase__			)




    def 				lowerCAmelCase__				(			self  :       str			):






       '''simple docstring'''






       lowercase_	:    List[str]           =     ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       lowercase_	:    Dict           =     {}
       for i, token in enumerate(lowercase__			):
          lowercase_	:    List[Any]           =     i
       lowercase_	:    str           =     WordpieceTokenizer(vocab=lowercase__  ,     unk_token="[UNK]"			)

       self.assertListEqual(tokenizer.tokenize(""			)  ,     []			)

       self.assertListEqual(tokenizer.tokenize("unwanted running"			)  ,     ["un", "##want", "##ed", "runn", "##ing"]			)

       self.assertListEqual(tokenizer.tokenize("unwantedX running"			)  ,     ["[UNK]", "runn", "##ing"]			)




    def 				lowerCAmelCase__				(			self  :       Union[str, Any]			):






       '''simple docstring'''






       self.assertTrue(_is_whitespace(" "			)			)
       self.assertTrue(_is_whitespace("\t"			)			)
       self.assertTrue(_is_whitespace("\r"			)			)
       self.assertTrue(_is_whitespace("\n"			)			)
       self.assertTrue(_is_whitespace("\u00A0"			)			)

       self.assertFalse(_is_whitespace("A"			)			)
       self.assertFalse(_is_whitespace("-"			)			)




    def 				lowerCAmelCase__				(			self  :       Tuple			):






       '''simple docstring'''






       self.assertTrue(_is_control("\u0005"			)			)

       self.assertFalse(_is_control("A"			)			)
       self.assertFalse(_is_control(" "			)			)
       self.assertFalse(_is_control("\t"			)			)
       self.assertFalse(_is_control("\r"			)			)




    def 				lowerCAmelCase__				(			self  :       Tuple			):






       '''simple docstring'''






       self.assertTrue(_is_punctuation("-"			)			)
       self.assertTrue(_is_punctuation("$"			)			)
       self.assertTrue(_is_punctuation("`"			)			)
       self.assertTrue(_is_punctuation("."			)			)

       self.assertFalse(_is_punctuation("A"			)			)
       self.assertFalse(_is_punctuation(" "			)			)




    def 				lowerCAmelCase__				(			self  :       Dict			):






       '''simple docstring'''






       lowercase_	:    Optional[Any]           =     self.get_tokenizer()
       lowercase_	:    Dict           =     self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(lowercase__			) for t in ["Test", "\xad", "test"]]  ,     [["[UNK]"], [], ["[UNK]"]]			)

       self.assertListEqual(
           [rust_tokenizer.tokenize(lowercase__			) for t in ["Test", "\xad", "test"]]  ,     [["[UNK]"], [], ["[UNK]"]]			)




    @slow
    def 				lowerCAmelCase__				(			self  :       Dict			):






       '''simple docstring'''






       lowercase_	:    Optional[Any]           =     self.tokenizer_class.from_pretrained("bert-base-uncased"			)

       lowercase_	:    Union[str, Any]           =     tokenizer.encode("sequence builders"  ,     add_special_tokens=lowercase__			)
       lowercase_	:    str           =     tokenizer.encode("multi-sequence build"  ,     add_special_tokens=lowercase__			)

       lowercase_	:    Dict           =     tokenizer.build_inputs_with_special_tokens(lowercase__			)
       lowercase_	:    Union[str, Any]           =     tokenizer.build_inputs_with_special_tokens(lowercase__  ,     lowercase__			)

       assert encoded_sentence == [1_0_1] + text + [1_0_2]
       assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




    def 				lowerCAmelCase__				(			self  :       Optional[int]			):






       '''simple docstring'''






       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
             lowercase_	:    List[Any]           =     self.rust_tokenizer_class.from_pretrained(lowercase__  ,     **lowercase__			)

             lowercase_	:    Tuple           =     f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
             lowercase_	:    str           =     tokenizer_r.encode_plus(
                 lowercase__  ,     return_attention_mask=lowercase__  ,     return_token_type_ids=lowercase__  ,     return_offsets_mapping=lowercase__  ,     add_special_tokens=lowercase__  ,     )

             lowercase_	:    int           =     tokenizer_r.do_lower_case if hasattr(lowercase__  ,     "do_lower_case"			) else False
             lowercase_	:    Tuple           =     (
                 [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "A"),
                     ((1, 2), ","),
                     ((3, 5), "na"),
                     ((5, 6), "##ï"),
                     ((6, 8), "##ve"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "Allen"),
                     ((2_1, 2_3), "##NL"),
                     ((2_3, 2_4), "##P"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
                 if not do_lower_case
                 else [
                     ((0, 0), tokenizer_r.cls_token),
                     ((0, 1), "a"),
                     ((1, 2), ","),
                     ((3, 8), "naive"),
                     ((9, 1_5), tokenizer_r.mask_token),
                     ((1_6, 2_1), "allen"),
                     ((2_1, 2_3), "##nl"),
                     ((2_3, 2_4), "##p"),
                     ((2_5, 3_3), "sentence"),
                     ((3_3, 3_4), "."),
                     ((0, 0), tokenizer_r.sep_token),
                 ]
             )

             self.assertEqual(
                 [e[1] for e in expected_results]  ,     tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]			)			)
             self.assertEqual([e[0] for e in expected_results]  ,     tokens["offset_mapping"]			)





    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       lowercase_	:    List[str]           =     ["的", "人", "有"]
       lowercase_	:    Optional[int]           =     "".join(lowercase__			)
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
          with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
             lowercase_	:    Optional[Any]           =     True
             lowercase_	:    int           =     self.tokenizer_class.from_pretrained(lowercase__  ,     **lowercase__			)
             lowercase_	:    Any           =     self.rust_tokenizer_class.from_pretrained(lowercase__  ,     **lowercase__			)

             lowercase_	:    Any           =     tokenizer_p.encode(lowercase__  ,     add_special_tokens=lowercase__			)
             lowercase_	:    Optional[Any]           =     tokenizer_r.encode(lowercase__  ,     add_special_tokens=lowercase__			)

             lowercase_	:    Optional[int]           =     tokenizer_r.convert_ids_to_tokens(lowercase__			)
             lowercase_	:    List[str]           =     tokenizer_p.convert_ids_to_tokens(lowercase__			)

             # it is expected that each Chinese character is not preceded by "##"
             self.assertListEqual(lowercase__  ,     lowercase__			)
             self.assertListEqual(lowercase__  ,     lowercase__			)

             lowercase_	:    Optional[Any]           =     False
             lowercase_	:    Union[str, Any]           =     self.rust_tokenizer_class.from_pretrained(lowercase__  ,     **lowercase__			)
             lowercase_	:    Union[str, Any]           =     self.tokenizer_class.from_pretrained(lowercase__  ,     **lowercase__			)

             lowercase_	:    List[str]           =     tokenizer_r.encode(lowercase__  ,     add_special_tokens=lowercase__			)
             lowercase_	:    Dict           =     tokenizer_p.encode(lowercase__  ,     add_special_tokens=lowercase__			)

             lowercase_	:    Dict           =     tokenizer_r.convert_ids_to_tokens(lowercase__			)
             lowercase_	:    Any           =     tokenizer_p.convert_ids_to_tokens(lowercase__			)

             # it is expected that only the first Chinese character is not preceded by "##".
             lowercase_	:    Optional[int]           =     [
                 f"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowercase__			)
             ]
             self.assertListEqual(lowercase__  ,     lowercase__			)
             self.assertListEqual(lowercase__  ,     lowercase__			)












'''simple docstring'''
import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader


class        _UpperCAmelCase						(		snake_case			):




    def __init__(			self  :       Tuple  ,     a  :       NestedDataStructureLike[PathLike]  ,     a  :       Optional[NamedSplit] = None  ,     a  :       Optional[Features] = None  ,     a  :       str = None  ,     a  :       bool = False  ,     a  :       bool = False  ,     a  :       Optional[str] = None  ,     a  :       Optional[int] = None  ,     **a  :       List[Any]  ,     ):






       '''simple docstring'''






       super().__init__(
           a  ,     split=a  ,     features=a  ,     cache_dir=a  ,     keep_in_memory=a  ,     streaming=a  ,     num_proc=a  ,     **a  ,     )
       lowercase_	:    str           =     field
       lowercase_	:    Optional[Any]           =     path_or_paths if isinstance(a  ,     a			) else {self.split: path_or_paths}
       lowercase_	:    Any           =     Json(
           cache_dir=a  ,     data_files=a  ,     features=a  ,     field=a  ,     **a  ,     )





    def 				lowerCAmelCase__				(			self  :       Optional[int]			):






       '''simple docstring'''






       if self.streaming:
          lowercase_	:    Any           =     self.builder.as_streaming_dataset(split=self.split			)
       # Build regular (map-style) dataset
       else:
          lowercase_	:    Dict           =     None
          lowercase_	:    Optional[int]           =     None
          lowercase_	:    str           =     None
          lowercase_	:    str           =     None

          self.builder.download_and_prepare(
              download_config=a  ,     download_mode=a  ,     verification_mode=a  ,     base_path=a  ,     num_proc=self.num_proc  ,     )
          lowercase_	:    int           =     self.builder.as_dataset(
              split=self.split  ,     verification_mode=a  ,     in_memory=self.keep_in_memory			)
       return dataset


class        _UpperCAmelCase						:




    def __init__(			self  :       str  ,     a  :       Dataset  ,     a  :       Union[PathLike, BinaryIO]  ,     a  :       Optional[int] = None  ,     a  :       Optional[int] = None  ,     **a  :       List[Any]  ,     ):






       '''simple docstring'''






       if num_proc is not None and num_proc <= 0:
          raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""			)

       lowercase_	:    Dict           =     dataset
       lowercase_	:    Optional[int]           =     path_or_buf
       lowercase_	:    List[Any]           =     batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
       lowercase_	:    Optional[Any]           =     num_proc
       lowercase_	:    List[Any]           =     "utf-8"
       lowercase_	:    List[str]           =     to_json_kwargs




    def 				lowerCAmelCase__				(			self  :       Optional[int]			):






       '''simple docstring'''






       lowercase_	:    str           =     self.to_json_kwargs.pop("path_or_buf"  ,     a			)
       lowercase_	:    Any           =     self.to_json_kwargs.pop("orient"  ,     "records"			)
       lowercase_	:    Any           =     self.to_json_kwargs.pop("lines"  ,     True if orient == "records" else False			)
       lowercase_	:    List[str]           =     self.to_json_kwargs.pop("index"  ,     False if orient in ["split", "table"] else True			)
       lowercase_	:    int           =     self.to_json_kwargs.pop("compression"  ,     a			)

       if compression not in [None, "infer", "gzip", "bz2", "xz"]:
          raise NotImplementedError(f"""`datasets` currently does not support {compression} compression"""			)

       if isinstance(self.path_or_buf  ,     (str, bytes, os.PathLike)			):
          with fsspec.open(self.path_or_buf  ,     "wb"  ,     compression=a			) as buffer:
             lowercase_	:    Dict           =     self._write(file_obj=a  ,     orient=a  ,     lines=a  ,     index=a  ,     **self.to_json_kwargs			)
       else:
          if compression:
             raise NotImplementedError(
                 f"""The compression parameter is not supported when writing to a buffer, but compression={compression}"""
                 " was passed. Please provide a local path instead."			)
          lowercase_	:    Dict           =     self._write(
              file_obj=self.path_or_buf  ,     orient=a  ,     lines=a  ,     index=a  ,     **self.to_json_kwargs			)
       return written




    def 				lowerCAmelCase__				(			self  :       Optional[int]  ,     a  :       List[str]			):






       '''simple docstring'''






       lowercase_    ,		lowercase_    ,		lowercase_    ,		lowercase_    ,		lowercase_	:    List[Any]           =     args

       lowercase_	:    Optional[int]           =     query_table(
           table=self.dataset.data  ,     key=slice(a  ,     offset + self.batch_size			)  ,     indices=self.dataset._indices  ,     )
       lowercase_	:    Dict           =     batch.to_pandas().to_json(
           path_or_buf=a  ,     orient=a  ,     lines=a  ,     index=a  ,     **a			)
       if not json_str.endswith("\n"			):
          json_str += "\n"
       return json_str.encode(self.encoding			)





    def 				lowerCAmelCase__				(			self  :       int  ,     a  :       BinaryIO  ,     a  :       int  ,     a  :       str  ,     a  :       Union[str, Any]  ,     **a  :       str  ,     ):






       '''simple docstring'''






       lowercase_	:    Union[str, Any]           =     0

       if self.num_proc is None or self.num_proc == 1:
          for offset in logging.tqdm(
              range(0  ,     len(self.dataset			)  ,     self.batch_size			)  ,     unit="ba"  ,     disable=not logging.is_progress_bar_enabled()  ,     desc="Creating json from Arrow format"  ,     ):
             lowercase_	:    Dict           =     self._batch_json((offset, orient, lines, index, to_json_kwargs)			)
             written += file_obj.write(a			)
       else:
          lowercase_    ,		lowercase_	:    Any           =     len(self.dataset			), self.batch_size
          with multiprocessing.Pool(self.num_proc			) as pool:
             for json_str in logging.tqdm(
                 pool.imap(
                     self._batch_json  ,     [(offset, orient, lines, index, to_json_kwargs) for offset in range(0  ,     a  ,     a			)]  ,     )  ,     total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size  ,     unit="ba"  ,     disable=not logging.is_progress_bar_enabled()  ,     desc="Creating json from Arrow format"  ,     ):
                written += file_obj.write(a			)

       return written




