





'''simple docstring'''




import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_A:					Any							=				get_tests_dir("""fixtures/test_sentencepiece.model""")






@require_sentencepiece
class UpperCAmelCase							(		UpperCAmelCase_					,			unittest.TestCase							):
    _A :			Union[str, Any]						   =       XLMProphetNetTokenizer
    _A :			Any						   =       False
    _A :			Any						   =       True



    def 					__lowerCamelCase						(  self					):
     super().setUp()

     # We have a SentencePiece fixture for testing
     __UpperCAmelCase					=   XLMProphetNetTokenizer(__A     ,			keep_accents=__A					)
     tokenizer.save_pretrained(self.tmpdirname					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   '[PAD]'
     __UpperCAmelCase					=   0

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(__A					)     ,			__A					)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(__A					)     ,			__A					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   list(self.get_tokenizer().get_vocab().keys()					)

     self.assertEqual(vocab_keys[0]     ,			'[PAD]'					)
     self.assertEqual(vocab_keys[1]     ,			'[CLS]'					)
     self.assertEqual(vocab_keys[-1]     ,			'j'					)
     self.assertEqual(len(__A					)     ,			1_012					)



    def 					__lowerCamelCase						(  self					):
     self.assertEqual(self.get_tokenizer().vocab_size     ,			1_012					)



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   XLMProphetNetTokenizer(__A     ,			keep_accents=__A					)

     __UpperCAmelCase					=   tokenizer.tokenize('This is a test'					)
     self.assertListEqual(__A     ,			['▁This', '▁is', '▁a', '▁t', 'est']					)

     self.assertListEqual(
         tokenizer.convert_tokens_to_ids(__A					)     ,			[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]     ,			)

     __UpperCAmelCase					=   tokenizer.tokenize('I was born in 92000, and this is falsé.'					)
     self.assertListEqual(
         __A     ,			[
             SPIECE_UNDERLINE + 'I',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '9',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             's',
             'é',
             '.',
         ]     ,			)
     __UpperCAmelCase					=   tokenizer.convert_tokens_to_ids(__A					)
     self.assertListEqual(
         __A     ,			[
             value + tokenizer.fairseq_offset
             for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
         ]     ,			)

     __UpperCAmelCase					=   tokenizer.convert_ids_to_tokens(__A					)
     self.assertListEqual(
         __A     ,			[
             SPIECE_UNDERLINE + 'I',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '[UNK]',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             's',
             '[UNK]',
             '.',
         ]     ,			)



    @cached_property
    def 					__lowerCamelCase						(  self					):
     return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'					)



    @slow
    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   'Hello World!'
     __UpperCAmelCase					=   [35_389, 6_672, 49, 2]
     self.assertListEqual(__A     ,			self.big_tokenizer.encode(__A					)					)




    @slow
    def 					__lowerCamelCase						(  self					):
     # fmt: off
     __UpperCAmelCase					=   {'input_ids': [[11_073, 82_783, 18, 26, 82_783, 549, 51_540, 248, 17_209, 1_301, 217, 20, 215_186, 1_325, 147, 17_209, 1_301, 217, 20, 56_370, 53, 122_020, 20, 16_477, 27, 87_355, 4_548, 20, 4_728, 78_392, 17, 159_969, 18, 26, 24_491, 629, 15, 538, 22_704, 5_439, 15, 2_788, 24_491, 9_885, 15, 43_534, 605, 15, 814, 18_403, 33_200, 29, 15, 43_534, 24_458, 12_410, 111, 24_966, 83_669, 9_637, 144_068, 26, 850, 22_346, 27, 147, 24_966, 83_669, 83_490, 26, 39_113, 735, 27, 689, 656, 2_800, 1_339, 4_600, 53, 122_020, 115_785, 34, 816, 1_339, 46_887, 18, 147, 53_905, 1_951, 42_238, 41_170, 17_732, 834, 436, 15, 27_523, 98_733, 217, 147, 5_542, 4_981, 930, 17_347, 16, 2], [20_091, 629, 94, 82_786, 58, 490, 20, 1_528, 84, 53_905, 344, 80_592, 110_128, 18_822, 5_267, 1_306, 62, 152_537, 308, 7_997, 401, 124_427, 549, 35_442, 225, 109, 15_055, 25_748, 147, 7_119, 43_712, 34, 767, 135_366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63_784, 119_466, 17, 147_808, 88_214, 18, 656, 81, 32, 3_296, 10_280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=__A     ,			model_name='microsoft/xprophetnet-large-wiki100-cased'     ,			revision='1acad1643ddd54a44df6a1b797ada8373685d90e'     ,			)












'''simple docstring'''




import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
    import sqlitea

    import sqlalchemy






class UpperCAmelCase							(		UpperCAmelCase_							):



    def __init__(  self     ,			__A     ,			__A     ,			__A = None     ,			__A = None     ,			__A = False     ,			**__A     ,			):
     super().__init__(features=__A     ,			cache_dir=__A     ,			keep_in_memory=__A     ,			**__A					)
     __UpperCAmelCase					=   Sql(
         cache_dir=__A     ,			features=__A     ,			sql=__A     ,			con=__A     ,			**__A     ,			)




    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   None
     __UpperCAmelCase					=   None
     __UpperCAmelCase					=   None
     __UpperCAmelCase					=   None

     self.builder.download_and_prepare(
         download_config=__A     ,			download_mode=__A     ,			verification_mode=__A     ,			base_path=__A     ,			)

     # Build dataset for splits
     __UpperCAmelCase					=   self.builder.as_dataset(
         split='train'     ,			verification_mode=__A     ,			in_memory=self.keep_in_memory					)
     return dataset






class UpperCAmelCase							:



    def __init__(  self     ,			__A     ,			__A     ,			__A     ,			__A = None     ,			__A = None     ,			**__A     ,			):
     if num_proc is not None and num_proc <= 0:
      raise ValueError(f'num_proc {num_proc} must be an integer > 0.'					)

     __UpperCAmelCase					=   dataset
     __UpperCAmelCase					=   name
     __UpperCAmelCase					=   con
     __UpperCAmelCase					=   batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
     __UpperCAmelCase					=   num_proc
     __UpperCAmelCase					=   to_sql_kwargs



    def 					__lowerCamelCase						(  self					):
     __UpperCAmelCase					=   self.to_sql_kwargs.pop('sql'     ,			__A					)
     __UpperCAmelCase					=   self.to_sql_kwargs.pop('con'     ,			__A					)
     __UpperCAmelCase					=   self.to_sql_kwargs.pop('index'     ,			__A					)

     __UpperCAmelCase					=   self._write(index=__A     ,			**self.to_sql_kwargs					)
     return written



    def 					__lowerCamelCase						(  self     ,			__A					):
     __UpperCAmelCase					,    __UpperCAmelCase					,    __UpperCAmelCase					=   args
     __UpperCAmelCase					=   {**to_sql_kwargs, 'if_exists': 'append'} if offset > 0 else to_sql_kwargs
     __UpperCAmelCase					=   query_table(
         table=self.dataset.data     ,			key=slice(__A     ,			offset + self.batch_size					)     ,			indices=self.dataset._indices     ,			)
     __UpperCAmelCase					=   batch.to_pandas()
     __UpperCAmelCase					=   df.to_sql(self.name     ,			self.con     ,			index=__A     ,			**__A					)
     return num_rows or len(__A					)




    def 					__lowerCamelCase						(  self     ,			__A     ,			**__A					):
     __UpperCAmelCase					=   0

     if self.num_proc is None or self.num_proc == 1:
      for offset in logging.tqdm(
          range(0     ,			len(self.dataset					)     ,			self.batch_size					)     ,			unit='ba'     ,			disable=not logging.is_progress_bar_enabled()     ,			desc='Creating SQL from Arrow format'     ,			):
       written += self._batch_sql((offset, index, to_sql_kwargs)					)
     else:
      __UpperCAmelCase					,    __UpperCAmelCase					=   len(self.dataset					), self.batch_size
      with multiprocessing.Pool(self.num_proc					) as pool:
       for num_rows in logging.tqdm(
           pool.imap(
               self._batch_sql     ,			[(offset, index, to_sql_kwargs) for offset in range(0     ,			__A     ,			__A					)]     ,			)     ,			total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size     ,			unit='ba'     ,			disable=not logging.is_progress_bar_enabled()     ,			desc='Creating SQL from Arrow format'     ,			):
        written += num_rows

     return written





