

import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
						from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


A_							:			Any											= get_tests_dir("fixtures/test_sentencepiece.model")

A_							:			List[Any]											= {"target_lang": "fi", "source_lang": "en"}
A_							:			Dict											= ">>zh<<"
A_							:			Dict											= "Helsinki-NLP/"

if is_torch_available():
						A_							:			int											= "pt"
elif is_tf_available():
						A_							:			Optional[Any]											= "tf"
else:
						A_							:			int											= "jax"





@require_sentencepiece
class    lowerCamelCase						(A__			,unittest.TestCase			):
			lowerCamelCase__   :		int							=							MarianTokenizer
			lowerCamelCase__   :		Dict							=							False
			lowerCamelCase__   :		Union[str, Any]							=							True

			def 					SCREAMING_SNAKE_CASE				(					self				:						Any       )   ->    Optional[Any]:
						super().setUp()
						SCREAMING_SNAKE_CASE__								=      ["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
						SCREAMING_SNAKE_CASE__								=      dict(zip(__UpperCAmelCase    ,				range(len(__UpperCAmelCase       )       )       )       )
						SCREAMING_SNAKE_CASE__								=      Path(self.tmpdirname       )
						save_json(__UpperCAmelCase    ,				save_dir / VOCAB_FILES_NAMES["""vocab"""]       )
						save_json(__UpperCAmelCase    ,				save_dir / VOCAB_FILES_NAMES["""tokenizer_config_file"""]       )
						if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
									copyfile(__UpperCAmelCase    ,				save_dir / VOCAB_FILES_NAMES["""source_spm"""]       )
									copyfile(__UpperCAmelCase    ,				save_dir / VOCAB_FILES_NAMES["""target_spm"""]       )

						SCREAMING_SNAKE_CASE__								=      MarianTokenizer.from_pretrained(self.tmpdirname       )
						tokenizer.save_pretrained(self.tmpdirname       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						Optional[Any]    ,				**__UpperCAmelCase				:						List[Any]       )   ->    MarianTokenizer:
						return MarianTokenizer.from_pretrained(self.tmpdirname    ,				**__UpperCAmelCase       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						Optional[int]    ,				__UpperCAmelCase				:						int       )   ->    List[Any]:
						return (
						    "This is a test",
						    "This is a test",
						)

			def 					SCREAMING_SNAKE_CASE				(					self				:						Tuple       )   ->    List[Any]:
						SCREAMING_SNAKE_CASE__								=      """</s>"""
						SCREAMING_SNAKE_CASE__								=      0

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCAmelCase       )    ,				__UpperCAmelCase       )
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCAmelCase       )    ,				__UpperCAmelCase       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						List[str]       )   ->    Any:
						SCREAMING_SNAKE_CASE__								=      list(self.get_tokenizer().get_vocab().keys()       )

						self.assertEqual(vocab_keys[0]    ,				"""</s>"""       )
						self.assertEqual(vocab_keys[1]    ,				"""<unk>"""       )
						self.assertEqual(vocab_keys[-1]    ,				"""<pad>"""       )
						self.assertEqual(len(__UpperCAmelCase       )    ,				9       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						Optional[Any]       )   ->    Tuple:
						self.assertEqual(self.get_tokenizer().vocab_size    ,				9       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						Optional[Any]       )   ->    Optional[int]:
						SCREAMING_SNAKE_CASE__								=      MarianTokenizer.from_pretrained(F"""{ORG_NAME}opus-mt-en-de"""       )
						SCREAMING_SNAKE_CASE__								=      en_de_tokenizer(["""I am a small frog"""]    ,				return_tensors=__UpperCAmelCase       )
						self.assertIsInstance(__UpperCAmelCase    ,				__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      [3_8, 1_2_1, 1_4, 6_9_7, 3_8_8_4_8, 0]
						self.assertListEqual(__UpperCAmelCase    ,				batch.input_ids[0]       )

						SCREAMING_SNAKE_CASE__								=      tempfile.mkdtemp()
						en_de_tokenizer.save_pretrained(__UpperCAmelCase       )
						SCREAMING_SNAKE_CASE__								=      [x.name for x in Path(__UpperCAmelCase       ).glob("""*"""       )]
						self.assertIn("""source.spm"""    ,				__UpperCAmelCase       )
						MarianTokenizer.from_pretrained(__UpperCAmelCase       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						Tuple       )   ->    Tuple:
						SCREAMING_SNAKE_CASE__								=      self.get_tokenizer()

						SCREAMING_SNAKE_CASE__								=      tok(
						    ["""I am a small frog""" * 1_0_0_0, """I am a small frog"""]    ,				padding=__UpperCAmelCase    ,				truncation=__UpperCAmelCase    ,				return_tensors=__UpperCAmelCase       )
						self.assertIsInstance(__UpperCAmelCase    ,				__UpperCAmelCase       )
						self.assertEqual(batch.input_ids.shape    ,				(2, 5_1_2)       )

			def 					SCREAMING_SNAKE_CASE				(					self				:						str       )   ->    str:
						SCREAMING_SNAKE_CASE__								=      self.get_tokenizer()
						SCREAMING_SNAKE_CASE__								=      tok(["""I am a tiny frog""", """I am a small frog"""]    ,				padding=__UpperCAmelCase    ,				return_tensors=__UpperCAmelCase       )
						self.assertIsInstance(__UpperCAmelCase    ,				__UpperCAmelCase       )
						self.assertEqual(batch_smaller.input_ids.shape    ,				(2, 1_0)       )

			@slow
			def 					SCREAMING_SNAKE_CASE				(					self				:						List[Any]       )   ->    int:
						# fmt: off
						SCREAMING_SNAKE_CASE__								=      {"""input_ids""": [[4_3_4_9_5, 4_6_2, 2_0, 4_2_1_6_4, 1_3_6_9, 5_2, 4_6_4, 1_3_2, 1_7_0_3, 4_9_2, 1_3, 7_4_9_1, 3_8_9_9_9, 6, 8, 4_6_4, 1_3_2, 1_7_0_3, 4_9_2, 1_3, 4_6_6_9, 3_7_8_6_7, 1_3, 7_5_2_5, 2_7, 1_5_9_3, 9_8_8, 1_3, 3_3_9_7_2, 7_0_2_9, 6, 2_0, 8_2_5_1, 3_8_3, 2, 2_7_0, 5_8_6_6, 3_7_8_8, 2, 2_3_5_3, 8_2_5_1, 1_2_3_3_8, 2, 1_3_9_5_8, 3_8_7, 2, 3_6_2_9, 6_9_5_3, 1_8_8, 2_9_0_0, 2, 1_3_9_5_8, 8_0_1_1, 1_1_5_0_1, 2_3, 8_4_6_0, 4_0_7_3, 3_4_0_0_9, 2_0, 4_3_5, 1_1_4_3_9, 2_7, 8, 8_4_6_0, 4_0_7_3, 6_0_0_4, 2_0, 9_9_8_8, 3_7_5, 2_7, 3_3, 2_6_6, 1_9_4_5, 1_0_7_6, 1_3_5_0, 3_7_8_6_7, 3_2_8_8, 5, 5_7_7, 1_0_7_6, 4_3_7_4, 8, 5_0_8_2, 5, 2_6_4_5_3, 2_5_7, 5_5_6, 4_0_3, 2, 2_4_2, 1_3_2, 3_8_3, 3_1_6, 4_9_2, 8, 1_0_7_6_7, 6, 3_1_6, 3_0_4, 4_2_3_9, 3, 0], [1_4_8, 1_5_7_2_2, 1_9, 1_8_3_9, 1_2, 1_3_5_0, 1_3, 2_2_3_2_7, 5_0_8_2, 5_4_1_8, 4_7_5_6_7, 3_5_9_3_8, 5_9, 3_1_8, 1_9_5_5_2, 1_0_8, 2_1_8_3, 5_4, 1_4_9_7_6, 4_8_3_5, 3_2, 5_4_7, 1_1_1_4, 8, 3_1_5, 2_4_1_7, 5, 9_2, 1_9_0_8_8, 3, 0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0], [3_6, 6_3_9_5, 1_2_5_7_0, 3_9_1_4_7, 1_1_5_9_7, 6, 2_6_6, 4, 4_5_4_0_5, 7_2_9_6, 3, 0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0, 5_8_1_0_0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=__UpperCAmelCase    ,				model_name="""Helsinki-NLP/opus-mt-en-de"""    ,				revision="""1a8c2263da11e68e50938f97e10cd57820bd504c"""    ,				decode_kwargs={"""use_source_tokenizer""": True}    ,				)





			def 					SCREAMING_SNAKE_CASE				(					self				:						Any       )   ->    int:
						SCREAMING_SNAKE_CASE__								=      MarianTokenizer.from_pretrained("""hf-internal-testing/test-marian-two-vocabs"""       )

						SCREAMING_SNAKE_CASE__								=      """Tämä on testi"""
						SCREAMING_SNAKE_CASE__								=      """This is a test"""

						SCREAMING_SNAKE_CASE__								=      [7_6, 7, 2_0_4_7, 2]
						SCREAMING_SNAKE_CASE__								=      [6_9, 1_2, 1_1, 9_4_0, 2]

						SCREAMING_SNAKE_CASE__								=      tokenizer(__UpperCAmelCase       ).input_ids
						self.assertListEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )

						SCREAMING_SNAKE_CASE__								=      tokenizer(text_target=__UpperCAmelCase       ).input_ids
						self.assertListEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )

						SCREAMING_SNAKE_CASE__								=      tokenizer.decode(__UpperCAmelCase    ,				skip_special_tokens=__UpperCAmelCase       )
						self.assertEqual(__UpperCAmelCase    ,				__UpperCAmelCase       )





"""simple docstring"""




# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

A_							:			Tuple											= "2.13.1"

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse("3.7"):
						raise ImportWarning(
						    "To use `datasets`, Python>=3.7 is required, and the current version of Python doesn't match this condition."
						)

if version.parse(pyarrow.__version__).major < 8:
						raise ImportWarning(
						    "To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn't match this condition.\n"
						    "If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`."
						)

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

A_							:			Tuple											= concatenate_datasets
A_							:			List[str]											= DownloadConfig
A_							:			int											= DownloadManager
A_							:			Optional[Any]											= DownloadMode
A_							:			Optional[int]											= DownloadConfig
A_							:			List[Any]											= DownloadMode
A_							:			Any											= DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager


