






'''simple docstring'''







import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin

class _lowercase			(       _lowercase       ,   unittest.TestCase	):
							a          = XLMTokenizer
							a          = False




							def 	lowerCamelCase_       (							self:       Optional[Any]  ):
														super().setUp()

														# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
														lowerCamelCase__      :			Union[str, Any] 						= [
														    """l""",
														    """o""",
														    """w""",
														    """e""",
														    """r""",
														    """s""",
														    """t""",
														    """i""",
														    """d""",
														    """n""",
														    """w</w>""",
														    """r</w>""",
														    """t</w>""",
														    """lo""",
														    """low""",
														    """er</w>""",
														    """low</w>""",
														    """lowest</w>""",
														    """newer</w>""",
														    """wider</w>""",
														    """<unk>""",
														]
														lowerCamelCase__      :			Any 						= dict(zip(UpperCamelCase__    ,							range(len(UpperCamelCase__  )  )  )  )
														lowerCamelCase__      :			Optional[Any] 						= ["""l o 123""", """lo w 1456""", """e r</w> 1789""", """"""]

														lowerCamelCase__      :			Tuple 						= os.path.join(self.tmpdirname    ,							VOCAB_FILES_NAMES["""vocab_file"""]  )
														lowerCamelCase__      :			Dict 						= os.path.join(self.tmpdirname    ,							VOCAB_FILES_NAMES["""merges_file"""]  )
														with open(self.vocab_file    ,							"""w"""  ) as fp:
																					fp.write(json.dumps(UpperCamelCase__  )  )
														with open(self.merges_file    ,							"""w"""  ) as fp:
																					fp.write("""\n""".join(UpperCamelCase__  )  )




							def 	lowerCamelCase_       (							self:       Tuple    ,							UpperCamelCase__:       Optional[int]  ):
														lowerCamelCase__      :			Tuple 						= """lower newer"""
														lowerCamelCase__      :			List[Any] 						= """lower newer"""
														return input_text, output_text




							def 	lowerCamelCase_       (							self:       List[Any]  ):
														lowerCamelCase__      :			Union[str, Any] 						= XLMTokenizer(self.vocab_file    ,							self.merges_file  )

														lowerCamelCase__      :			str 						= """lower"""
														lowerCamelCase__      :			int 						= ["""low""", """er</w>"""]
														lowerCamelCase__      :			str 						= tokenizer.tokenize(UpperCamelCase__  )
														self.assertListEqual(UpperCamelCase__    ,							UpperCamelCase__  )

														lowerCamelCase__      :			Tuple 						= tokens + ["""<unk>"""]
														lowerCamelCase__      :			str 						= [14, 15, 20]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__  )    ,							UpperCamelCase__  )




							@slow
							def 	lowerCamelCase_       (							self:       List[str]  ):
														lowerCamelCase__      :			List[str] 						= XLMTokenizer.from_pretrained("""xlm-mlm-en-2048"""  )

														lowerCamelCase__      :			Tuple 						= tokenizer.encode("""sequence builders"""    ,							add_special_tokens=UpperCamelCase__  )
														lowerCamelCase__      :			Optional[int] 						= tokenizer.encode("""multi-sequence build"""    ,							add_special_tokens=UpperCamelCase__  )

														lowerCamelCase__      :			Any 						= tokenizer.build_inputs_with_special_tokens(UpperCamelCase__  )
														lowerCamelCase__      :			List[Any] 						= tokenizer.build_inputs_with_special_tokens(UpperCamelCase__    ,							UpperCamelCase__  )

														assert encoded_sentence == [0] + text + [1]
														assert encoded_pair == [0] + text + [1] + text_a + [1]









'''simple docstring'''







# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


_A			:    Dict					  ='''3'''

print('''Python version:''', sys.version)

print('''OS platform:''', platform.platform())
print('''OS architecture:''', platform.machine())

try:
			import torch

			print('''Torch version:''', torch.__version__)
			print('''Cuda available:''', torch.cuda.is_available())
			print('''Cuda version:''', torch.version.cuda)
			print('''CuDNN version:''', torch.backends.cudnn.version())
			print('''Number of GPUs available:''', torch.cuda.device_count())
except ImportError:
			print('''Torch version:''', None)

try:
			import transformers

			print('''transformers version:''', transformers.__version__)
except ImportError:
			print('''transformers version:''', None)

