


# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


__UpperCAmelCase : List[Any] 			=				"3"

print("Python version:", sys.version)

print("OS platform:", platform.platform())
print("OS architecture:", platform.machine())

try:
    import torch

    print("Torch version:", torch.__version__)
    print("Cuda available:", torch.cuda.is_available())
    print("Cuda version:", torch.version.cuda)
    print("CuDNN version:", torch.backends.cudnn.version())
    print("Number of GPUs available:", torch.cuda.device_count())
except ImportError:
    print("Torch version:", None)

try:
    import transformers

    print("transformers version:", transformers.__version__)
except ImportError:
    print("transformers version:", None)





import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class    UpperCAmelCase_	(							_a,     unittest.TestCase):
       '''simple docstring'''
       __UpperCamelCase      :     str							=					DebertaTokenizer
       __UpperCamelCase      :     Optional[int]							=					True
       __UpperCamelCase      :     Optional[int]							=					DebertaTokenizerFast






       def  _lowercase   (     self						):




              """simple docstring"""




              super().setUp()

              # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
              UpperCamelCase    :	Optional[int]   	=		[
                  '''l''',
                  '''o''',
                  '''w''',
                  '''e''',
                  '''r''',
                  '''s''',
                  '''t''',
                  '''i''',
                  '''d''',
                  '''n''',
                  '''\u0120''',
                  '''\u0120l''',
                  '''\u0120n''',
                  '''\u0120lo''',
                  '''\u0120low''',
                  '''er''',
                  '''\u0120lowest''',
                  '''\u0120newer''',
                  '''\u0120wider''',
                  '''[UNK]''',
              ]
              UpperCamelCase    :	Tuple   	=		dict(zip(__SCREAMING_SNAKE_CASE ,					range(len(__SCREAMING_SNAKE_CASE						)						)						)						)
              UpperCamelCase    :	Any   	=		['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
              UpperCamelCase    :	List[Any]   	=		{'''unk_token''': '''[UNK]'''}

              UpperCamelCase    :	Union[str, Any]   	=		os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['''vocab_file''']						)
              UpperCamelCase    :	Optional[int]   	=		os.path.join(self.tmpdirname ,					VOCAB_FILES_NAMES['''merges_file''']						)
              with open(self.vocab_file ,					'''w''' ,					encoding='''utf-8'''						) as fp:
                     fp.write(json.dumps(__SCREAMING_SNAKE_CASE						) + '''\n'''						)
              with open(self.merges_file ,					'''w''' ,					encoding='''utf-8'''						) as fp:
                     fp.write('''\n'''.join(__SCREAMING_SNAKE_CASE						)						)






       def  _lowercase   (     self ,					**__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              kwargs.update(self.special_tokens_map						)
              return self.tokenizer_class.from_pretrained(self.tmpdirname ,					**__SCREAMING_SNAKE_CASE						)






       def  _lowercase   (     self ,					__SCREAMING_SNAKE_CASE						):




              """simple docstring"""




              UpperCamelCase    :	int   	=		'''lower newer'''
              UpperCamelCase    :	Union[str, Any]   	=		'''lower newer'''
              return input_text, output_text






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	List[str]   	=		self.get_tokenizer()
              UpperCamelCase    :	int   	=		'''lower newer'''
              UpperCamelCase    :	Union[str, Any]   	=		['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
              UpperCamelCase    :	Tuple   	=		tokenizer.tokenize(__SCREAMING_SNAKE_CASE						)
              self.assertListEqual(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)

              UpperCamelCase    :	List[Any]   	=		tokens + [tokenizer.unk_token]
              UpperCamelCase    :	Optional[int]   	=		[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
              self.assertListEqual(tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE						) ,					__SCREAMING_SNAKE_CASE						)






       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	int   	=		self.get_tokenizer()
              UpperCamelCase    :	Optional[Any]   	=		tokenizer('''Hello''' ,					'''World'''						)
              UpperCamelCase    :	List[str]   	=		[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
              self.assertListEqual(tokd['''token_type_ids'''] ,					__SCREAMING_SNAKE_CASE						)






       @slow
       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	Union[str, Any]   	=		self.tokenizer_class.from_pretrained('''microsoft/deberta-base'''						)

              UpperCamelCase    :	Optional[Any]   	=		tokenizer.encode('''sequence builders''' ,					add_special_tokens=__SCREAMING_SNAKE_CASE						)
              UpperCamelCase    :	int   	=		tokenizer.encode('''multi-sequence build''' ,					add_special_tokens=__SCREAMING_SNAKE_CASE						)

              UpperCamelCase    :	Dict   	=		tokenizer.encode(
                  '''sequence builders''' ,					add_special_tokens=__SCREAMING_SNAKE_CASE ,					add_prefix_space=__SCREAMING_SNAKE_CASE						)
              UpperCamelCase    :	Union[str, Any]   	=		tokenizer.encode(
                  '''sequence builders''' ,					'''multi-sequence build''' ,					add_special_tokens=__SCREAMING_SNAKE_CASE ,					add_prefix_space=__SCREAMING_SNAKE_CASE						)

              UpperCamelCase    :	List[str]   	=		tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE						)
              UpperCamelCase    :	Union[str, Any]   	=		tokenizer.build_inputs_with_special_tokens(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)

              assert encoded_sentence == encoded_text_from_decode
              assert encoded_pair == encoded_pair_from_decode






       @slow
       def  _lowercase   (     self						):




              """simple docstring"""




              UpperCamelCase    :	Optional[Any]   	=		[self.tokenizer_class]
              if self.test_rust_tokenizer:
                     tokenizer_classes.append(self.rust_tokenizer_class						)

              for tokenizer_class in tokenizer_classes:
                     UpperCamelCase    :	Optional[int]   	=		tokenizer_class.from_pretrained('''microsoft/deberta-base'''						)

                     UpperCamelCase    :	str   	=		[
                         '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
                         '''ALBERT incorporates two parameter reduction techniques''',
                         '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
                         ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
                         ''' vocabulary embedding.''',
                     ]

                     UpperCamelCase    :	Union[str, Any]   	=		tokenizer(__SCREAMING_SNAKE_CASE ,					padding=__SCREAMING_SNAKE_CASE						)
                     UpperCamelCase    :	Union[str, Any]   	=		[tokenizer.decode(__SCREAMING_SNAKE_CASE ,					skip_special_tokens=__SCREAMING_SNAKE_CASE						) for seq in encoding['''input_ids''']]

                     # fmt: off
                     UpperCamelCase    :	int   	=		{
                         '''input_ids''': [
                             [1, 2_118, 11_126, 565, 35, 83, 25_191, 163, 18_854, 13, 12_156, 12, 16_101, 25_376, 13_807, 9, 22_205, 27_893, 1_635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                             [1, 2_118, 11_126, 565, 24_536, 80, 43_797, 4_878, 7_373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                             [1, 133, 78, 65, 16, 10, 3_724, 1_538, 33_183, 11_303, 43_797, 1_938, 4, 870, 24_165, 29_105, 5, 739, 32_644, 33_183, 11_303, 36_173, 88, 80, 650, 7_821, 45_940, 6, 52, 2_559, 5, 1_836, 9, 5, 7_397, 13_171, 31, 5, 1_836, 9, 32_644, 33_183, 11_303, 4, 2]
                         ],
                         '''token_type_ids''': [
                             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                             [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                         ],
                         '''attention_mask''': [
                             [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                             [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                             [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                         ]
                     }
                     # fmt: on

                     UpperCamelCase    :	List[str]   	=		[
                         '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
                         '''ALBERT incorporates two parameter reduction techniques''',
                         '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
                         ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
                         ''' vocabulary embedding.''',
                     ]

                     self.assertDictEqual(encoding.data ,					__SCREAMING_SNAKE_CASE						)

                     for expected, decoded in zip(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						):
                            self.assertEqual(__SCREAMING_SNAKE_CASE ,					__SCREAMING_SNAKE_CASE						)

