





import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 			__A       (      __a ,unittest.TestCase  ):




      """simple docstring"""







      UpperCAmelCase__						=      CodeGenTokenizer
      UpperCAmelCase__						=      CodeGenTokenizerFast
      UpperCAmelCase__						=      True
      UpperCAmelCase__						=      {"add_prefix_space": True}
      UpperCAmelCase__						=      False


      def 				__snake_case			(		self):



             """simple docstring"""





             super().setUp()

             # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
             _lowerCamelCase							:     str	=		[
                 '''l''',
                 '''o''',
                 '''w''',
                 '''e''',
                 '''r''',
                 '''s''',
                 '''t''',
                 '''i''',
                 '''d''',
                 '''n''',
                 '''\u0120''',
                 '''\u0120l''',
                 '''\u0120n''',
                 '''\u0120lo''',
                 '''\u0120low''',
                 '''er''',
                 '''\u0120lowest''',
                 '''\u0120newer''',
                 '''\u0120wider''',
                 '''<unk>''',
                 '''<|endoftext|>''',
             ]
             _lowerCamelCase							:     str	=		dict(zip(lowerCAmelCase_      ,							range(len(lowerCAmelCase_))))
             _lowerCamelCase							:     List[str]	=		['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
             _lowerCamelCase							:     Optional[Any]	=		{'''unk_token''': '''<unk>'''}

             _lowerCamelCase							:     int	=		os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['''vocab_file'''])
             _lowerCamelCase							:     int	=		os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES['''merges_file'''])
             with open(self.vocab_file      ,							'''w'''      ,							encoding='''utf-8''') as fp:
                    fp.write(json.dumps(lowerCAmelCase_) + '''\n''')
             with open(self.merges_file      ,							'''w'''      ,							encoding='''utf-8''') as fp:
                    fp.write('''\n'''.join(lowerCAmelCase_))


      def 				__snake_case			(		self      ,							**a__):



             """simple docstring"""





             kwargs.update(self.special_tokens_map)
             return CodeGenTokenizer.from_pretrained(self.tmpdirname      ,							**lowerCAmelCase_)


      def 				__snake_case			(		self      ,							**a__):



             """simple docstring"""





             kwargs.update(self.special_tokens_map)
             return CodeGenTokenizerFast.from_pretrained(self.tmpdirname      ,							**lowerCAmelCase_)


      def 				__snake_case			(		self      ,							a__):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		'''lower newer'''
             _lowerCamelCase							:     Optional[Any]	=		'''lower newer'''
             return input_text, output_text


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     List[Any]	=		CodeGenTokenizer(self.vocab_file      ,							self.merges_file      ,							**self.special_tokens_map)
             _lowerCamelCase							:     Tuple	=		'''lower newer'''
             _lowerCamelCase							:     List[Any]	=		['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
             _lowerCamelCase							:     Union[str, Any]	=		tokenizer.tokenize(lowerCAmelCase_      ,							add_prefix_space=lowerCAmelCase_)
             self.assertListEqual(lowerCAmelCase_      ,							lowerCAmelCase_)

             _lowerCamelCase							:     List[str]	=		tokens + [tokenizer.unk_token]
             _lowerCamelCase							:     Dict	=		[14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_)      ,							lowerCAmelCase_)


      def 				__snake_case			(		self):



             """simple docstring"""





             if not self.test_rust_tokenizer:
                    return

             _lowerCamelCase							:     Union[str, Any]	=		self.get_tokenizer()
             _lowerCamelCase							:     Dict	=		self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_)

             _lowerCamelCase							:     int	=		'''lower newer'''

             # Testing tokenization
             _lowerCamelCase							:     List[Any]	=		tokenizer.tokenize(lowerCAmelCase_      ,							add_prefix_space=lowerCAmelCase_)
             _lowerCamelCase							:     int	=		rust_tokenizer.tokenize(lowerCAmelCase_)
             self.assertListEqual(lowerCAmelCase_      ,							lowerCAmelCase_)

             # Testing conversion to ids without special tokens
             _lowerCamelCase							:     Optional[int]	=		tokenizer.encode(lowerCAmelCase_      ,							add_special_tokens=lowerCAmelCase_      ,							add_prefix_space=lowerCAmelCase_)
             _lowerCamelCase							:     Union[str, Any]	=		rust_tokenizer.encode(lowerCAmelCase_      ,							add_special_tokens=lowerCAmelCase_)
             self.assertListEqual(lowerCAmelCase_      ,							lowerCAmelCase_)

             # Testing conversion to ids with special tokens
             _lowerCamelCase							:     str	=		self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase_)
             _lowerCamelCase							:     Union[str, Any]	=		tokenizer.encode(lowerCAmelCase_      ,							add_prefix_space=lowerCAmelCase_)
             _lowerCamelCase							:     Tuple	=		rust_tokenizer.encode(lowerCAmelCase_)
             self.assertListEqual(lowerCAmelCase_      ,							lowerCAmelCase_)

             # Testing the unknown token
             _lowerCamelCase							:     Optional[int]	=		tokens + [rust_tokenizer.unk_token]
             _lowerCamelCase							:     Optional[Any]	=		[14, 15, 10, 9, 3, 2, 15, 19]
             self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCAmelCase_)      ,							lowerCAmelCase_)


      def 				__snake_case			(		self      ,							*a__      ,							**a__):



             """simple docstring"""





             pass


      def 				__snake_case			(		self      ,							a__=15):



             """simple docstring"""





             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                    with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                           _lowerCamelCase							:     str	=		self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_      ,							**lowerCAmelCase_)

                           # Simple input
                           _lowerCamelCase							:     Dict	=		'''This is a simple input'''
                           _lowerCamelCase							:     Optional[Any]	=		['''This is a simple input 1''', '''This is a simple input 2''']
                           _lowerCamelCase							:     str	=		('''This is a simple input''', '''This is a pair''')
                           _lowerCamelCase							:     Any	=		[
                               ('''This is a simple input 1''', '''This is a simple input 2'''),
                               ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                           ]

                           # Simple input tests
                           self.assertRaises(lowerCAmelCase_      ,							tokenizer_r.encode      ,							lowerCAmelCase_      ,							max_length=lowerCAmelCase_      ,							padding='''max_length''')

                           # Simple input
                           self.assertRaises(lowerCAmelCase_      ,							tokenizer_r.encode_plus      ,							lowerCAmelCase_      ,							max_length=lowerCAmelCase_      ,							padding='''max_length''')

                           # Simple input
                           self.assertRaises(
                               lowerCAmelCase_      ,							tokenizer_r.batch_encode_plus      ,							lowerCAmelCase_      ,							max_length=lowerCAmelCase_      ,							padding='''max_length'''      ,							)

                           # Pair input
                           self.assertRaises(lowerCAmelCase_      ,							tokenizer_r.encode      ,							lowerCAmelCase_      ,							max_length=lowerCAmelCase_      ,							padding='''max_length''')

                           # Pair input
                           self.assertRaises(lowerCAmelCase_      ,							tokenizer_r.encode_plus      ,							lowerCAmelCase_      ,							max_length=lowerCAmelCase_      ,							padding='''max_length''')

                           # Pair input
                           self.assertRaises(
                               lowerCAmelCase_      ,							tokenizer_r.batch_encode_plus      ,							lowerCAmelCase_      ,							max_length=lowerCAmelCase_      ,							padding='''max_length'''      ,							)


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Any	=		CodeGenTokenizer.from_pretrained(self.tmpdirname      ,							pad_token='''<pad>''')

             # Simple input
             _lowerCamelCase							:     Dict	=		'''This is a simple input'''
             _lowerCamelCase							:     Dict	=		['''This is a simple input looooooooong''', '''This is a simple input''']
             _lowerCamelCase							:     Any	=		('''This is a simple input''', '''This is a pair''')
             _lowerCamelCase							:     Tuple	=		[
                 ('''This is a simple input loooooong''', '''This is a simple input'''),
                 ('''This is a simple pair loooooong''', '''This is a simple pair'''),
             ]

             _lowerCamelCase							:     Union[str, Any]	=		tokenizer.pad_token_id

             _lowerCamelCase							:     Dict	=		tokenizer(lowerCAmelCase_      ,							padding='''max_length'''      ,							max_length=30      ,							return_tensors='''np''')
             _lowerCamelCase							:     List[str]	=		tokenizer(lowerCAmelCase_      ,							padding=lowerCAmelCase_      ,							truncate=lowerCAmelCase_      ,							return_tensors='''np''')
             _lowerCamelCase							:     str	=		tokenizer(*lowerCAmelCase_      ,							padding='''max_length'''      ,							max_length=60      ,							return_tensors='''np''')
             _lowerCamelCase							:     List[str]	=		tokenizer(lowerCAmelCase_      ,							padding=lowerCAmelCase_      ,							truncate=lowerCAmelCase_      ,							return_tensors='''np''')

             # s
             # test single string max_length padding
             self.assertEqual(out_s['''input_ids'''].shape[-1]      ,							30)
             self.assertTrue(pad_token_id in out_s['''input_ids'''])
             self.assertTrue(0 in out_s['''attention_mask'''])

             # s2
             # test automatic padding
             self.assertEqual(out_sa['''input_ids'''].shape[-1]      ,							33)
             # long slice doesn't have padding
             self.assertFalse(pad_token_id in out_sa['''input_ids'''][0])
             self.assertFalse(0 in out_sa['''attention_mask'''][0])
             # short slice does have padding
             self.assertTrue(pad_token_id in out_sa['''input_ids'''][1])
             self.assertTrue(0 in out_sa['''attention_mask'''][1])

             # p
             # test single pair max_length padding
             self.assertEqual(out_p['''input_ids'''].shape[-1]      ,							60)
             self.assertTrue(pad_token_id in out_p['''input_ids'''])
             self.assertTrue(0 in out_p['''attention_mask'''])

             # p2
             # test automatic padding pair
             self.assertEqual(out_pa['''input_ids'''].shape[-1]      ,							52)
             # long slice pair doesn't have padding
             self.assertFalse(pad_token_id in out_pa['''input_ids'''][0])
             self.assertFalse(0 in out_pa['''attention_mask'''][0])
             # short slice pair does have padding
             self.assertTrue(pad_token_id in out_pa['''input_ids'''][1])
             self.assertTrue(0 in out_pa['''attention_mask'''][1])


      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Union[str, Any]	=		'''$$$'''
             _lowerCamelCase							:     Union[str, Any]	=		CodeGenTokenizer.from_pretrained(self.tmpdirname      ,							bos_token=lowerCAmelCase_      ,							add_bos_token=lowerCAmelCase_)

             _lowerCamelCase							:     Optional[int]	=		'''This is a simple input'''
             _lowerCamelCase							:     Any	=		['''This is a simple input 1''', '''This is a simple input 2''']

             _lowerCamelCase							:     Any	=		tokenizer.bos_token_id

             _lowerCamelCase							:     Optional[int]	=		tokenizer(lowerCAmelCase_)
             _lowerCamelCase							:     Optional[Any]	=		tokenizer(lowerCAmelCase_)

             self.assertEqual(out_s.input_ids[0]      ,							lowerCAmelCase_)
             self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

             _lowerCamelCase							:     Any	=		tokenizer.decode(out_s.input_ids)
             _lowerCamelCase							:     Optional[int]	=		tokenizer.batch_decode(out_sa.input_ids)

             self.assertEqual(decode_s.split()[0]      ,							lowerCAmelCase_)
             self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))


      @slow
      def 				__snake_case			(		self):



             """simple docstring"""





             _lowerCamelCase							:     Optional[Any]	=		CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono''')

             _lowerCamelCase							:     List[str]	=		'''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
             _lowerCamelCase							:     Tuple	=		'''\nif len_a > len_b:      result = a\nelse:      result = b'''

             _lowerCamelCase							:     List[Any]	=		tokenizer.encode(lowerCAmelCase_)
             _lowerCamelCase							:     str	=		['''^#''', re.escape('''<|endoftext|>'''), '''^\'\'\'''', '''^\"\"\"''', '''\n\n\n''']
             _lowerCamelCase							:     Optional[Any]	=		tokenizer.decode(lowerCAmelCase_      ,							truncate_before_pattern=lowerCAmelCase_)
             self.assertEqual(lowerCAmelCase_      ,							lowerCAmelCase_)



      def 				__snake_case			(		self):



             """simple docstring"""





             pass








# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pathlib import Path

import torch

from ...utils import is_npu_available, is_xpu_available
from .config_args import ClusterConfig, default_json_config_file
from .config_utils import SubcommandHelpFormatter


_lowerCamelCase			       =      'Create a default config file for Accelerate with only a few flags set.'




def    __UpperCAmelCase(      lowercase_="no" ,    lowercase_ = default_json_config_file ,    lowercase_ = False      ):
       _lowerCamelCase							:     List[str]	=		Path(lowercase_      )
       path.parent.mkdir(parents=lowercase_ ,    exist_ok=lowercase_      )
       if path.exists():
              print(
                  F"""Configuration already exists at {save_location}, will not override. Run `accelerate config` manually or pass a different `save_location`."""      )
              return False
       _lowerCamelCase							:     List[Any]	=		mixed_precision.lower()
       if mixed_precision not in ["no", "fp16", "bf16", "fp8"]:
              raise ValueError(
                  F"""`mixed_precision` should be one of 'no', 'fp16', 'bf16', or 'fp8'. Received {mixed_precision}"""      )
       _lowerCamelCase							:     Optional[int]	=		{
           '''compute_environment''': '''LOCAL_MACHINE''',
           '''mixed_precision''': mixed_precision,
       }
       if torch.cuda.is_available():
              _lowerCamelCase							:     Any	=		torch.cuda.device_count()
              _lowerCamelCase							:     str	=		num_gpus
              _lowerCamelCase							:     Union[str, Any]	=		False
              if num_gpus > 1:
                     _lowerCamelCase							:     str	=		'''MULTI_GPU'''
              else:
                     _lowerCamelCase							:     List[str]	=		'''NO'''
       elif is_xpu_available() and use_xpu:
              _lowerCamelCase							:     Union[str, Any]	=		torch.xpu.device_count()
              _lowerCamelCase							:     int	=		num_xpus
              _lowerCamelCase							:     Dict	=		False
              if num_xpus > 1:
                     _lowerCamelCase							:     List[str]	=		'''MULTI_XPU'''
              else:
                     _lowerCamelCase							:     Any	=		'''NO'''
       elif is_npu_available():
              _lowerCamelCase							:     List[str]	=		torch.npu.device_count()
              _lowerCamelCase							:     Optional[int]	=		num_npus
              _lowerCamelCase							:     str	=		False
              if num_npus > 1:
                     _lowerCamelCase							:     List[Any]	=		'''MULTI_NPU'''
              else:
                     _lowerCamelCase							:     str	=		'''NO'''
       else:
              _lowerCamelCase							:     Tuple	=		0
              _lowerCamelCase							:     Optional[Any]	=		True
              _lowerCamelCase							:     List[str]	=		1
              _lowerCamelCase							:     Union[str, Any]	=		'''NO'''
       _lowerCamelCase							:     Any	=		ClusterConfig(**lowercase_      )
       config.to_json_file(lowercase_      )
       return path




def    __UpperCAmelCase(      lowercase_ ,    lowercase_      ):
       _lowerCamelCase							:     List[str]	=		parser.add_parser('''default''' ,    parents=lowercase_ ,    help=lowercase_ ,    formatter_class=lowercase_      )
       parser.add_argument(
           '''--config_file''' ,    default=lowercase_ ,    help=(
               '''The path to use to store the config file. Will default to a file named default_config.yaml in the cache '''
               '''location, which is the content of the environment `HF_HOME` suffixed with \'accelerate\', or if you don\'t have '''
               '''such an environment variable, your cache directory (\'~/.cache\' or the content of `XDG_CACHE_HOME`) suffixed '''
               '''with \'huggingface\'.'''
           ) ,    dest='''save_location''' ,    )

       parser.add_argument(
           '''--mixed_precision''' ,    choices=['''no''', '''fp16''', '''bf16'''] ,    type=lowercase_ ,    help='''Whether or not to use mixed precision training. '''
           '''Choose between FP16 and BF16 (bfloat16) training. '''
           '''BF16 training is only supported on Nvidia Ampere GPUs and PyTorch 1.10 or later.''' ,    default='''no''' ,    )
       parser.set_defaults(func=lowercase_      )
       return parser




def    __UpperCAmelCase(      lowercase_      ):
       _lowerCamelCase							:     str	=		write_basic_config(args.mixed_precision ,    args.save_location      )
       if config_file:
              print(F"""accelerate configuration saved at {config_file}"""      )

