"""simple docstring"""




import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 		__A							(				SCREAMING_SNAKE_CASE_					,			unittest.TestCase	):
     _UpperCamelCase							:    Any	      =	OpenAIGPTTokenizer
     _UpperCamelCase							:    Tuple	      =	OpenAIGPTTokenizerFast
     _UpperCamelCase							:    List[Any]	      =	True
     _UpperCamelCase							:    Tuple	      =	False






     def 					__A						(	self						):
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       _lowerCAmelCase		: Union[str, Any]						    =  [
           """l""",
           """o""",
           """w""",
           """e""",
           """r""",
           """s""",
           """t""",
           """i""",
           """d""",
           """n""",
           """w</w>""",
           """r</w>""",
           """t</w>""",
           """lo""",
           """low""",
           """er</w>""",
           """low</w>""",
           """lowest</w>""",
           """newer</w>""",
           """wider</w>""",
           """<unk>""",
       ]
       _lowerCAmelCase		: Union[str, Any]						    =  dict(zip(a__       ,  range(len(a__						)						)						)						)
       _lowerCAmelCase		: int						    =  ["""#version: 0.2""", """l o""", """lo w""", """e r</w>""", """"""]

       _lowerCAmelCase		: List[str]						    =  os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES["""vocab_file"""]						)
       _lowerCAmelCase		: int						    =  os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES["""merges_file"""]						)
       with open(self.vocab_file       ,  """w"""						) as fp:
         fp.write(json.dumps(a__						)						)
       with open(self.merges_file       ,  """w"""						) as fp:
         fp.write("""\n""".join(a__						)						)






     def 					__A						(	self       ,  a__						):
       return "lower newer", "lower newer"






     def 					__A						(	self						):
       _lowerCAmelCase		: Optional[Any]						    =  OpenAIGPTTokenizer(self.vocab_file       ,  self.merges_file						)

       _lowerCAmelCase		: Optional[int]						    =  """lower"""
       _lowerCAmelCase		: Optional[Any]						    =  ["""low""", """er</w>"""]
       _lowerCAmelCase		: Union[str, Any]						    =  tokenizer.tokenize(a__						)
       self.assertListEqual(a__       ,  a__						)

       _lowerCAmelCase		: int						    =  tokens + ["""<unk>"""]
       _lowerCAmelCase		: List[Any]						    =  [14, 15, 20]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(a__						)       ,  a__						)






     def 					__A						(	self       ,  a__=15						):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
           _lowerCAmelCase		: Any						    =  self.rust_tokenizer_class.from_pretrained(a__       ,  **a__						)

           # Simple input
           _lowerCAmelCase		: Tuple						    =  """This is a simple input"""
           _lowerCAmelCase		: Tuple						    =  ["""This is a simple input 1""", """This is a simple input 2"""]
           _lowerCAmelCase		: List[Any]						    =  ("""This is a simple input""", """This is a pair""")
           _lowerCAmelCase		: Any						    =  [
               ("""This is a simple input 1""", """This is a simple input 2"""),
               ("""This is a simple pair 1""", """This is a simple pair 2"""),
           ]

           # Simple input tests
           self.assertRaises(a__       ,  tokenizer_r.encode       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Simple input
           self.assertRaises(a__       ,  tokenizer_r.encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Simple input
           self.assertRaises(
               a__       ,  tokenizer_r.batch_encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""       ,  )

           # Pair input
           self.assertRaises(a__       ,  tokenizer_r.encode       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Pair input
           self.assertRaises(a__       ,  tokenizer_r.encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""						)

           # Pair input
           self.assertRaises(
               a__       ,  tokenizer_r.batch_encode_plus       ,  a__       ,  max_length=a__       ,  padding="""max_length"""       ,  )






     def 					__A						(	self						):
       pass





@require_ftfy
@require_spacy
@require_tokenizers
class 		__A							(				SCREAMING_SNAKE_CASE_	):
     pass





"""simple docstring"""




import pickle
import unittest

import torch

from accelerate import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils import require_cpu
@require_cpu
class 		__A							(				unittest.TestCase	):






     def 					__A						(	self						):
       _lowerCAmelCase		: str						    =  torch.nn.Linear(10       ,  10						)
       _lowerCAmelCase		: Optional[Any]						    =  torch.optim.SGD(model.parameters()       ,  0.1						)
       _lowerCAmelCase		: Optional[Any]						    =  Accelerator()
       _lowerCAmelCase		: Tuple						    =  accelerator.prepare(a__						)
       try:
         pickle.loads(pickle.dumps(a__						)						)
       except Exception as e:
         self.fail(F"Accelerated optimizer pickling failed with {e}"						)
       AcceleratorState._reset_state()




