'''simple docstring'''
import os
import re
import shutil
import sys
import tempfile
import unittest

import black


UpperCamelCase__ =			os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, 'utils'))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
UpperCamelCase__ =			'''    def __init__(self, config):
        super().__init__()
        self.transform = BertPredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = nn.Parameter(torch.zeros(config.vocab_size))

        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states
'''


class        _UpperCAmelCase						(		unittest.TestCase			):




    def 				lowerCAmelCase__				(			self  :       List[Any]			):






       '''simple docstring'''






       lowercase_	:    Any           =     tempfile.mkdtemp()
       os.makedirs(os.path.join(self.transformer_dir  ,     "models/bert/"			)			)
       lowercase_	:    List[str]           =     self.transformer_dir
       shutil.copy(
           os.path.join(_A  ,     "src/transformers/models/bert/modeling_bert.py"			)  ,     os.path.join(self.transformer_dir  ,     "models/bert/modeling_bert.py"			)  ,     )




    def 				lowerCAmelCase__				(			self  :       int			):






       '''simple docstring'''






       lowercase_	:    List[str]           =     'src/transformers'
       shutil.rmtree(self.transformer_dir			)




    def 				lowerCAmelCase__				(			self  :       Optional[int]  ,     a  :       int  ,     a  :       int  ,     a  :       Optional[Any]  ,     a  :       Dict=None			):






       '''simple docstring'''






       lowercase_	:    int           =     comment + f"""\nclass {class_name}(nn.Module):\n""" + class_code
       if overwrite_result is not None:
          lowercase_	:    Dict           =     comment + f"""\nclass {class_name}(nn.Module):\n""" + overwrite_result
       lowercase_	:    List[Any]           =     black.Mode(target_versions={black.TargetVersion.PYaa}  ,     line_length=1_1_9			)
       lowercase_	:    List[str]           =     black.format_str(_A  ,     mode=_A			)
       lowercase_	:    List[Any]           =     os.path.join(self.transformer_dir  ,     "new_code.py"			)
       with open(_A  ,     "w"  ,     newline="\n"			) as f:
          f.write(_A			)
       if overwrite_result is None:
          self.assertTrue(len(check_copies.is_copy_consistent(_A			)			) == 0			)
       else:
          check_copies.is_copy_consistent(f.name  ,     overwrite=_A			)
          with open(_A  ,     "r"			) as f:
             self.assertTrue(f.read()  ,     _A			)




    def 				lowerCAmelCase__				(			self  :       Union[str, Any]			):






       '''simple docstring'''






       lowercase_	:    Optional[int]           =     check_copies.find_code_in_transformers("models.bert.modeling_bert.BertLMPredictionHead"			)
       self.assertEqual(_A  ,     _A			)




    def 				lowerCAmelCase__				(			self  :       List[Any]			):






       '''simple docstring'''






       self.check_copy_consistency(
           "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"  ,     "BertLMPredictionHead"  ,     REFERENCE_CODE + "\n"  ,     )

       # With no empty line at the end
       self.check_copy_consistency(
           "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead"  ,     "BertLMPredictionHead"  ,     _A  ,     )

       # Copy consistency with rename
       self.check_copy_consistency(
           "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"  ,     "TestModelLMPredictionHead"  ,     re.sub("Bert"  ,     "TestModel"  ,     _A			)  ,     )

       # Copy consistency with a really long name
       lowercase_	:    Any           =     'TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason'
       self.check_copy_consistency(
           f"""# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}"""  ,     f"""{long_class_name}LMPredictionHead"""  ,     re.sub("Bert"  ,     _A  ,     _A			)  ,     )

       # Copy consistency with overwrite
       self.check_copy_consistency(
           "# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel"  ,     "TestModelLMPredictionHead"  ,     _A  ,     overwrite_result=re.sub("Bert"  ,     "TestModel"  ,     _A			)  ,     )





    def 				lowerCAmelCase__				(			self  :       List[str]			):






       '''simple docstring'''






       lowercase_	:    Union[str, Any]           =     check_copies.LOCALIZED_READMES['README_zh-hans.md']

       lowercase_	:    Union[str, Any]           =     (
           '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the'
           ' Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for'
           ' Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong'
           ' Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1.'
           ' **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),'
           ' released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and'
           ' lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same'
           ' method has been applied to compress GPT2 into'
           ' [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into'
           ' [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),'
           ' Multilingual BERT into'
           ' [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German'
           ' version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**'
           ' (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders'
           ' as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang'
           ' Luong, Quoc V. Le, Christopher D. Manning.'
       )
       lowercase_	:    Optional[Any]           =     (
           '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
           ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
           ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
           ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
       )
       lowercase_	:    Optional[int]           =     (
           '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
           ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
           ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
           ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1.'
           ' **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文'
           ' [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and'
           ' lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same'
           ' method has been applied to compress GPT2 into'
           ' [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into'
           ' [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),'
           ' Multilingual BERT into'
           ' [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German'
           ' version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自'
           ' Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather'
           ' than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,'
           ' Christopher D. Manning 发布。\n'
       )

       lowercase_	:    Union[str, Any]           =     check_copies.convert_to_localized_md(
           _A  ,     _A  ,     localized_readme["format_model_list"]			)

       self.assertFalse(_A			)
       self.assertEqual(_A  ,     _A			)

       lowercase_	:    Any           =     check_copies.convert_to_localized_md(
           _A  ,     _A  ,     localized_readme["format_model_list"]			)

       # Check whether the number of models is equal to README.md after conversion.
       self.assertTrue(_A			)

       lowercase_	:    int           =     (
           '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the'
           ' Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for'
           ' Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong'
           ' Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.'
       )
       lowercase_	:    int           =     (
           '1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and'
           ' the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
           ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
           ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
       )
       lowercase_	:    Tuple           =     (
           '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
           ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
           ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
           ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
       )

       lowercase_	:    Union[str, Any]           =     check_copies.convert_to_localized_md(
           _A  ,     _A  ,     localized_readme["format_model_list"]			)

       # Check if the model link is synchronized.
       self.assertEqual(_A  ,     _A			)












'''simple docstring'''
from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__ =			logging.get_logger(__name__)

UpperCamelCase__ =			{
    'abeja/gpt-neox-japanese-2.7b': 'https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/config.json',
}


class        _UpperCAmelCase						(		snake_case			):
    __lowerCamelCase:   List[Any]		       =    'gpt_neox_japanese'




    def __init__(			self  :       List[str]  ,     a  :       List[Any]=3_2_0_0_0  ,     a  :       Union[str, Any]=2_5_6_0  ,     a  :       Optional[Any]=3_2  ,     a  :       Any=3_2  ,     a  :       str=4  ,     a  :       Optional[int]="gelu"  ,     a  :       Optional[Any]=1.00  ,     a  :       Dict=1_0_0_0_0  ,     a  :       List[Any]=2_0_4_8  ,     a  :       Dict=0.02  ,     a  :       int=1e-5  ,     a  :       Optional[int]=True  ,     a  :       Union[str, Any]=3_1_9_9_6  ,     a  :       List[Any]=3_1_9_9_9  ,     a  :       List[str]=0.1  ,     a  :       Dict=0.0  ,     **a  :       Union[str, Any]  ,     ):






       '''simple docstring'''






       super().__init__(bos_token_id=a  ,     eos_token_id=a  ,     **a			)
       lowercase_	:    int           =     vocab_size
       lowercase_	:    int           =     max_position_embeddings
       lowercase_	:    List[str]           =     hidden_size
       lowercase_	:    Any           =     num_hidden_layers
       lowercase_	:    Any           =     num_attention_heads
       lowercase_	:    List[Any]           =     intermediate_multiple_size
       lowercase_	:    List[str]           =     hidden_act
       lowercase_	:    Optional[int]           =     rotary_pct
       lowercase_	:    Tuple           =     rotary_emb_base
       lowercase_	:    Optional[Any]           =     initializer_range
       lowercase_	:    List[str]           =     layer_norm_eps
       lowercase_	:    List[str]           =     use_cache
       lowercase_	:    Any           =     attention_dropout
       lowercase_	:    List[Any]           =     hidden_dropout




