



import argparse
import json
import os
from collections import OrderedDict

import numpy as np
import tensorflow as tf
import torch






def        A       (      _lowerCamelCase						):





       '''simple docstring'''




       _lowerCAmelCase  :							List[Any]				     =					os.path.join(args.tf_model_dir       ,				"parameters.json"						)
       _lowerCAmelCase  :							Optional[Any]				     =					json.loads(open(_lowerCamelCase						).read()						)
       if not params:
              raise ValueError(
                  F"It seems that the json file at {parameter_file} is empty. Make sure you have a correct json file."						)
       if not args.output.endswith(".pt"						):
              _lowerCAmelCase  :							Dict				     =					args.output + ".pt"
       _lowerCAmelCase  :							Any				     =					OrderedDict()
       with tf.device("/CPU:0"						):
              _lowerCAmelCase  :							List[Any]				     =					tf.train.load_checkpoint(args.tf_model_dir						)
              _lowerCAmelCase  :							Dict				     =					reader.get_variable_to_shape_map()
              for key_name in shapes.keys():
                     _lowerCAmelCase  :							Tuple				     =					reader.get_tensor(_lowerCamelCase						).astype(np.floataa						)
                     if key_name.endswith("/adam_m"						) or key_name.endswith("/adam_v"						):
                            continue
                     if key_name.startswith("pasts/"						):
                            if key_name.startswith("pasts/mlp"						):
                                   _lowerCAmelCase  :							Any				     =					int(key_name[9]						)
                            elif key_name.startswith("pasts/out"						):
                                   _lowerCAmelCase  :							Union[str, Any]				     =					8
                            _lowerCAmelCase  :							Union[str, Any]				     =					"model.sqout.%d.weight" % (player * 2)  # enter to nn.Sequencial with Tanh, so 2 at a time
                            _lowerCAmelCase  :							str				     =					vnp.transpose([1, 0]						).copy()  # Mesh-Tensorflow is a diagonal matrix
                            _lowerCAmelCase  :							Tuple				     =					torch.tensor(_lowerCamelCase						)
                     elif key_name.startswith("model/moe"						):
                            _lowerCAmelCase  :							Optional[int]				     =					int(key_name[9:].split("/"						)[0]						)
                            if key_name.endswith("/switch_gating/kernel"						):
                                   _lowerCAmelCase  :							List[str]				     =					"model.blocks.%d.feed_forward.mlp.router.classifier.weight" % player
                                   _lowerCAmelCase  :							Tuple				     =					vnp.transpose([1, 0]						).copy()  # Mesh-Tensorflow is a diagonal matrix
                                   _lowerCAmelCase  :							Optional[int]				     =					torch.tensor(_lowerCamelCase						)
                            elif key_name.endswith("/softmlp/kernel"						):
                                   _lowerCAmelCase  :							int				     =					"model.blocks.%d.feed_forward.soft_bypass_mlp.weight" % player
                                   _lowerCAmelCase  :							List[Any]				     =					vnp.transpose([1, 0]						).copy()  # Mesh-Tensorflow is a diagonal matrix
                                   _lowerCAmelCase  :							List[Any]				     =					torch.tensor(_lowerCamelCase						)
                            elif key_name.endswith("/wo/kernel"						) or key_name.endswith("/wi/kernel"						):
                                   _lowerCAmelCase  :							List[Any]				     =					key_name[-9:-7]
                                   for i in range(16						):
                                          _lowerCAmelCase  :							Tuple				     =					"model.blocks.%d.feed_forward.mlp.experts.expert_%d.%s.weight" % (player, i, nlayer)
                                          _lowerCAmelCase  :							Any				     =					(
                                              vnp[i].transpose([1, 0]						).copy()
                                          )  # In Mesh-Tensorflow, it is one array, so it is divided
                                          _lowerCAmelCase  :							Any				     =					torch.tensor(_lowerCamelCase						)
                     elif key_name.startswith("model/mlp"						):
                            _lowerCAmelCase  :							List[Any]				     =					int(key_name[9:].split("/"						)[0]						)
                            if key_name.endswith("/p1/kernel"						):
                                   _lowerCAmelCase  :							Optional[int]				     =					"model.blocks.%d.feed_forward.mlp.wi.weight" % player
                                   _lowerCAmelCase  :							str				     =					vnp.transpose([1, 0]						).copy()  # Mesh-Tensorflow is a diagonal matrix
                                   _lowerCAmelCase  :							List[Any]				     =					torch.tensor(_lowerCamelCase						)
                            elif key_name.endswith("/p1/bias"						):
                                   _lowerCAmelCase  :							str				     =					"model.blocks.%d.feed_forward.mlp.wi.bias" % player
                                   _lowerCAmelCase  :							List[Any]				     =					vnp.copy()  # same because it is one dimensional
                                   _lowerCAmelCase  :							int				     =					torch.tensor(_lowerCamelCase						)
                            elif key_name.endswith("/p2/kernel"						):
                                   _lowerCAmelCase  :							Union[str, Any]				     =					"model.blocks.%d.feed_forward.mlp.wo.weight" % player
                                   _lowerCAmelCase  :							str				     =					vnp.transpose([1, 0]						).copy()  # Mesh-Tensorflow is a diagonal matrix
                                   _lowerCAmelCase  :							Any				     =					torch.tensor(_lowerCamelCase						)
                            elif key_name.endswith("/p2/bias"						):
                                   _lowerCAmelCase  :							Union[str, Any]				     =					"model.blocks.%d.feed_forward.mlp.wo.bias" % player
                                   _lowerCAmelCase  :							List[str]				     =					vnp.copy()  # same because it is one dimensional
                                   _lowerCAmelCase  :							Dict				     =					torch.tensor(_lowerCamelCase						)
                     elif key_name.startswith("model/ln"						):
                            _lowerCAmelCase  :							Tuple				     =					int(key_name[8:].split("/"						)[0]						)
                            if key_name.endswith("/b"						):
                                   _lowerCAmelCase  :							Dict				     =					"model.blocks.%d.feed_forward.norm.bias" % player
                                   _lowerCAmelCase  :							Dict				     =					vnp.copy()  # same because it is one dimensional
                                   _lowerCAmelCase  :							Union[str, Any]				     =					torch.tensor(_lowerCamelCase						)
                            elif key_name.endswith("/g"						):
                                   _lowerCAmelCase  :							str				     =					"model.blocks.%d.feed_forward.norm.weight" % player
                                   _lowerCAmelCase  :							Any				     =					vnp.copy()  # same because it is one dimensional
                                   _lowerCAmelCase  :							str				     =					torch.tensor(_lowerCamelCase						)
                     elif key_name.startswith("model/att"						):
                            _lowerCAmelCase  :							Union[str, Any]				     =					int(key_name[9:].split("/"						)[0]						)
                            if key_name.endswith("/qkv/kernel"						):
                                   _lowerCAmelCase  :							Any				     =					vnp.copy()  # Compute same dimension as Mesh-tensorflow using einsum
                                   _lowerCAmelCase  :							Tuple				     =					state[:, 0, :, :]
                                   _lowerCAmelCase  :							int				     =					state[:, 1, :, :]
                                   _lowerCAmelCase  :							List[str]				     =					state[:, 2, :, :]
                                   _lowerCAmelCase  :							Dict				     =					(
                                       state_q.reshape([state_q.shape[0], state_q.shape[1] * state_q.shape[2]]						)
                                       .transpose([1, 0]						)
                                       .copy()
                                   )  # Mesh-Tensorflow is a diagonal matrix
                                   _lowerCAmelCase  :							Any				     =					(
                                       state_k.reshape([state_k.shape[0], state_k.shape[1] * state_k.shape[2]]						)
                                       .transpose([1, 0]						)
                                       .copy()
                                   )  # Mesh-Tensorflow is a diagonal matrix
                                   _lowerCAmelCase  :							Optional[Any]				     =					(
                                       state_v.reshape([state_v.shape[0], state_v.shape[1] * state_v.shape[2]]						)
                                       .transpose([1, 0]						)
                                       .copy()
                                   )  # Mesh-Tensorflow is a diagonal matrix
                                   _lowerCAmelCase  :							str				     =					"model.blocks.%d.self_attn.self_attn.q_proj.weight" % player
                                   _lowerCAmelCase  :							Any				     =					torch.tensor(_lowerCamelCase						)
                                   _lowerCAmelCase  :							Any				     =					"model.blocks.%d.self_attn.self_attn.k_proj.weight" % player
                                   _lowerCAmelCase  :							Optional[Any]				     =					torch.tensor(_lowerCamelCase						)
                                   _lowerCAmelCase  :							Tuple				     =					"model.blocks.%d.self_attn.self_attn.v_proj.weight" % player
                                   _lowerCAmelCase  :							Any				     =					torch.tensor(_lowerCamelCase						)
                            elif key_name.endswith("/o/kernel"						):
                                   _lowerCAmelCase  :							str				     =					"model.blocks.%d.self_attn.self_attn.out_proj.weight" % player
                                   _lowerCAmelCase  :							Dict				     =					(
                                       vnp.reshape([vnp.shape[0] * vnp.shape[1], vnp.shape[2]]						).transpose([1, 0]						).copy()
                                   )  # Mesh-Tensorflow is a diagonal matrix
                                   _lowerCAmelCase  :							int				     =					torch.tensor(_lowerCamelCase						)
                     elif key_name.startswith("model/an"						):
                            _lowerCAmelCase  :							Union[str, Any]				     =					int(key_name[8:].split("/"						)[0]						)
                            if key_name.endswith("/b"						):
                                   _lowerCAmelCase  :							Optional[int]				     =					"model.blocks.%d.self_attn.norm.bias" % player
                                   _lowerCAmelCase  :							Optional[Any]				     =					vnp.copy()  # same because it is one dimensional
                                   _lowerCAmelCase  :							Union[str, Any]				     =					torch.tensor(_lowerCamelCase						)
                            elif key_name.endswith("/g"						):
                                   _lowerCAmelCase  :							Tuple				     =					"model.blocks.%d.self_attn.norm.weight" % player
                                   _lowerCAmelCase  :							Dict				     =					vnp.copy()  # same because it is one dimensional
                                   _lowerCAmelCase  :							int				     =					torch.tensor(_lowerCamelCase						)
                     elif (
                         key_name.startswith("model/wte"						)
                         or key_name.startswith("model/wpe"						)
                         or key_name.startswith("model/ete"						)
                     ):
                            _lowerCAmelCase  :							Tuple				     =					{"wte": "embed_tokens", "wpe": "position_embeddings", "ete": "extra_position_embeddings"}[
                                key_name[-3:]
                            ]
                            _lowerCAmelCase  :							Union[str, Any]				     =					"model.%s.weight" % nlayer
                            _lowerCAmelCase  :							Union[str, Any]				     =					vnp.copy()  # same in embedded
                            _lowerCAmelCase  :							Any				     =					torch.tensor(_lowerCamelCase						)
                            if key_name.startswith("model/wte"						):
                                   _lowerCAmelCase  :							Any				     =					"lm_head.weight"
                                   _lowerCAmelCase  :							str				     =					vnp.copy()  # same in embedded
                                   _lowerCAmelCase  :							Optional[int]				     =					torch.tensor(_lowerCamelCase						)
                     elif key_name.startswith("model/wob"						):
                            _lowerCAmelCase  :							Optional[int]				     =					"final_logits_bias"
                            _lowerCAmelCase  :							Dict				     =					vnp.copy()  # same in embedded
                            _lowerCAmelCase  :							List[str]				     =					state.reshape((1, -1)						)
                            _lowerCAmelCase  :							Union[str, Any]				     =					torch.tensor(_lowerCamelCase						)
                     elif key_name == "model/dense/kernel":
                            _lowerCAmelCase  :							Union[str, Any]				     =					"model.last_project.weight"
                            _lowerCAmelCase  :							str				     =					vnp.transpose([1, 0]						).copy()  # Mesh-Tensorflow is a diagonal matrix
                            _lowerCAmelCase  :							Any				     =					torch.tensor(_lowerCamelCase						)
                     elif key_name == "model/dense_1/bias":
                            _lowerCAmelCase  :							Optional[Any]				     =					"model.last_project.bias"
                            _lowerCAmelCase  :							Optional[int]				     =					vnp.copy()  # same because it is one dimensional
                            _lowerCAmelCase  :							Dict				     =					torch.tensor(_lowerCamelCase						)
       torch.save(_lowerCamelCase       ,				args.output						)


if __name__ == "__main__":
      _snake_case	=						argparse.ArgumentParser(
          description="model converter.", formatter_class=argparse.ArgumentDefaultsHelpFormatter
      )
      parser.add_argument("--tf_model_dir", metavar="PATH", type=str, required=True, help="import model")
      parser.add_argument("--output", metavar="PATH", type=str, required=True, help="output model")
      _snake_case	=						parser.parse_args()
      convert_tf_gptsan_to_pt(args)





import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case	=						logging.get_logger(__name__)

_snake_case	=						{
    "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/config.json",
    "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/config.json",
}



class 						UpperCAmelCase_					( a):
							lowerCamelCase__         =	'xlnet'
							lowerCamelCase__         =	['mems']
							lowerCamelCase__         =	{
							    'n_token': 'vocab_size',  # Backward compatibility
							    'hidden_size': 'd_model',
							    'num_attention_heads': 'n_head',
							    'num_hidden_layers': 'n_layer',
							}







							def __init__(     self,    __a=3_2000,    __a=1024,    __a=24,    __a=16,    __a=4096,    __a="gelu",    __a=True,    __a="bi",    __a=0.02,    __a=1E-12,    __a=0.1,    __a=512,    __a=None,    __a=True,    __a=False,    __a=False,    __a=-1,    __a=False,    __a="last",    __a=True,    __a="tanh",    __a=0.1,    __a=5,    __a=5,    __a=5,    __a=1,    __a=2,    **__a,    ):

														'''simple docstring'''

														_lowerCAmelCase  :							int				     =					vocab_size
														_lowerCAmelCase  :							Optional[int]				     =					d_model
														_lowerCAmelCase  :							Tuple				     =					n_layer
														_lowerCAmelCase  :							List[Any]				     =					n_head
														if d_model % n_head != 0:
																					raise ValueError(f"'d_model % n_head' ({d_model % n_head}) should be equal to 0")
														if "d_head" in kwargs:
																					if kwargs["d_head"] != d_model // n_head:
																												raise ValueError(
																												    f"`d_head` ({kwargs['d_head']}) should be equal to `d_model // n_head` ({d_model // n_head})")
														_lowerCAmelCase  :							Optional[int]				     =					d_model // n_head
														_lowerCAmelCase  :							List[str]				     =					ff_activation
														_lowerCAmelCase  :							Tuple				     =					d_inner
														_lowerCAmelCase  :							List[Any]				     =					untie_r
														_lowerCAmelCase  :							List[str]				     =					attn_type

														_lowerCAmelCase  :							Union[str, Any]				     =					initializer_range
														_lowerCAmelCase  :							Any				     =					layer_norm_eps

														_lowerCAmelCase  :							List[Any]				     =					dropout
														_lowerCAmelCase  :							Optional[int]				     =					mem_len
														_lowerCAmelCase  :							Union[str, Any]				     =					reuse_len
														_lowerCAmelCase  :							List[str]				     =					bi_data
														_lowerCAmelCase  :							List[str]				     =					clamp_len
														_lowerCAmelCase  :							Any				     =					same_length

														_lowerCAmelCase  :							List[str]				     =					summary_type
														_lowerCAmelCase  :							int				     =					summary_use_proj
														_lowerCAmelCase  :							Optional[Any]				     =					summary_activation
														_lowerCAmelCase  :							Tuple				     =					summary_last_dropout
														_lowerCAmelCase  :							Union[str, Any]				     =					start_n_top
														_lowerCAmelCase  :							Optional[int]				     =					end_n_top

														_lowerCAmelCase  :							Tuple				     =					bos_token_id
														_lowerCAmelCase  :							List[Any]				     =					pad_token_id
														_lowerCAmelCase  :							Dict				     =					eos_token_id

														if "use_cache" in kwargs:
																					warnings.warn(
																					    "The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`"
																					    " instead.",    __a,    )
																					_lowerCAmelCase  :							Union[str, Any]				     =					kwargs["use_cache"]

														_lowerCAmelCase  :							Union[str, Any]				     =					use_mems_eval
														_lowerCAmelCase  :							Any				     =					use_mems_train
														super().__init__(pad_token_id=__a,    bos_token_id=__a,    eos_token_id=__a,    **__a)







							@property
							def      snake_case__   (     self):

														'''simple docstring'''

														logger.info(f"The model {self.model_type} is one of the few models that has no sequence length limit.")
														return -1







							@max_position_embeddings.setter
							def      snake_case__   (     self,    __a):

														'''simple docstring'''

														raise NotImplementedError(
														    f"The model {self.model_type} is one of the few models that has no sequence length limit.")
