


"""simple docstring"""


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


a_       							=    datasets.utils.logging.get_logger(__name__)
@dataclass
class snake_case      (  datasets.BuilderConfig):
		__UpperCamelCase           =			None
		__UpperCamelCase           =			"utf-8"
		__UpperCamelCase           =			None
		__UpperCamelCase           =			None
		__UpperCamelCase           =			True  # deprecated
		__UpperCamelCase           =			None  # deprecated
		__UpperCamelCase           =			10 << 20  # 10MB
		__UpperCamelCase           =			None





class snake_case      (  datasets.ArrowBasedBuilder):
		__UpperCamelCase           =			JsonConfig







		def  a_ (					self      :       str		)       ->						Dict:





						'''simple docstring'''







						if self.config.block_size is not None:
										logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"		)
										_A					  =			self.config.block_size
						if self.config.use_threads is not True:
										logger.warning(
										    "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."		)
						if self.config.newlines_in_values is not None:
										raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported"		)
						return datasets.DatasetInfo(features=self.config.features		)







		def  a_ (					self      :       Union[str, Any]    ,   a__      :       str		)       ->						List[str]:





						'''simple docstring'''







						if not self.config.data_files:
										raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""		)
						_A					  =			dl_manager.download_and_extract(self.config.data_files		)
						if isinstance(A__    ,   (str, list, tuple)		):
										_A					  =			data_files
										if isinstance(A__    ,   A__		):
														_A					  =			[files]
										_A					  =			[dl_manager.iter_files(A__		) for file in files]
										return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,   gen_kwargs={"files": files}		)]
						_A					  =			[]
						for split_name, files in data_files.items():
										if isinstance(A__    ,   A__		):
														_A					  =			[files]
										_A					  =			[dl_manager.iter_files(A__		) for file in files]
										splits.append(datasets.SplitGenerator(name=A__    ,   gen_kwargs={"files": files}		)		)
						return splits







		def  a_ (					self      :       Union[str, Any]    ,   a__      :       List[Any]		)       ->						Optional[int]:





						'''simple docstring'''







						if self.config.features is not None:
										# adding missing columns
										for column_name in set(self.config.features		) - set(pa_table.column_names		):
														_A					  =			self.config.features.arrow_schema.field(A__		).type
														_A					  =			pa_table.append_column(A__    ,   pa.array([None] * len(A__		)    ,   type=A__		)		)
										# more expensive cast to support nested structures with keys in a different order
										# allows str <-> int/float or str to Audio for example
										_A					  =			table_cast(A__    ,   self.config.features.arrow_schema		)
						return pa_table







		def  a_ (					self      :       List[Any]    ,   a__      :       int		)       ->						Tuple:





						'''simple docstring'''







						for file_idx, file in enumerate(itertools.chain.from_iterable(A__		)		):
										# If the file is one json object and if we need to look at the list of items in one specific field
										if self.config.field is not None:
														with open(A__    ,   encoding=self.config.encoding    ,   errors=self.config.encoding_errors		) as f:
																		_A					  =			json.load(A__		)

														# We keep only the field we are interested in
														_A					  =			dataset[self.config.field]

														# We accept two format: a list of dicts or a dict of lists
														if isinstance(A__    ,   (list, tuple)		):
																		_A					  =			set().union(*[row.keys() for row in dataset]		)
																		_A					  =			{col: [row.get(A__		) for row in dataset] for col in keys}
														else:
																		_A					  =			dataset
														_A					  =			pa.Table.from_pydict(A__		)
														yield file_idx, self._cast_table(A__		)

										# If the file has one json object per line
										else:
														with open(A__    ,   "rb"		) as f:
																		_A					  =			0
																		# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																		# Set a default minimum value of 16kB if the chunk size is really small
																		_A					  =			max(self.config.chunksize // 32    ,   16 << 10		)
																		_A					  =			(
																		    self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
																		)
																		while True:
																						_A					  =			f.read(self.config.chunksize		)
																						if not batch:
																										break
																						# Finish current line
																						try:
																										batch += f.readline()
																						except (AttributeError, io.UnsupportedOperation):
																										batch += readline(A__		)
																						# PyArrow only accepts utf-8 encoded bytes
																						if self.config.encoding != "utf-8":
																										_A					  =			batch.decode(self.config.encoding    ,   errors=A__		).encode("utf-8"		)
																						try:
																										while True:
																														try:
																																		_A					  =			paj.read_json(
																																		    io.BytesIO(A__		)    ,   read_options=paj.ReadOptions(block_size=A__		)		)
																																		break
																														except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																		if (
																																		    isinstance(A__    ,   pa.ArrowInvalid		)
																																		    and "straddling" not in str(A__		)
																																		    or block_size > len(A__		)
																																		):
																																						raise
																																		else:
																																						# Increase the block size in case it was too small.
																																						# The block size will be reset for the next file.
																																						logger.debug(
																																						    F"""Batch of {len(A__		)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""		)
																																						block_size *= 2
																						except pa.ArrowInvalid as e:
																										try:
																														with open(
																														    A__    ,   encoding=self.config.encoding    ,   errors=self.config.encoding_errors		) as f:
																																		_A					  =			json.load(A__		)
																										except json.JSONDecodeError:
																														logger.error(F"""Failed to read file '{file}' with error {type(A__		)}: {e}"""		)
																														raise e
																										# If possible, parse the file as a list of json objects and exit the loop
																										if isinstance(A__    ,   A__		):  # list is the only sequence type supported in JSON
																														try:
																																		_A					  =			set().union(*[row.keys() for row in dataset]		)
																																		_A					  =			{col: [row.get(A__		) for row in dataset] for col in keys}
																																		_A					  =			pa.Table.from_pydict(A__		)
																														except (pa.ArrowInvalid, AttributeError) as e:
																																		logger.error(F"""Failed to read file '{file}' with error {type(A__		)}: {e}"""		)
																																		raise ValueError(F"""Not able to read records in the JSON file at {file}."""		) from None
																														yield file_idx, self._cast_table(A__		)
																														break
																										else:
																														logger.error(F"""Failed to read file '{file}' with error {type(A__		)}: {e}"""		)
																														raise ValueError(
																														    F"""Not able to read records in the JSON file at {file}. """
																														    F"""You should probably indicate the field of the JSON file containing your records. """
																														    F"""This JSON file contain the following fields: {str(list(dataset.keys()		)		)}. """
																														    F"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """		) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																						yield (file_idx, batch_idx), self._cast_table(A__		)
																						batch_idx += 1



"""simple docstring"""


import argparse
import json
import os
from collections import OrderedDict

import numpy as np
import tensorflow as tf
import torch






def      a__ (				__lowercase   )	->	List[Any]:
				_A					  =			os.path.join(args.tf_model_dir     ,			"parameters.json"   )
				_A					  =			json.loads(open(__lowercase   ).read()   )
				if not params:
								raise ValueError(
								    f"""It seems that the json file at {parameter_file} is empty. Make sure you have a correct json file."""   )
				if not args.output.endswith(".pt"   ):
								_A					  =			args.output + ".pt"
				_A					  =			OrderedDict()
				with tf.device("/CPU:0"   ):
								_A					  =			tf.train.load_checkpoint(args.tf_model_dir   )
								_A					  =			reader.get_variable_to_shape_map()
								for key_name in shapes.keys():
												_A					  =			reader.get_tensor(__lowercase   ).astype(np.floataa   )
												if key_name.endswith("/adam_m"   ) or key_name.endswith("/adam_v"   ):
																continue
												if key_name.startswith("pasts/"   ):
																if key_name.startswith("pasts/mlp"   ):
																				_A					  =			int(key_name[9]   )
																elif key_name.startswith("pasts/out"   ):
																				_A					  =			8
																_A					  =			"model.sqout.%d.weight" % (player * 2)  # enter to nn.Sequencial with Tanh, so 2 at a time
																_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/moe"   ):
																_A					  =			int(key_name[9:].split("/"   )[0]   )
																if key_name.endswith("/switch_gating/kernel"   ):
																				_A					  =			"model.blocks.%d.feed_forward.mlp.router.classifier.weight" % player
																				_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/softmlp/kernel"   ):
																				_A					  =			"model.blocks.%d.feed_forward.soft_bypass_mlp.weight" % player
																				_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/wo/kernel"   ) or key_name.endswith("/wi/kernel"   ):
																				_A					  =			key_name[-9:-7]
																				for i in range(16   ):
																								_A					  =			"model.blocks.%d.feed_forward.mlp.experts.expert_%d.%s.weight" % (player, i, nlayer)
																								_A					  =			(
																								    vnp[i].transpose([1, 0]   ).copy()
																								)  # In Mesh-Tensorflow, it is one array, so it is divided
																								_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/mlp"   ):
																_A					  =			int(key_name[9:].split("/"   )[0]   )
																if key_name.endswith("/p1/kernel"   ):
																				_A					  =			"model.blocks.%d.feed_forward.mlp.wi.weight" % player
																				_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/p1/bias"   ):
																				_A					  =			"model.blocks.%d.feed_forward.mlp.wi.bias" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/p2/kernel"   ):
																				_A					  =			"model.blocks.%d.feed_forward.mlp.wo.weight" % player
																				_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/p2/bias"   ):
																				_A					  =			"model.blocks.%d.feed_forward.mlp.wo.bias" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/ln"   ):
																_A					  =			int(key_name[8:].split("/"   )[0]   )
																if key_name.endswith("/b"   ):
																				_A					  =			"model.blocks.%d.feed_forward.norm.bias" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/g"   ):
																				_A					  =			"model.blocks.%d.feed_forward.norm.weight" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/att"   ):
																_A					  =			int(key_name[9:].split("/"   )[0]   )
																if key_name.endswith("/qkv/kernel"   ):
																				_A					  =			vnp.copy()  # Compute same dimension as Mesh-tensorflow using einsum
																				_A					  =			state[:, 0, :, :]
																				_A					  =			state[:, 1, :, :]
																				_A					  =			state[:, 2, :, :]
																				_A					  =			(
																				    state_q.reshape([state_q.shape[0], state_q.shape[1] * state_q.shape[2]]   )
																				    .transpose([1, 0]   )
																				    .copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			(
																				    state_k.reshape([state_k.shape[0], state_k.shape[1] * state_k.shape[2]]   )
																				    .transpose([1, 0]   )
																				    .copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			(
																				    state_v.reshape([state_v.shape[0], state_v.shape[1] * state_v.shape[2]]   )
																				    .transpose([1, 0]   )
																				    .copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			"model.blocks.%d.self_attn.self_attn.q_proj.weight" % player
																				_A					  =			torch.tensor(__lowercase   )
																				_A					  =			"model.blocks.%d.self_attn.self_attn.k_proj.weight" % player
																				_A					  =			torch.tensor(__lowercase   )
																				_A					  =			"model.blocks.%d.self_attn.self_attn.v_proj.weight" % player
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/o/kernel"   ):
																				_A					  =			"model.blocks.%d.self_attn.self_attn.out_proj.weight" % player
																				_A					  =			(
																				    vnp.reshape([vnp.shape[0] * vnp.shape[1], vnp.shape[2]]   ).transpose([1, 0]   ).copy()
																				)  # Mesh-Tensorflow is a diagonal matrix
																				_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/an"   ):
																_A					  =			int(key_name[8:].split("/"   )[0]   )
																if key_name.endswith("/b"   ):
																				_A					  =			"model.blocks.%d.self_attn.norm.bias" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
																elif key_name.endswith("/g"   ):
																				_A					  =			"model.blocks.%d.self_attn.norm.weight" % player
																				_A					  =			vnp.copy()  # same because it is one dimensional
																				_A					  =			torch.tensor(__lowercase   )
												elif (
												    key_name.startswith("model/wte"   )
												    or key_name.startswith("model/wpe"   )
												    or key_name.startswith("model/ete"   )
												):
																_A					  =			{"wte": "embed_tokens", "wpe": "position_embeddings", "ete": "extra_position_embeddings"}[
																    key_name[-3:]
																]
																_A					  =			"model.%s.weight" % nlayer
																_A					  =			vnp.copy()  # same in embedded
																_A					  =			torch.tensor(__lowercase   )
																if key_name.startswith("model/wte"   ):
																				_A					  =			"lm_head.weight"
																				_A					  =			vnp.copy()  # same in embedded
																				_A					  =			torch.tensor(__lowercase   )
												elif key_name.startswith("model/wob"   ):
																_A					  =			"final_logits_bias"
																_A					  =			vnp.copy()  # same in embedded
																_A					  =			state.reshape((1, -1)   )
																_A					  =			torch.tensor(__lowercase   )
												elif key_name == "model/dense/kernel":
																_A					  =			"model.last_project.weight"
																_A					  =			vnp.transpose([1, 0]   ).copy()  # Mesh-Tensorflow is a diagonal matrix
																_A					  =			torch.tensor(__lowercase   )
												elif key_name == "model/dense_1/bias":
																_A					  =			"model.last_project.bias"
																_A					  =			vnp.copy()  # same because it is one dimensional
																_A					  =			torch.tensor(__lowercase   )
				torch.save(__lowercase     ,			args.output   )


if __name__ == "__main__":
		a_       							=    argparse.ArgumentParser(
		    description="model converter.", formatter_class=argparse.ArgumentDefaultsHelpFormatter
		)
		parser.add_argument("--tf_model_dir", metavar="PATH", type=str, required=True, help="import model")
		parser.add_argument("--output", metavar="PATH", type=str, required=True, help="output model")
		a_       							=    parser.parse_args()
		convert_tf_gptsan_to_pt(args)