



'''simple docstring'''




import argparse
import gc
import json
import os
import re

import torch
from huggingface_hub import hf_hub_download

from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedTokenizerFast, RwkvConfig
from transformers.modeling_utils import WEIGHTS_INDEX_NAME, shard_checkpoint


a     :    int      						=					{
    """169M""": 1_2,
    """430M""": 2_4,
    """1B5""": 2_4,
    """3B""": 3_2,
    """7B""": 3_2,
    """14B""": 4_0,
}

a     :    List[str]      						=					{
    """169M""": 7_6_8,
    """430M""": 1_0_2_4,
    """1B5""": 2_0_4_8,
    """3B""": 2_5_6_0,
    """7B""": 4_0_9_6,
    """14B""": 5_1_2_0,
}



def    __lowerCamelCase						(      _lowercase	)   ->    Optional[Any]:
							UpperCAmelCase  :	Optional[Any]							  =	list(state_dict.keys()	)
							for name in state_dict_keys:
														UpperCAmelCase  :	List[Any]							  =	state_dict.pop(_lowercase	)
														# emb -> embedding
														if name.startswith("""emb."""	):
																					UpperCAmelCase  :	Any							  =	name.replace("""emb."""			,       """embeddings."""	)
														# ln_0 -> pre_ln (only present at block 0)
														if name.startswith("""blocks.0.ln0"""	):
																					UpperCAmelCase  :	Dict							  =	name.replace("""blocks.0.ln0"""			,       """blocks.0.pre_ln"""	)
														# att -> attention
														UpperCAmelCase  :	Optional[Any]							  =	re.sub(R"""blocks\.(\d+)\.att"""			,       R"""blocks.\1.attention"""			,       _lowercase	)
														# ffn -> feed_forward
														UpperCAmelCase  :	List[Any]							  =	re.sub(R"""blocks\.(\d+)\.ffn"""			,       R"""blocks.\1.feed_forward"""			,       _lowercase	)
														# time_mix_k -> time_mix_key and reshape
														if name.endswith(""".time_mix_k"""	):
																					UpperCAmelCase  :	List[str]							  =	name.replace(""".time_mix_k"""			,       """.time_mix_key"""	)
														# time_mix_v -> time_mix_value and reshape
														if name.endswith(""".time_mix_v"""	):
																					UpperCAmelCase  :	Any							  =	name.replace(""".time_mix_v"""			,       """.time_mix_value"""	)
														# time_mix_r -> time_mix_key and reshape
														if name.endswith(""".time_mix_r"""	):
																					UpperCAmelCase  :	Optional[Any]							  =	name.replace(""".time_mix_r"""			,       """.time_mix_receptance"""	)

														if name != "head.weight":
																					UpperCAmelCase  :	Tuple							  =	"""rwkv.""" + name

														UpperCAmelCase  :	Union[str, Any]							  =	weight
							return state_dict



def    __lowerCamelCase						(      _lowercase			,       _lowercase			,       _lowercase			,       _lowercase=None			,       _lowercase=None			,       _lowercase=False			,       _lowercase=None	)   ->    Dict:
							# 1. If possible, build the tokenizer.
							if tokenizer_file is None:
														print("""No `--tokenizer_file` provided, we will use the default tokenizer."""	)
														UpperCAmelCase  :	Dict							  =	5_0_2_7_7
														UpperCAmelCase  :	str							  =	AutoTokenizer.from_pretrained("""EleutherAI/gpt-neox-20b"""	)
							else:
														UpperCAmelCase  :	List[Any]							  =	PreTrainedTokenizerFast(tokenizer_file=_lowercase	)
														UpperCAmelCase  :	int							  =	len(_lowercase	)
							tokenizer.save_pretrained(_lowercase	)

							# 2. Build the config
							UpperCAmelCase  :	Dict							  =	list(NUM_HIDDEN_LAYERS_MAPPING.keys()	)
							if size is None:
														# Try to infer size from the checkpoint name
														for candidate in possible_sizes:
																					if candidate in checkpoint_file:
																												UpperCAmelCase  :	Any							  =	candidate
																												break
														if size is None:
																					raise ValueError("""Could not infer the size, please provide it with the `--size` argument."""	)
							if size not in possible_sizes:
														raise ValueError(F'''`size` should be one of {possible_sizes}, got {size}.'''	)

							UpperCAmelCase  :	Any							  =	RwkvConfig(
							    vocab_size=_lowercase			,       num_hidden_layers=NUM_HIDDEN_LAYERS_MAPPING[size]			,       hidden_size=HIDEN_SIZE_MAPPING[size]			,       )
							config.save_pretrained(_lowercase	)

							# 3. Download model file then convert state_dict
							UpperCAmelCase  :	Optional[int]							  =	hf_hub_download(_lowercase			,       _lowercase	)
							UpperCAmelCase  :	int							  =	torch.load(_lowercase			,       map_location="""cpu"""	)
							UpperCAmelCase  :	List[str]							  =	convert_state_dict(_lowercase	)

							# 4. Split in shards and save
							UpperCAmelCase  :	Union[str, Any]							  =	shard_checkpoint(_lowercase	)
							for shard_file, shard in shards.items():
														torch.save(_lowercase			,       os.path.join(_lowercase			,       _lowercase	)	)

							if index is not None:
														UpperCAmelCase  :	Optional[Any]							  =	os.path.join(_lowercase			,       _lowercase	)
														# Save the index as well
														with open(_lowercase			,       """w"""			,       encoding="""utf-8"""	) as f:
																					UpperCAmelCase  :	Any							  =	json.dumps(_lowercase			,       indent=2			,       sort_keys=_lowercase	) + """\n"""
																					f.write(_lowercase	)

														# 5. Clean up shards (for some reason the file PyTorch saves take the same space as the whole state_dict
														print(
														    """Cleaning up shards. This may error with an OOM error, it this is the case don't worry you still have converted the model."""	)
														UpperCAmelCase  :	Optional[int]							  =	list(shards.keys()	)

														del state_dict
														del shards
														gc.collect()

														for shard_file in shard_files:
																					UpperCAmelCase  :	Union[str, Any]							  =	torch.load(os.path.join(_lowercase			,       _lowercase	)	)
																					torch.save({k: v.cpu().clone() for k, v in state_dict.items()}			,       os.path.join(_lowercase			,       _lowercase	)	)

							del state_dict
							gc.collect()

							if push_to_hub:
														if model_name is None:
																					raise ValueError("""Please provide a `model_name` to push the model to the Hub."""	)
														UpperCAmelCase  :	Any							  =	AutoModelForCausalLM.from_pretrained(_lowercase	)
														model.push_to_hub(_lowercase			,       max_shard_size="""2GB"""	)
														tokenizer.push_to_hub(_lowercase	)


if __name__ == "__main__":
			a     :    Any      						=					argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    """--repo_id""", default=None, type=str, required=True, help="""Repo ID from which to pull the checkpoint."""
			)
			parser.add_argument(
			    """--checkpoint_file""", default=None, type=str, required=True, help="""Name of the checkpoint file in the repo."""
			)
			parser.add_argument(
			    """--output_dir""", default=None, type=str, required=True, help="""Where to save the converted model."""
			)
			parser.add_argument(
			    """--tokenizer_file""",
			    default=None,
			    type=str,
			    help="""Path to the tokenizer file to use (if not provided, only the model is converted).""",
			)
			parser.add_argument(
			    """--size""",
			    default=None,
			    type=str,
			    help="""Size of the model. Will be inferred from the `checkpoint_file` if not passed.""",
			)
			parser.add_argument(
			    """--push_to_hub""",
			    action="""store_true""",
			    help="""Push to the Hub the converted model.""",
			)
			parser.add_argument(
			    """--model_name""",
			    default=None,
			    type=str,
			    help="""Name of the pushed model on the Hub, including the username / organization.""",
			)

			a     :    Any      						=					parser.parse_args()
			convert_rmkv_checkpoint_to_hf_format(
			    args.repo_id,
			    args.checkpoint_file,
			    args.output_dir,
			    size=args.size,
			    tokenizer_file=args.tokenizer_file,
			    push_to_hub=args.push_to_hub,
			    model_name=args.model_name,
			)






'''simple docstring'''




from typing import List, Union

import numpy as np

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, logging
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, ChunkPipeline


a     :    List[str]      						=					logging.get_logger(__name__)

class     UpperCamelCase_				(							__magic_name__    ):





	def 				_lowercase(						self	,	A					)    ->    Optional[int]:
								if isinstance(A	,	A					):
															UpperCAmelCase  :	Union[str, Any]							  =	[label.strip() for label in labels.split(""","""					) if label.strip()]
								return labels






	def __call__(						self	,	A	,	A	,	A					)    ->    str:
								if len(A					) == 0 or len(A					) == 0:
															raise ValueError("""You must include at least one label and at least one sequence."""					)
								if hypothesis_template.format(labels[0]					) == hypothesis_template:
															raise ValueError(
															    (
															        """The provided hypothesis_template \"{}\" was not able to be formatted with the target labels. """
															        """Make sure the passed template includes formatting syntax such as {{}} where the label should go."""
															    ).format(A					)					)

								if isinstance(A	,	A					):
															UpperCAmelCase  :	Tuple							  =	[sequences]

								UpperCAmelCase  :	Optional[Any]							  =	[]
								for sequence in sequences:
															sequence_pairs.extend([[sequence, hypothesis_template.format(A					)] for label in labels]					)

								return sequence_pairs, sequences




@add_end_docstrings(__magic_name__    )
class     UpperCamelCase_				(							__magic_name__    ):





	def __init__(						self	,	A=ZeroShotClassificationArgumentHandler()	,	*A	,	**A					)    ->    Optional[int]:
								UpperCAmelCase  :	Tuple							  =	args_parser
								super().__init__(*A	,	**A					)
								if self.entailment_id == -1:
															logger.warning(
															    """Failed to determine 'entailment' label id from the label2id mapping in the model config. Setting to """
															    """-1. Define a descriptive label2id mapping in the model config to ensure correct outputs."""					)





	@property
	def 				_lowercase(						self					)    ->    List[Any]:
								for label, ind in self.model.config.labelaid.items():
															if label.lower().startswith("""entail"""					):
																						return ind
								return -1





	def 				_lowercase(						self	,	A	,	A=True	,	A=True	,	A=TruncationStrategy.ONLY_FIRST	,	**A					)    ->    str:
								UpperCAmelCase  :	Tuple							  =	self.framework
								if self.tokenizer.pad_token is None:
															# Override for tokenizers not supporting padding
															logger.error(
															    """Tokenizer was not supporting padding necessary for zero-shot, attempting to use """
															    """ `pad_token=eos_token`"""					)
															UpperCAmelCase  :	Any							  =	self.tokenizer.eos_token
								try:
															UpperCAmelCase  :	Tuple							  =	self.tokenizer(
															    A	,	add_special_tokens=A	,	return_tensors=A	,	padding=A	,	truncation=A	,	)
								except Exception as e:
															if "too short" in str(A					):
																						# tokenizers might yell that we want to truncate
																						# to a value that is not even reached by the input.
																						# In that case we don't want to truncate.
																						# It seems there's not a really better way to catch that
																						# exception.

																						UpperCAmelCase  :	List[str]							  =	self.tokenizer(
																						    A	,	add_special_tokens=A	,	return_tensors=A	,	padding=A	,	truncation=TruncationStrategy.DO_NOT_TRUNCATE	,	)
															else:
																						raise e

								return inputs





	def 				_lowercase(						self	,	**A					)    ->    Tuple:
								if kwargs.get("""multi_class"""	,	A					) is not None:
															UpperCAmelCase  :	Any							  =	kwargs["""multi_class"""]
															logger.warning(
															    """The `multi_class` argument has been deprecated and renamed to `multi_label`. """
															    """`multi_class` will be removed in a future version of Transformers."""					)
								UpperCAmelCase  :	int							  =	{}
								if "candidate_labels" in kwargs:
															UpperCAmelCase  :	Tuple							  =	self._args_parser._parse_labels(kwargs["""candidate_labels"""]					)
								if "hypothesis_template" in kwargs:
															UpperCAmelCase  :	List[Any]							  =	kwargs["""hypothesis_template"""]

								UpperCAmelCase  :	Dict							  =	{}
								if "multi_label" in kwargs:
															UpperCAmelCase  :	Union[str, Any]							  =	kwargs["""multi_label"""]
								return preprocess_params, {}, postprocess_params





	def __call__(						self	,	A	,	*A	,	**A	,	)    ->    Tuple:
								if len(A					) == 0:
															pass
								elif len(A					) == 1 and "candidate_labels" not in kwargs:
															UpperCAmelCase  :	Optional[Any]							  =	args[0]
								else:
															raise ValueError(f'''Unable to understand extra arguments {args}'''					)

								return super().__call__(A	,	**A					)





	def 				_lowercase(						self	,	A	,	A=None	,	A="This example is {}."					)    ->    List[Any]:
								UpperCAmelCase  ,  UpperCAmelCase  :	List[Any]							  =	self._args_parser(A	,	A	,	A					)

								for i, (candidate_label, sequence_pair) in enumerate(zip(A	,	A					)					):
															UpperCAmelCase  :	Any							  =	self._parse_and_tokenize([sequence_pair]					)

															yield {
															    "candidate_label": candidate_label,
															    "sequence": sequences[0],
															    "is_last": i == len(A					) - 1,
															    **model_input,
															}





	def 				_lowercase(						self	,	A					)    ->    Optional[int]:
								UpperCAmelCase  :	Optional[Any]							  =	inputs["""candidate_label"""]
								UpperCAmelCase  :	Tuple							  =	inputs["""sequence"""]
								UpperCAmelCase  :	List[Any]							  =	{k: inputs[k] for k in self.tokenizer.model_input_names}
								UpperCAmelCase  :	Tuple							  =	self.model(**A					)

								UpperCAmelCase  :	Optional[int]							  =	{
								    """candidate_label""": candidate_label,
								    """sequence""": sequence,
								    """is_last""": inputs["""is_last"""],
								    **outputs,
								}
								return model_outputs






	def 				_lowercase(						self	,	A	,	A=False					)    ->    List[str]:
								UpperCAmelCase  :	Dict							  =	[outputs["""candidate_label"""] for outputs in model_outputs]
								UpperCAmelCase  :	List[Any]							  =	[outputs["""sequence"""] for outputs in model_outputs]
								UpperCAmelCase  :	List[Any]							  =	np.concatenate([output["""logits"""].numpy() for output in model_outputs]					)
								UpperCAmelCase  :	Optional[Any]							  =	logits.shape[0]
								UpperCAmelCase  :	int							  =	len(A					)
								UpperCAmelCase  :	List[Any]							  =	N // n
								UpperCAmelCase  :	int							  =	logits.reshape((num_sequences, n, -1)					)

								if multi_label or len(A					) == 1:
															# softmax over the entailment vs. contradiction dim for each label independently
															UpperCAmelCase  :	str							  =	self.entailment_id
															UpperCAmelCase  :	str							  =	-1 if entailment_id == 0 else 0
															UpperCAmelCase  :	Optional[Any]							  =	reshaped_outputs[..., [contradiction_id, entailment_id]]
															UpperCAmelCase  :	int							  =	np.exp(A					) / np.exp(A					).sum(-1	,	keepdims=A					)
															UpperCAmelCase  :	int							  =	scores[..., 1]
								else:
															# softmax the "entailment" logits over all candidate labels
															UpperCAmelCase  :	Dict							  =	reshaped_outputs[..., self.entailment_id]
															UpperCAmelCase  :	Optional[int]							  =	np.exp(A					) / np.exp(A					).sum(-1	,	keepdims=A					)

								UpperCAmelCase  :	int							  =	list(reversed(scores[0].argsort()					)					)
								return {
								    "sequence": sequences[0],
								    "labels": [candidate_labels[i] for i in top_inds],
								    "scores": scores[0, top_inds].tolist(),
								}

