






"""simple docstring"""




import argparse
import os

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_task_guides.py
UpperCamelCase							       =    """src/transformers"""
UpperCamelCase							       =    """docs/source/en/tasks"""






def        lowerCAmelCase (      UpperCamelCase_:							List[str]     , UpperCamelCase_:							List[str]     , UpperCamelCase_:							Dict					)       ->							int:







  '''simple docstring'''







  with open(UpperCamelCase_     , "r"     , encoding="utf-8"     , newline="\n"					) as f:
    _a							=							f.readlines()
  # Find the start prompt.
  _a							=							0
  while not lines[start_index].startswith(UpperCamelCase_					):
    start_index += 1
  start_index += 1

  _a							=							start_index
  while not lines[end_index].startswith(UpperCamelCase_					):
    end_index += 1
  end_index -= 1

  while len(lines[start_index]					) <= 1:
    start_index += 1
  while len(lines[end_index]					) <= 1:
    end_index -= 1
  end_index += 1
  return "".join(lines[start_index:end_index]					), start_index, end_index, lines


# This is to make sure the transformers module imported is the one in the repo.
UpperCamelCase							       =    direct_transformers_import(TRANSFORMERS_PATH)

UpperCamelCase							       =    {
    """asr.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,
    """audio_classification.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,
    """language_modeling.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    """image_classification.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
    """masked_language_modeling.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_MASKED_LM_MAPPING_NAMES,
    """multiple_choice.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES,
    """object_detection.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,
    """question_answering.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
    """semantic_segmentation.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,
    """sequence_classification.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
    """summarization.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    """token_classification.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,
    """translation.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    """video_classification.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES,
    """document_question_answering.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,
    """monocular_depth_estimation.md""": transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES,
}

# This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any
# `MODEL_MAPPING_NAMES` or any `MODEL_FOR_XXX_MAPPING_NAMES`).
UpperCamelCase							       =    {
    """summarization.md""": ("""nllb""",),
    """translation.md""": ("""nllb""",),
}






def        lowerCAmelCase (      UpperCamelCase_:							Optional[int]					)       ->							Dict:







  '''simple docstring'''







  _a							=							TASK_GUIDE_TO_MODELS[task_guide]
  _a							=							SPECIAL_TASK_GUIDE_TO_MODEL_TYPES.get(UpperCamelCase_     , set()					)
  _a							=							{
      code: name
      for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
      if (code in model_maping_names or code in special_model_types)
  }
  return ", ".join([f'''[{name}](../model_doc/{code})''' for code, name in model_names.items()]					) + "\n"






def        lowerCAmelCase (      UpperCamelCase_:							Any     , UpperCamelCase_:							str=False					)       ->							Union[str, Any]:







  '''simple docstring'''







  _a							=							_find_text_in_file(
      filename=os.path.join(UpperCamelCase_     , UpperCamelCase_					)     , start_prompt="<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->"     , end_prompt="<!--End of the generated tip-->"     , )

  _a							=							get_model_list_for_task(UpperCamelCase_					)

  if current_list != new_list:
    if overwrite:
      with open(os.path.join(UpperCamelCase_     , UpperCamelCase_					)     , "w"     , encoding="utf-8"     , newline="\n"					) as f:
        f.writelines(lines[:start_index] + [new_list] + lines[end_index:]					)
    else:
      raise ValueError(
          f'''The list of models that can be used in the {task_guide} guide needs an update. Run `make fix-copies`'''
          " to fix this."					)


if __name__ == "__main__":
     UpperCamelCase							       =    argparse.ArgumentParser()
     parser.add_argument("""--fix_and_overwrite""", action="""store_true""", help="""Whether to fix inconsistencies.""")
     UpperCamelCase							       =    parser.parse_args()

     for task_guide in TASK_GUIDE_TO_MODELS.keys():
          check_model_list_for_task(task_guide, args.fix_and_overwrite)










"""simple docstring"""




import argparse
import json
import os
from collections import OrderedDict

import numpy as np
import tensorflow as tf
import torch






def        lowerCAmelCase (      UpperCamelCase_:							Dict					)       ->							Any:







  '''simple docstring'''







  _a							=	os.path.join(args.tf_model_dir     , "parameters.json"					)
  _a							=	json.loads(open(UpperCamelCase_					).read()					)
  if not params:
    raise ValueError(
        f'''It seems that the json file at {parameter_file} is empty. Make sure you have a correct json file.'''					)
  if not args.output.endswith(".pt"					):
    _a							=	args.output + ".pt"
  _a							=	OrderedDict()
  with tf.device("/CPU:0"					):
    _a							=	tf.train.load_checkpoint(args.tf_model_dir					)
    _a							=	reader.get_variable_to_shape_map()
    for key_name in shapes.keys():
      _a							=	reader.get_tensor(UpperCamelCase_					).astype(np.floataa					)
      if key_name.endswith("/adam_m"					) or key_name.endswith("/adam_v"					):
        continue
      if key_name.startswith("pasts/"					):
        if key_name.startswith("pasts/mlp"					):
          _a							=	int(key_name[9]					)
        elif key_name.startswith("pasts/out"					):
          _a							=	8
        _a							=	"model.sqout.%d.weight" % (player * 2)  # enter to nn.Sequencial with Tanh, so 2 at a time
        _a							=	vnp.transpose([1, 0]					).copy()  # Mesh-Tensorflow is a diagonal matrix
        _a							=	torch.tensor(UpperCamelCase_					)
      elif key_name.startswith("model/moe"					):
        _a							=	int(key_name[9:].split("/"					)[0]					)
        if key_name.endswith("/switch_gating/kernel"					):
          _a							=	"model.blocks.%d.feed_forward.mlp.router.classifier.weight" % player
          _a							=	vnp.transpose([1, 0]					).copy()  # Mesh-Tensorflow is a diagonal matrix
          _a							=	torch.tensor(UpperCamelCase_					)
        elif key_name.endswith("/softmlp/kernel"					):
          _a							=	"model.blocks.%d.feed_forward.soft_bypass_mlp.weight" % player
          _a							=	vnp.transpose([1, 0]					).copy()  # Mesh-Tensorflow is a diagonal matrix
          _a							=	torch.tensor(UpperCamelCase_					)
        elif key_name.endswith("/wo/kernel"					) or key_name.endswith("/wi/kernel"					):
          _a							=	key_name[-9:-7]
          for i in range(16					):
            _a							=	"model.blocks.%d.feed_forward.mlp.experts.expert_%d.%s.weight" % (player, i, nlayer)
            _a							=	(
                vnp[i].transpose([1, 0]					).copy()
            )  # In Mesh-Tensorflow, it is one array, so it is divided
            _a							=	torch.tensor(UpperCamelCase_					)
      elif key_name.startswith("model/mlp"					):
        _a							=	int(key_name[9:].split("/"					)[0]					)
        if key_name.endswith("/p1/kernel"					):
          _a							=	"model.blocks.%d.feed_forward.mlp.wi.weight" % player
          _a							=	vnp.transpose([1, 0]					).copy()  # Mesh-Tensorflow is a diagonal matrix
          _a							=	torch.tensor(UpperCamelCase_					)
        elif key_name.endswith("/p1/bias"					):
          _a							=	"model.blocks.%d.feed_forward.mlp.wi.bias" % player
          _a							=	vnp.copy()  # same because it is one dimensional
          _a							=	torch.tensor(UpperCamelCase_					)
        elif key_name.endswith("/p2/kernel"					):
          _a							=	"model.blocks.%d.feed_forward.mlp.wo.weight" % player
          _a							=	vnp.transpose([1, 0]					).copy()  # Mesh-Tensorflow is a diagonal matrix
          _a							=	torch.tensor(UpperCamelCase_					)
        elif key_name.endswith("/p2/bias"					):
          _a							=	"model.blocks.%d.feed_forward.mlp.wo.bias" % player
          _a							=	vnp.copy()  # same because it is one dimensional
          _a							=	torch.tensor(UpperCamelCase_					)
      elif key_name.startswith("model/ln"					):
        _a							=	int(key_name[8:].split("/"					)[0]					)
        if key_name.endswith("/b"					):
          _a							=	"model.blocks.%d.feed_forward.norm.bias" % player
          _a							=	vnp.copy()  # same because it is one dimensional
          _a							=	torch.tensor(UpperCamelCase_					)
        elif key_name.endswith("/g"					):
          _a							=	"model.blocks.%d.feed_forward.norm.weight" % player
          _a							=	vnp.copy()  # same because it is one dimensional
          _a							=	torch.tensor(UpperCamelCase_					)
      elif key_name.startswith("model/att"					):
        _a							=	int(key_name[9:].split("/"					)[0]					)
        if key_name.endswith("/qkv/kernel"					):
          _a							=	vnp.copy()  # Compute same dimension as Mesh-tensorflow using einsum
          _a							=	state[:, 0, :, :]
          _a							=	state[:, 1, :, :]
          _a							=	state[:, 2, :, :]
          _a							=	(
              state_q.reshape([state_q.shape[0], state_q.shape[1] * state_q.shape[2]]					)
              .transpose([1, 0]					)
              .copy()
          )  # Mesh-Tensorflow is a diagonal matrix
          _a							=	(
              state_k.reshape([state_k.shape[0], state_k.shape[1] * state_k.shape[2]]					)
              .transpose([1, 0]					)
              .copy()
          )  # Mesh-Tensorflow is a diagonal matrix
          _a							=	(
              state_v.reshape([state_v.shape[0], state_v.shape[1] * state_v.shape[2]]					)
              .transpose([1, 0]					)
              .copy()
          )  # Mesh-Tensorflow is a diagonal matrix
          _a							=	"model.blocks.%d.self_attn.self_attn.q_proj.weight" % player
          _a							=	torch.tensor(UpperCamelCase_					)
          _a							=	"model.blocks.%d.self_attn.self_attn.k_proj.weight" % player
          _a							=	torch.tensor(UpperCamelCase_					)
          _a							=	"model.blocks.%d.self_attn.self_attn.v_proj.weight" % player
          _a							=	torch.tensor(UpperCamelCase_					)
        elif key_name.endswith("/o/kernel"					):
          _a							=	"model.blocks.%d.self_attn.self_attn.out_proj.weight" % player
          _a							=	(
              vnp.reshape([vnp.shape[0] * vnp.shape[1], vnp.shape[2]]					).transpose([1, 0]					).copy()
          )  # Mesh-Tensorflow is a diagonal matrix
          _a							=	torch.tensor(UpperCamelCase_					)
      elif key_name.startswith("model/an"					):
        _a							=	int(key_name[8:].split("/"					)[0]					)
        if key_name.endswith("/b"					):
          _a							=	"model.blocks.%d.self_attn.norm.bias" % player
          _a							=	vnp.copy()  # same because it is one dimensional
          _a							=	torch.tensor(UpperCamelCase_					)
        elif key_name.endswith("/g"					):
          _a							=	"model.blocks.%d.self_attn.norm.weight" % player
          _a							=	vnp.copy()  # same because it is one dimensional
          _a							=	torch.tensor(UpperCamelCase_					)
      elif (
          key_name.startswith("model/wte"					)
          or key_name.startswith("model/wpe"					)
          or key_name.startswith("model/ete"					)
      ):
        _a							=	{"wte": "embed_tokens", "wpe": "position_embeddings", "ete": "extra_position_embeddings"}[
            key_name[-3:]
        ]
        _a							=	"model.%s.weight" % nlayer
        _a							=	vnp.copy()  # same in embedded
        _a							=	torch.tensor(UpperCamelCase_					)
        if key_name.startswith("model/wte"					):
          _a							=	"lm_head.weight"
          _a							=	vnp.copy()  # same in embedded
          _a							=	torch.tensor(UpperCamelCase_					)
      elif key_name.startswith("model/wob"					):
        _a							=	"final_logits_bias"
        _a							=	vnp.copy()  # same in embedded
        _a							=	state.reshape((1, -1)					)
        _a							=	torch.tensor(UpperCamelCase_					)
      elif key_name == "model/dense/kernel":
        _a							=	"model.last_project.weight"
        _a							=	vnp.transpose([1, 0]					).copy()  # Mesh-Tensorflow is a diagonal matrix
        _a							=	torch.tensor(UpperCamelCase_					)
      elif key_name == "model/dense_1/bias":
        _a							=	"model.last_project.bias"
        _a							=	vnp.copy()  # same because it is one dimensional
        _a							=	torch.tensor(UpperCamelCase_					)
  torch.save(UpperCamelCase_     , args.output					)


if __name__ == "__main__":
     UpperCamelCase							       =    argparse.ArgumentParser(
         description="""model converter.""", formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
     parser.add_argument("""--tf_model_dir""", metavar="""PATH""", type=str, required=True, help="""import model""")
     parser.add_argument("""--output""", metavar="""PATH""", type=str, required=True, help="""output model""")
     UpperCamelCase							       =    parser.parse_args()
     convert_tf_gptsan_to_pt(args)


