



import argparse
import os

import torch

from transformers.utils import WEIGHTS_NAME


a_			=					["""small""", """medium""", """large"""]

a_			=					"""lm_head.decoder.weight"""
a_			=					"""lm_head.weight"""





def 		a__			(		_UpperCamelCase			: str ,_UpperCamelCase			: str							):
   __lowerCamelCase													=    torch.load(_UpperCamelCase							)
   __lowerCamelCase													=    d.pop(_UpperCamelCase							)
   os.makedirs(_UpperCamelCase ,exist_ok=_UpperCamelCase							)
   torch.save(_UpperCamelCase ,os.path.join(_UpperCamelCase ,_UpperCamelCase							)							)


if __name__ == "__main__":
  a_			=					argparse.ArgumentParser()
  parser.add_argument("""--dialogpt_path""", default=""".""", type=str)
  a_			=					parser.parse_args()
  for MODEL in DIALOGPT_MODELS:
    a_			=					os.path.join(args.dialogpt_path, f"{MODEL}_ft.pkl")
    a_			=					f"./DialoGPT-{MODEL}"
    convert_dialogpt_checkpoint(
        checkpoint_path,
        pytorch_dump_folder_path,
    )











import unicodedata
from dataclasses import dataclass
from typing import Optional, Union

import numpy as np

from transformers.data.data_collator import DataCollatorMixin
from transformers.file_utils import PaddingStrategy
from transformers.tokenization_utils_base import PreTrainedTokenizerBase





def 		a__			(		_UpperCamelCase			: Tuple ,_UpperCamelCase			: Any ,_UpperCamelCase			: Union[str, Any] ,_UpperCamelCase			: Any							):
   if isinstance(_UpperCamelCase ,_UpperCamelCase							):
      __lowerCamelCase													=    np.full((len(_UpperCamelCase							), sequence_length, 2) ,_UpperCamelCase							)
   else:
      __lowerCamelCase													=    np.full((len(_UpperCamelCase							), sequence_length) ,_UpperCamelCase							)

   for i, tensor in enumerate(_UpperCamelCase							):
      if padding_side == "right":
         if isinstance(_UpperCamelCase ,_UpperCamelCase							):
            __lowerCamelCase													=    tensor[:sequence_length]
         else:
            __lowerCamelCase													=    tensor[:sequence_length]
      else:
         if isinstance(_UpperCamelCase ,_UpperCamelCase							):
            __lowerCamelCase													=    tensor[:sequence_length]
         else:
            __lowerCamelCase													=    tensor[:sequence_length]

   return out_tensor.tolist()





def 		a__			(		_UpperCamelCase			: Dict							):
   __lowerCamelCase													=    ord(_UpperCamelCase							)
   if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 1_23 and cp <= 1_26):
      return True
   __lowerCamelCase													=    unicodedata.category(_UpperCamelCase							)
   if cat.startswith('''P'''							):
      return True
   return False





@dataclass
class  __lowerCAmelCase     (	lowerCAmelCase__   ):
 lowerCAmelCase__       					=  42
 lowerCAmelCase__       					=  True
 lowerCAmelCase__       					=  None
 lowerCAmelCase__       					=  None
 lowerCAmelCase__       					=  -1_0_0
 lowerCAmelCase__       					=  "pt"




 def        lowerCamelCase      (						self	,				__UpperCAmelCase				):



    '''simple docstring'''
    import torch

    __lowerCamelCase													=    '''label''' if '''label''' in features[0].keys() else '''labels'''
    __lowerCamelCase													=    [feature[label_name] for feature in features] if label_name in features[0].keys() else None
    __lowerCamelCase													=    self.tokenizer.pad(
        __UpperCAmelCase	,				padding=self.padding	,				max_length=self.max_length	,				pad_to_multiple_of=self.pad_to_multiple_of	,				return_tensors='''pt''' if labels is None else None	,				)

    if labels is None:
       return batch

    __lowerCamelCase													=    torch.tensor(batch['''entity_ids''']				).shape[1]
    __lowerCamelCase													=    self.tokenizer.padding_side
    if padding_side == "right":
       __lowerCamelCase													=    [
           list(__UpperCAmelCase				) + [self.label_pad_token_id] * (sequence_length - len(__UpperCAmelCase				)) for label in labels
       ]
    else:
       __lowerCamelCase													=    [
           [self.label_pad_token_id] * (sequence_length - len(__UpperCAmelCase				)) + list(__UpperCAmelCase				) for label in labels
       ]

    __lowerCamelCase													=    [feature['''ner_tags'''] for feature in features]
    __lowerCamelCase													=    padding_tensor(__UpperCAmelCase	,				-1	,				__UpperCAmelCase	,				__UpperCAmelCase				)
    __lowerCamelCase													=    [feature['''original_entity_spans'''] for feature in features]
    __lowerCamelCase													=    padding_tensor(__UpperCAmelCase	,				(-1, -1)	,				__UpperCAmelCase	,				__UpperCAmelCase				)
    __lowerCamelCase													=    {k: torch.tensor(__UpperCAmelCase	,				dtype=torch.intaa				) for k, v in batch.items()}

    return batch






